File: scripts/build.py

import os
import sys
from pathlib import Path

import PyInstaller.__main__  # type: ignore
import pykakasi
import yt_dlp
import ytmusicapi

from spotdl._version import __version__

LOCALES_PATH = str((Path(ytmusicapi.__file__).parent / "locales"))
PYKAKASI_PATH = str((Path(pykakasi.__file__).parent / "data"))
YTDLP_PATH = str(Path(yt_dlp.__file__).parent / "__pyinstaller")

PyInstaller.__main__.run(
    [
        "spotdl/__main__.py",
        "--onefile",
        "--add-data",
        f"{LOCALES_PATH}{os.pathsep}ytmusicapi/locales",
        "--add-data",
        f"{PYKAKASI_PATH}{os.pathsep}pykakasi/data",
        f"--additional-hooks-dir={YTDLP_PATH}",
        "--name",
        f"spotdl-{__version__}-{sys.platform}",
        "--console",
    ]
)



File: scripts/docs/gen_ref_nav.py

"""Generate the code reference pages and navigation."""

from pathlib import Path

import mkdocs_gen_files

nav = mkdocs_gen_files.Nav()  # type: ignore

IGNORE = (
    ("_version",),
    # ('__init__',)
)

for path in Path("spotdl").glob("**/*.py"):
    module_path = path.relative_to("spotdl").with_suffix("")
    doc_path = path.relative_to("spotdl").with_suffix(".md")
    full_doc_path = Path("reference", doc_path)

    if module_path.parts in IGNORE:
        continue

    parts = tuple(module_path.parts)

    if parts[-1] == "__init__":
        if len(parts) != 1:
            parts = parts[:-1]

        doc_path = doc_path.with_name("index.md")
        full_doc_path = full_doc_path.with_name("index.md")
    elif parts[-1] == "__main__":
        continue

    nav[parts] = doc_path.as_posix()

    with mkdocs_gen_files.open(full_doc_path, "w") as fd:
        if parts == ("__init__",):
            fd.write("::: spotdl")
            continue

        IDENT = "spotdl." + ".".join(parts)
        fd.write(f"::: {IDENT}")

    mkdocs_gen_files.set_edit_path(full_doc_path, path)

with mkdocs_gen_files.open("reference/SUMMARY.md", "w") as nav_file:
    nav_file.writelines(nav.build_literate_nav())



File: spotdl/_version.py

"""
Version module for spotdl.
"""

__version__ = "4.2.8"



File: spotdl/__init__.py

"""
Init module for spotdl. This module contains the main entry point for spotdl.
And Spotdl class
"""

import asyncio
import concurrent.futures
import logging
from pathlib import Path
from typing import List, Optional, Tuple, Union

from spotdl._version import __version__
from spotdl.console import console_entry_point
from spotdl.download.downloader import Downloader
from spotdl.types.options import DownloaderOptionalOptions, DownloaderOptions
from spotdl.types.song import Song
from spotdl.utils.search import parse_query
from spotdl.utils.spotify import SpotifyClient

__all__ = ["Spotdl", "console_entry_point", "__version__"]

logger = logging.getLogger(__name__)


class Spotdl:
    """
    Spotdl class, which simplifies the process of downloading songs from Spotify.

    ```python
    from spotdl import Spotdl

    spotdl = Spotdl(client_id='your-client-id', client_secret='your-client-secret')

    songs = spotdl.search(['joji - test drive',
        'https://open.spotify.com/track/4cOdK2wGLETKBW3PvgPWqT'])

    results = spotdl.download_songs(songs)
    song, path = spotdl.download(songs[0])
    ```
    """

    def __init__(
        self,
        client_id: str,
        client_secret: str,
        user_auth: bool = False,
        cache_path: Optional[str] = None,
        no_cache: bool = False,
        headless: bool = False,
        downloader_settings: Optional[
            Union[DownloaderOptionalOptions, DownloaderOptions]
        ] = None,
        loop: Optional[asyncio.AbstractEventLoop] = None,
    ):
        """
        Initialize the Spotdl class

        ### Arguments
        - client_id: Spotify client id
        - client_secret: Spotify client secret
        - user_auth: If true, user will be prompted to authenticate
        - cache_path: Path to cache directory
        - no_cache: If true, no cache will be used
        - headless: If true, no browser will be opened
        - downloader_settings: Settings for the downloader
        - loop: Event loop to use
        """

        if downloader_settings is None:
            downloader_settings = {}

        # Initialize spotify client
        SpotifyClient.init(
            client_id=client_id,
            client_secret=client_secret,
            user_auth=user_auth,
            cache_path=cache_path,
            no_cache=no_cache,
            headless=headless,
        )

        # Initialize downloader
        self.downloader = Downloader(
            settings=downloader_settings,
            loop=loop,
        )

    def search(self, query: List[str]) -> List[Song]:
        """
        Search for songs.

        ### Arguments
        - query: List of search queries

        ### Returns
        - A list of Song objects

        ### Notes
        - query can be a list of song titles, urls, uris
        """

        return parse_query(
            query=query,
            threads=self.downloader.settings["threads"],
            use_ytm_data=self.downloader.settings["ytm_data"],
            playlist_numbering=self.downloader.settings["playlist_numbering"],
            album_type=self.downloader.settings["album_type"],
        )

    def get_download_urls(self, songs: List[Song]) -> List[Optional[str]]:
        """
        Get the download urls for a list of songs.

        ### Arguments
        - songs: List of Song objects

        ### Returns
        - A list of urls if successful.

        ### Notes
        - This function is multi-threaded.
        """

        urls: List[Optional[str]] = []
        with concurrent.futures.ThreadPoolExecutor(
            max_workers=self.downloader.settings["threads"]
        ) as executor:
            future_to_song = {
                executor.submit(self.downloader.search, song): song for song in songs
            }
            for future in concurrent.futures.as_completed(future_to_song):
                song = future_to_song[future]
                try:
                    data = future.result()
                    urls.append(data)
                except Exception as exc:
                    logger.error("%s generated an exception: %s", song, exc)

        return urls

    def download(self, song: Song) -> Tuple[Song, Optional[Path]]:
        """
        Download and convert song to the output format.

        ### Arguments
        - song: Song object

        ### Returns
        - A tuple containing the song and the path to the downloaded file if successful.
        """

        return self.downloader.download_song(song)

    def download_songs(self, songs: List[Song]) -> List[Tuple[Song, Optional[Path]]]:
        """
        Download and convert songs to the output format.

        ### Arguments
        - songs: List of Song objects

        ### Returns
        - A list of tuples containing the song and the path to the downloaded file if successful.
        """

        return self.downloader.download_multiple_songs(songs)



File: spotdl/__main__.py

"""
Main module for spotdl. Exports version and main function.
"""

from spotdl._version import __version__
from spotdl.console import console_entry_point

if __name__ == "__main__":
    console_entry_point()



File: spotdl/types/options.py

"""
This file contains types for spotdl/downloader/web modules.
Options types have all the fields marked as required.
Settings types have all the fields marked as optional.
"""

from typing import List, Optional, Union

from typing_extensions import TypedDict

__all__ = [
    "SpotifyOptions",
    "DownloaderOptions",
    "WebOptions",
    "SpotDLOptions",
    "SpotifyOptionalOptions",
    "DownloaderOptionalOptions",
    "WebOptionalOptions",
    "SpotDLOptionalOptions",
]


class SpotifyOptions(TypedDict):
    """
    Options used for initializing the Spotify client.
    """

    client_id: str
    client_secret: str
    auth_token: Optional[str]
    user_auth: bool
    headless: bool
    cache_path: str
    no_cache: bool
    max_retries: int
    use_cache_file: bool


class DownloaderOptions(TypedDict):
    """
    Options used for initializing the Downloader.
    """

    audio_providers: List[str]
    lyrics_providers: List[str]
    genius_token: str
    playlist_numbering: bool
    scan_for_songs: bool
    m3u: Optional[str]
    output: str
    overwrite: str
    search_query: Optional[str]
    ffmpeg: str
    bitrate: Optional[Union[str, int]]
    ffmpeg_args: Optional[str]
    format: str
    save_file: Optional[str]
    filter_results: bool
    album_type: Optional[str]
    threads: int
    cookie_file: Optional[str]
    restrict: Optional[str]
    print_errors: bool
    sponsor_block: bool
    preload: bool
    archive: Optional[str]
    load_config: bool
    log_level: str
    simple_tui: bool
    fetch_albums: bool
    id3_separator: str
    ytm_data: bool
    add_unavailable: bool
    generate_lrc: bool
    force_update_metadata: bool
    only_verified_results: bool
    sync_without_deleting: bool
    max_filename_length: Optional[int]
    yt_dlp_args: Optional[str]
    detect_formats: Optional[List[str]]
    save_errors: Optional[str]
    ignore_albums: Optional[List[str]]
    proxy: Optional[str]
    skip_explicit: Optional[bool]
    log_format: Optional[str]
    redownload: Optional[bool]
    skip_album_art: Optional[bool]
    create_skip_file: Optional[bool]
    respect_skip_file: Optional[bool]
    sync_remove_lrc: Optional[bool]


class WebOptions(TypedDict):
    """
    Options used for initializing the Web server.
    """

    web_use_output_dir: bool
    port: int
    host: str
    keep_alive: bool
    enable_tls: bool
    key_file: Optional[str]
    cert_file: Optional[str]
    ca_file: Optional[str]
    allowed_origins: Optional[List[str]]
    keep_sessions: bool
    force_update_gui: bool
    web_gui_repo: Optional[str]
    web_gui_location: Optional[str]


class SpotDLOptions(SpotifyOptions, DownloaderOptions, WebOptions):
    """
    Options used for initializing the SpotDL client.
    """


class SpotifyOptionalOptions(TypedDict, total=False):
    """
    Options used for initializing the Spotify client.
    """

    client_id: str
    client_secret: str
    auth_token: Optional[str]
    user_auth: bool
    headless: bool
    cache_path: str
    no_cache: bool
    max_retries: int
    use_cache_file: bool


class DownloaderOptionalOptions(TypedDict, total=False):
    """
    Options used for initializing the Downloader.
    """

    audio_providers: List[str]
    lyrics_providers: List[str]
    genius_token: str
    playlist_numbering: bool
    scan_for_songs: bool
    m3u: Optional[str]
    output: str
    overwrite: str
    search_query: Optional[str]
    ffmpeg: str
    bitrate: Optional[Union[str, int]]
    ffmpeg_args: Optional[str]
    format: str
    save_file: Optional[str]
    filter_results: bool
    album_type: Optional[str]
    threads: int
    cookie_file: Optional[str]
    restrict: Optional[str]
    print_errors: bool
    sponsor_block: bool
    preload: bool
    archive: Optional[str]
    load_config: bool
    log_level: str
    simple_tui: bool
    fetch_albums: bool
    id3_separator: str
    ytm_data: bool
    add_unavailable: bool
    generate_lrc: bool
    force_update_metadata: bool
    only_verified_results: bool
    sync_without_deleting: bool
    max_filename_length: Optional[int]
    yt_dlp_args: Optional[str]
    detect_formats: Optional[List[str]]
    save_errors: Optional[str]
    proxy: Optional[str]
    skip_explicit: Optional[bool]
    log_format: Optional[str]
    redownload: Optional[bool]
    skip_album_art: Optional[bool]
    create_skip_file: Optional[bool]
    respect_skip_file: Optional[bool]
    sync_remove_lrc: Optional[bool]


class WebOptionalOptions(TypedDict, total=False):
    """
    Options used for initializing the Web server.
    """

    web_use_output_dir: bool
    port: int
    host: str
    keep_alive: bool
    enable_tls: bool
    key_file: Optional[str]
    cert_file: Optional[str]
    ca_file: Optional[str]
    allowed_origins: Optional[str]
    keep_sessions: bool
    force_update_gui: bool
    web_gui_repo: Optional[str]
    web_gui_location: Optional[str]


class SpotDLOptionalOptions(
    SpotifyOptionalOptions, DownloaderOptionalOptions, WebOptionalOptions
):
    """
    Options used for initializing the SpotDL client.
    This type is modified to not require all the fields.
    """



File: spotdl/types/__init__.py

"""
Types for the spotdl package.
"""



File: spotdl/types/album.py

"""
Artist module for retrieving artist data from Spotify.
"""

from dataclasses import dataclass
from typing import Any, Dict, List, Tuple

from spotdl.types.song import Song, SongList
from spotdl.utils.spotify import SpotifyClient

__all__ = ["Album", "AlbumError"]


class AlbumError(Exception):
    """
    Base class for all exceptions related to albums.
    """


@dataclass(frozen=True)
class Album(SongList):
    """
    Album class for retrieving album data from Spotify.
    """

    artist: Dict[str, Any]

    @staticmethod
    def get_metadata(url: str) -> Tuple[Dict[str, Any], List[Song]]:
        """
        Get metadata for album.

        ### Arguments
        - url: The URL of the album.

        ### Returns
        - A dictionary with metadata.
        """

        spotify_client = SpotifyClient()

        album_metadata = spotify_client.album(url)
        if album_metadata is None:
            raise AlbumError(
                "Couldn't get metadata, check if you have passed correct album id"
            )

        metadata = {
            "name": album_metadata["name"],
            "artist": album_metadata["artists"][0],
            "url": url,
        }

        album_response = spotify_client.album_tracks(url)
        if album_response is None:
            raise AlbumError(
                "Couldn't get metadata, check if you have passed correct album id"
            )

        tracks = album_response["items"]

        # Get all tracks from album
        while album_response["next"]:
            album_response = spotify_client.next(album_response)

            # Failed to get response, break the loop
            if album_response is None:
                break

            tracks.extend(album_response["items"])

        if album_response is None:
            raise AlbumError(f"Failed to get album response: {url}")

        songs = []
        for track in tracks:
            if not isinstance(track, dict) or track.get("is_local"):
                continue

            release_date = album_metadata["release_date"]
            artists = artists = [artist["name"] for artist in track["artists"]]

            song = Song.from_missing_data(
                name=track["name"],
                artists=artists,
                artist=artists[0],
                album_id=album_metadata["id"],
                album_name=album_metadata["name"],
                album_artist=album_metadata["artists"][0]["name"],
                album_type=album_metadata["album_type"],
                disc_number=track["disc_number"],
                disc_count=int(album_metadata["tracks"]["items"][-1]["disc_number"]),
                duration=int(track["duration_ms"] / 1000),
                year=release_date[:4],
                date=release_date,
                track_number=track["track_number"],
                tracks_count=album_metadata["total_tracks"],
                song_id=track["id"],
                explicit=track["explicit"],
                publisher=album_metadata["label"],
                url=track["external_urls"]["spotify"],
                cover_url=(
                    max(
                        album_metadata["images"], key=lambda i: i["width"] * i["height"]
                    )["url"]
                    if album_metadata["images"]
                    else None
                ),
                copyright_text=(
                    album_metadata["copyrights"][0]["text"]
                    if album_metadata["copyrights"]
                    else None
                ),
            )

            songs.append(song)

        return metadata, songs



File: spotdl/types/artist.py

"""
Artist module for retrieving artist data from Spotify.
"""

from dataclasses import dataclass
from typing import Any, Dict, List, Set, Tuple

from spotdl.types.album import Album
from spotdl.types.song import Song, SongList
from spotdl.utils.formatter import slugify
from spotdl.utils.spotify import SpotifyClient

__all__ = ["Artist", "ArtistError"]


class ArtistError(Exception):
    """
    Base class for all exceptions related to artists.
    """


@dataclass(frozen=True)
class Artist(SongList):
    """
    Artist class.
    Contains all the information about an artist.
    Frozen to prevent accidental modification.
    """

    genres: List[str]
    albums: List[Album]

    @staticmethod
    def get_metadata(url: str) -> Tuple[Dict[str, Any], List[Song]]:
        """
        Get metadata for artist.

        ### Arguments
        - url: The URL of the artist.

        ### Returns
        - Dict with metadata for artist.
        """

        # query spotify for artist details
        spotify_client = SpotifyClient()

        # get artist info
        raw_artist_meta = spotify_client.artist(url)

        if raw_artist_meta is None:
            raise ArtistError(
                "Couldn't get metadata, check if you have passed correct artist id"
            )

        artist_albums = spotify_client.artist_albums(url, album_type="album,single")
        # check if there is response
        if not artist_albums:
            raise ArtistError(
                "Couldn't get albums, check if you have passed correct artist id"
            )

        # get artist albums and remove duplicates
        # duplicates can occur if the artist has the same album available in
        # different countries
        albums: List[str] = []
        known_albums: Set[str] = set()
        for album in artist_albums["items"]:
            albums.append(album["external_urls"]["spotify"])
            known_albums.add(slugify(album["name"]))

        # Fetch all artist albums
        while artist_albums and artist_albums["next"]:
            artist_albums = spotify_client.next(artist_albums)
            if artist_albums is None:
                break

            for album in artist_albums["items"]:
                album_name = slugify(album["name"])

                if album_name not in known_albums:
                    albums.append(album["external_urls"]["spotify"])
                    known_albums.add(album_name)

        songs = []
        for album in albums:
            album_obj = Album.from_url(album, fetch_songs=False)
            songs.extend(album_obj.songs)

        # Very aggressive deduplication
        songs_list = []
        songs_names = set()
        for song in songs:
            slug_name = slugify(song.name)
            if song.name not in songs_names:
                songs_list.append(song)
                songs_names.add(slug_name)

        metadata = {
            "name": raw_artist_meta["name"],
            "genres": raw_artist_meta["genres"],
            "url": url,
            "albums": albums,
        }

        return metadata, songs_list



File: spotdl/types/result.py

"""
Result is a class that contains all the information about a result from search
performed by audio provider.
"""

import json
from dataclasses import asdict, dataclass
from typing import Any, Dict, Optional, Tuple

__all__ = ["Result"]


@dataclass(frozen=True, eq=True)
class Result:
    """
    Result is a class that contains all the information about a result from search
    perfoermed by audio provider.
    """

    # Required fields
    source: str  # Source of the result
    url: str  # URL of the result
    verified: bool  # Whether the result is from a verified source or not
    name: str  # Name of the result
    duration: float  # Duration of the result in seconds
    author: str  # Author of the result
    result_id: str  # ID of the result

    # Search related fields
    isrc_search: Optional[bool] = (
        None  # Whether the result is from an ISRC search or not
    )
    search_query: Optional[str] = None  # The search query used to find the result

    # Optional fields
    artists: Optional[Tuple[str, ...]] = None
    views: Optional[int] = None
    explicit: Optional[bool] = None
    album: Optional[str] = None
    year: Optional[int] = None
    track_number: Optional[int] = None
    genre: Optional[str] = None
    lyrics: Optional[str] = None

    @classmethod
    def from_data_dump(cls, data: str) -> "Result":
        """
        Create a Result object from a data dump.

        ### Arguments
        - data: The data dump.

        ### Returns
        - The Song object.
        """

        # Create dict from json string
        data_dict = json.loads(data)

        # Return product object
        return cls(**data_dict)

    @classmethod
    def from_dict(cls, data: Dict[str, Any]) -> "Result":
        """
        Create a Song object from a dictionary.

        ### Arguments
        - data: The dictionary.

        ### Returns
        - The Song object.
        """

        # Return product object
        return cls(**data)

    @property
    def json(self) -> Dict[str, Any]:
        """
        Returns a dictionary of the song's data.

        ### Returns
        - The dictionary.
        """

        return asdict(self)



File: spotdl/types/playlist.py

"""
Playlist module for retrieving playlist data from Spotify.
"""

import logging
from dataclasses import dataclass
from typing import Any, Dict, List, Tuple

from spotdl.types.song import Song, SongList
from spotdl.utils.spotify import SpotifyClient

__all__ = ["Playlist", "PlaylistError"]

logger = logging.getLogger(__name__)


class PlaylistError(Exception):
    """
    Base class for all exceptions related to playlists.
    """


@dataclass(frozen=True)
class Playlist(SongList):
    """
    Playlist class for retrieving playlist data from Spotify.
    """

    description: str
    author_url: str
    author_name: str
    cover_url: str

    @staticmethod
    def get_metadata(url: str) -> Tuple[Dict[str, Any], List[Song]]:
        """
        Get metadata for a playlist.

        ### Arguments
        - url: The URL of the playlist.

        ### Returns
        - A dictionary with metadata.
        """

        spotify_client = SpotifyClient()

        playlist = spotify_client.playlist(url)
        if playlist is None:
            raise PlaylistError("Invalid playlist URL.")

        metadata = {
            "name": playlist["name"],
            "url": url,
            "description": playlist["description"],
            "author_url": playlist["external_urls"]["spotify"],
            "author_name": playlist["owner"]["display_name"],
            "cover_url": (
                max(
                    playlist["images"],
                    key=lambda i: (
                        0
                        if i["width"] is None or i["height"] is None
                        else i["width"] * i["height"]
                    ),
                )["url"]
                if (playlist.get("images") is not None and len(playlist["images"]) > 0)
                else ""
            ),
        }

        playlist_response = spotify_client.playlist_items(url)
        if playlist_response is None:
            raise PlaylistError(f"Wrong playlist id: {url}")

        # Get all tracks from playlist
        tracks = playlist_response["items"]
        while playlist_response["next"]:
            playlist_response = spotify_client.next(playlist_response)

            # Failed to get response, break the loop
            if playlist_response is None:
                break

            # Add tracks to the list
            tracks.extend(playlist_response["items"])

        songs = []
        for track_no, track in enumerate(tracks):
            if not isinstance(track, dict) or track.get("track") is None:
                continue

            track_meta = track["track"]

            if track_meta.get("is_local") or track_meta.get("type") != "track":
                logger.warning(
                    "Skipping track: %s local tracks and %s are not supported",
                    track_meta.get("id"),
                    track_meta.get("type"),
                )

                continue

            track_id = track_meta.get("id")
            if track_id is None or track_meta.get("duration_ms") == 0:
                continue

            album_meta = track_meta.get("album", {})
            release_date = album_meta.get("release_date")
            artists = [artist["name"] for artist in track_meta.get("artists", [])]
            song = Song.from_missing_data(
                name=track_meta["name"],
                artists=artists,
                artist=artists[0],
                album_id=album_meta.get("id"),
                album_name=album_meta.get("name"),
                album_artist=(
                    album_meta.get("artists", [])[0]["name"]
                    if album_meta.get("artists")
                    else None
                ),
                album_type=album_meta.get("album_type"),
                disc_number=track_meta["disc_number"],
                duration=int(track_meta["duration_ms"] / 1000),
                year=release_date[:4] if release_date else None,
                date=release_date,
                track_number=track_meta["track_number"],
                tracks_count=album_meta.get("total_tracks"),
                song_id=track_meta["id"],
                explicit=track_meta["explicit"],
                url=track_meta["external_urls"]["spotify"],
                isrc=track_meta.get("external_ids", {}).get("isrc"),
                cover_url=(
                    max(album_meta["images"], key=lambda i: i["width"] * i["height"])[
                        "url"
                    ]
                    if (len(album_meta.get("images", [])) > 0)
                    else None
                ),
                list_position=track_no + 1,
            )

            songs.append(song)

        return metadata, songs



File: spotdl/types/saved.py

"""
Saved module for handing the saved tracks from user library
"""

from dataclasses import dataclass
from typing import Any, Dict, List, Tuple

from spotdl.types.song import Song, SongList
from spotdl.utils.spotify import SpotifyClient

__all__ = ["Saved", "SavedError"]


class SavedError(Exception):
    """
    Base class for all exceptions related to saved tracks.
    """


@dataclass(frozen=True)
class Saved(SongList):
    """
    Saved class for handling the saved tracks from user library.
    """

    @staticmethod
    def get_metadata(url: str = "saved") -> Tuple[Dict[str, Any], List[Song]]:
        """
        Returns metadata for a saved list.

        ### Arguments
        - url: Not required, but used to match the signature of the other get_metadata methods.

        ### Returns
        - metadata: A dictionary containing the metadata for the saved list.
        - songs: A list of Song objects.
        """

        metadata = {"name": "Saved tracks", "url": url}

        spotify_client = SpotifyClient()
        if spotify_client.user_auth is False:  # type: ignore
            raise SavedError("You must be logged in to use this function")

        saved_tracks_response = spotify_client.current_user_saved_tracks()
        if saved_tracks_response is None:
            raise SavedError("Couldn't get saved tracks")

        saved_tracks = saved_tracks_response["items"]

        # Fetch all saved tracks
        while saved_tracks_response and saved_tracks_response["next"]:
            response = spotify_client.next(saved_tracks_response)
            if response is None:
                break

            saved_tracks_response = response
            saved_tracks.extend(saved_tracks_response["items"])

        songs = []
        for track in saved_tracks:
            if not isinstance(track, dict) or track.get("track", {}).get("is_local"):
                continue

            track_meta = track["track"]
            album_meta = track_meta["album"]

            release_date = album_meta["release_date"]
            artists = artists = [artist["name"] for artist in track_meta["artists"]]

            song = Song.from_missing_data(
                name=track_meta["name"],
                artists=artists,
                artist=artists[0],
                album_id=album_meta["id"],
                album_name=album_meta["name"],
                album_artist=album_meta["artists"][0]["name"],
                album_type=album_meta["album_type"],
                disc_number=track_meta["disc_number"],
                duration=int(track_meta["duration_ms"] / 1000),
                year=release_date[:4],
                date=release_date,
                track_number=track_meta["track_number"],
                tracks_count=album_meta["total_tracks"],
                song_id=track_meta["id"],
                explicit=track_meta["explicit"],
                url=track_meta["external_urls"]["spotify"],
                isrc=track_meta.get("external_ids", {}).get("isrc"),
                cover_url=(
                    max(album_meta["images"], key=lambda i: i["width"] * i["height"])[
                        "url"
                    ]
                    if album_meta["images"]
                    else None
                ),
            )

            songs.append(song)

        return metadata, songs



File: spotdl/types/song.py

"""
Song module that hold the Song and SongList classes.
"""

import json
from dataclasses import asdict, dataclass
from typing import Any, Dict, List, Optional, Tuple

from rapidfuzz import fuzz

from spotdl.utils.spotify import SpotifyClient

__all__ = ["Song", "SongList", "SongError"]


class SongError(Exception):
    """
    Base class for all exceptions related to songs.
    """


class SongListError(Exception):
    """
    Base class for all exceptions related to song lists.
    """


@dataclass
class Song:
    """
    Song class. Contains all the information about a song.
    """

    name: str
    artists: List[str]
    artist: str
    genres: List[str]
    disc_number: int
    disc_count: int
    album_name: str
    album_artist: str
    album_type: str
    duration: int
    year: int
    date: str
    track_number: int
    tracks_count: int
    song_id: str
    explicit: bool
    publisher: str
    url: str
    isrc: Optional[str]
    cover_url: Optional[str]
    copyright_text: Optional[str]
    download_url: Optional[str] = None
    lyrics: Optional[str] = None
    popularity: Optional[int] = None
    album_id: Optional[str] = None
    list_name: Optional[str] = None
    list_url: Optional[str] = None
    list_position: Optional[int] = None
    list_length: Optional[int] = None
    artist_id: Optional[str] = None

    @classmethod
    def from_url(cls, url: str) -> "Song":
        """
        Creates a Song object from a URL.

        ### Arguments
        - url: The URL of the song.

        ### Returns
        - The Song object.
        """

        if "open.spotify.com" not in url or "track" not in url:
            raise SongError(f"Invalid URL: {url}")

        # query spotify for song, artist, album details
        spotify_client = SpotifyClient()

        # get track info
        raw_track_meta = spotify_client.track(url)

        if raw_track_meta is None:
            raise SongError(
                "Couldn't get metadata, check if you have passed correct track id"
            )

        if raw_track_meta["duration_ms"] == 0 or raw_track_meta["name"].strip() == "":
            raise SongError(f"Track no longer exists: {url}")

        # get artist info
        primary_artist_id = raw_track_meta["artists"][0]["id"]
        raw_artist_meta: Dict[str, Any] = spotify_client.artist(primary_artist_id)  # type: ignore

        # get album info
        album_id = raw_track_meta["album"]["id"]
        raw_album_meta: Dict[str, Any] = spotify_client.album(album_id)  # type: ignore

        # create song object
        return cls(
            name=raw_track_meta["name"],
            artists=[artist["name"] for artist in raw_track_meta["artists"]],
            artist=raw_track_meta["artists"][0]["name"],
            artist_id=primary_artist_id,
            album_id=album_id,
            album_name=raw_album_meta["name"],
            album_artist=raw_album_meta["artists"][0]["name"],
            album_type=raw_album_meta["album_type"],
            copyright_text=(
                raw_album_meta["copyrights"][0]["text"]
                if raw_album_meta["copyrights"]
                else None
            ),
            genres=raw_album_meta["genres"] + raw_artist_meta["genres"],
            disc_number=raw_track_meta["disc_number"],
            disc_count=int(raw_album_meta["tracks"]["items"][-1]["disc_number"]),
            duration=int(raw_track_meta["duration_ms"] / 1000),
            year=int(raw_album_meta["release_date"][:4]),
            date=raw_album_meta["release_date"],
            track_number=raw_track_meta["track_number"],
            tracks_count=raw_album_meta["total_tracks"],
            isrc=raw_track_meta.get("external_ids", {}).get("isrc"),
            song_id=raw_track_meta["id"],
            explicit=raw_track_meta["explicit"],
            publisher=raw_album_meta["label"],
            url=raw_track_meta["external_urls"]["spotify"],
            popularity=raw_track_meta["popularity"],
            cover_url=(
                max(raw_album_meta["images"], key=lambda i: i["width"] * i["height"])[
                    "url"
                ]
                if raw_album_meta["images"]
                else None
            ),
        )

    @staticmethod
    def search(search_term: str):
        """
        Searches for Songs from a search term.

        ### Arguments
        - search_term: The search term to use.

        ### Returns
        - The raw search results
        """
        spotify_client = SpotifyClient()
        raw_search_results = spotify_client.search(search_term)

        if raw_search_results is None:
            raise SongError(f"Spotipy error, no response: {search_term}")

        return raw_search_results

    @classmethod
    def from_search_term(cls, search_term: str) -> "Song":
        """
        Creates a list of Song objects from a search term.

        ### Arguments
        - search_term: The search term to use.

        ### Returns
        - The Song object.
        """

        raw_search_results = Song.search(search_term)

        if len(raw_search_results["tracks"]["items"]) == 0:
            raise SongError(f"No results found for: {search_term}")

        return Song.from_url(
            "http://open.spotify.com/track/"
            + raw_search_results["tracks"]["items"][0]["id"]
        )

    @classmethod
    def list_from_search_term(cls, search_term: str) -> "List[Song]":
        """
        Creates a list of Song objects from a search term.

        ### Arguments
        - search_term: The search term to use.

        ### Returns
        - The list of Song objects.
        """

        raw_search_results = Song.search(search_term)

        songs = []
        for idx, _ in enumerate(raw_search_results.get("tracks", []).get("items", [])):
            songs.append(
                Song.from_url(
                    "http://open.spotify.com/track/"
                    + raw_search_results["tracks"]["items"][idx]["id"]
                )
            )

        return songs

    @classmethod
    def from_data_dump(cls, data: str) -> "Song":
        """
        Create a Song object from a data dump.

        ### Arguments
        - data: The data dump.

        ### Returns
        - The Song object.
        """

        # Create dict from json string
        data_dict = json.loads(data)

        # Return product object
        return cls(**data_dict)

    @classmethod
    def from_dict(cls, data: Dict[str, Any]) -> "Song":
        """
        Create a Song object from a dictionary.

        ### Arguments
        - data: The dictionary.

        ### Returns
        - The Song object.
        """

        # Return product object
        return cls(**data)

    @classmethod
    def from_missing_data(cls, **kwargs) -> "Song":
        """
        Create a Song object from a dictionary with missing data.
        For example, data dict doesn't contain all the required
        attributes for the Song class.

        ### Arguments
        - data: The dictionary.

        ### Returns
        - The Song object.
        """

        song_data: Dict[str, Any] = {}
        for key in cls.__dataclass_fields__:  # pylint: disable=E1101
            song_data.setdefault(key, kwargs.get(key))

        return cls(**song_data)

    @property
    def display_name(self) -> str:
        """
        Returns a display name for the song.

        ### Returns
        - The display name.
        """

        return f"{self.artist} - {self.name}"

    @property
    def json(self) -> Dict[str, Any]:
        """
        Returns a dictionary of the song's data.

        ### Returns
        - The dictionary.
        """

        return asdict(self)


@dataclass(frozen=True)
class SongList:
    """
    SongList class. Base class for all other song lists subclasses.
    """

    name: str
    url: str
    urls: List[str]
    songs: List[Song]

    @classmethod
    def from_url(cls, url: str, fetch_songs: bool = True):
        """
        Create a SongList object from a url.

        ### Arguments
        - url: The url of the list.
        - fetch_songs: Whether to fetch missing metadata for songs.

        ### Returns
        - The SongList object.
        """

        metadata, songs = cls.get_metadata(url)
        urls = [song.url for song in songs]

        if fetch_songs:
            songs = [Song.from_url(song.url) for song in songs]

        return cls(**metadata, urls=urls, songs=songs)

    @classmethod
    def from_search_term(cls, search_term: str, fetch_songs: bool = True):
        """
        Creates a SongList object from a search term.

        ### Arguments
        - search_term: The search term to use.

        ### Returns
        - The SongList object.
        """

        list_type = cls.__name__.lower()
        spotify_client = SpotifyClient()
        raw_search_results = spotify_client.search(search_term, type=list_type)

        if (
            raw_search_results is None
            or len(raw_search_results.get(f"{list_type}s", {}).get("items", [])) == 0
        ):
            raise SongListError(
                f"No {list_type} matches found on spotify for '{search_term}'"
            )

        matches = {}
        for result in raw_search_results[f"{list_type}s"]["items"]:
            score = fuzz.ratio(search_term.split(":", 1)[1].strip(), result["name"])
            matches[result["id"]] = score

        best_match = max(matches, key=matches.get)  # type: ignore

        return cls.from_url(
            f"http://open.spotify.com/{list_type}/{best_match}",
            fetch_songs,
        )

    @property
    def length(self) -> int:
        """
        Get list length (number of songs).

        ### Returns
        - The list length.
        """

        return max(len(self.urls), len(self.songs))

    @property
    def json(self) -> Dict[str, Any]:
        """
        Returns a dictionary of the song list's data.

        ### Returns
        - The dictionary.
        """

        return asdict(self)

    @staticmethod
    def get_metadata(url: str) -> Tuple[Dict[str, Any], List[Song]]:
        """
        Get metadata for a song list.

        ### Arguments
        - url: The url of the song list.

        ### Returns
        - The metadata.
        """

        raise NotImplementedError



File: spotdl/providers/__init__.py

"""
Different types of data providers for spotdl.
"""



File: spotdl/providers/lyrics/genius.py

"""
Genius Lyrics module.
"""

from typing import Dict, List, Optional

import requests
from bs4 import BeautifulSoup

from spotdl.providers.lyrics.base import LyricsProvider
from spotdl.utils.config import GlobalConfig

__all__ = ["Genius"]


class Genius(LyricsProvider):
    """
    Genius lyrics provider class.
    """

    def __init__(self, access_token: str):
        """
        Init the lyrics provider search and set headers.
        """

        super().__init__()

        self.access_token = access_token

        self.headers.update(
            {
                "Authorization": f"Bearer {self.access_token}",
            }
        )

        self.session = requests.Session()
        self.session.headers.update(self.headers)

    def get_results(self, name: str, artists: List[str], **_) -> Dict[str, str]:
        """
        Returns the results for the given song.

        ### Arguments
        - name: The name of the song.
        - artists: The artists of the song.
        - kwargs: Additional arguments.

        ### Returns
        - A dictionary with the results. (The key is the title and the value is the url.)
        """

        artists_str = ", ".join(artists)
        title = f"{name} - {artists_str}"

        search_response = self.session.get(
            "https://api.genius.com/search",
            params={"q": title},
            headers=self.headers,
            timeout=10,
            proxies=GlobalConfig.get_parameter("proxies"),
        )

        results: Dict[str, str] = {}
        for hit in search_response.json()["response"]["hits"]:
            results[hit["result"]["full_title"]] = hit["result"]["id"]

        return results

    def extract_lyrics(self, url: str, **_) -> Optional[str]:
        """
        Extracts the lyrics from the given url.

        ### Arguments
        - url: The url to extract the lyrics from.
        - kwargs: Additional arguments.

        ### Returns
        - The lyrics of the song or None if no lyrics were found.
        """

        url = f"https://api.genius.com/songs/{url}"
        song_response = self.session.get(
            url,
            headers=self.headers,
            timeout=10,
            proxies=GlobalConfig.get_parameter("proxies"),
        )
        url = song_response.json()["response"]["song"]["url"]

        soup = None
        counter = 0
        while counter < 4:
            genius_page_response = self.session.get(
                url,
                headers=self.headers,
                timeout=10,
                proxies=GlobalConfig.get_parameter("proxies"),
            )

            if not genius_page_response.ok:
                counter += 1
                continue

            soup = BeautifulSoup(
                genius_page_response.text.replace("<br/>", "\n"), "html.parser"
            )

            break

        if soup is None:
            return None

        lyrics_div = soup.select_one("div.lyrics")
        lyrics_containers = soup.select("div[class^=Lyrics__Container]")

        # Get lyrics
        if lyrics_div:
            lyrics = lyrics_div.get_text()
        elif lyrics_containers:
            lyrics = "\n".join(con.get_text() for con in lyrics_containers)
        else:
            return None

        if not lyrics:
            return None

        # Clean lyrics
        lyrics = lyrics.strip()

        # Remove desc at the beginning if it exists
        for to_remove in ["desc", "Desc"]:
            lyrics.replace(to_remove, "", 1)

        return lyrics



File: spotdl/providers/lyrics/musixmatch.py

"""
MusixMatch lyrics provider.
"""

from typing import Dict, List, Optional
from urllib.parse import quote

import requests
from bs4 import BeautifulSoup

from spotdl.providers.lyrics.base import LyricsProvider
from spotdl.utils.config import GlobalConfig

__all__ = ["MusixMatch"]


class MusixMatch(LyricsProvider):
    """
    MusixMatch lyrics provider class.
    """

    def extract_lyrics(self, url: str, **_) -> Optional[str]:
        """
        Extracts the lyrics from the given url.

        ### Arguments
        - url: The url to extract the lyrics from.
        - kwargs: Additional arguments.

        ### Returns
        - The lyrics of the song or None if no lyrics were found.
        """

        lyrics_resp = requests.get(
            url,
            headers=self.headers,
            timeout=10,
            proxies=GlobalConfig.get_parameter("proxies"),
        )

        lyrics_soup = BeautifulSoup(lyrics_resp.text, "html.parser")
        lyrics_paragraphs = lyrics_soup.select("p.mxm-lyrics__content")
        lyrics = "\n".join(i.get_text() for i in lyrics_paragraphs)

        return lyrics

    def get_results(self, name: str, artists: List[str], **kwargs) -> Dict[str, str]:
        """
        Returns the results for the given song.

        ### Arguments
        - name: The name of the song.
        - artists: The artists of the song.
        - kwargs: Additional arguments.

        ### Returns
        - A dictionary with the results. (The key is the title and the value is the url.)
        """

        track_search = kwargs.get("track_search", False)
        artists_str = ", ".join(
            artist for artist in artists if artist.lower() not in name.lower()
        )

        # quote the query so that it's safe to use in a url
        # e.g "Au/Ra" -> "Au%2FRa"
        query = quote(f"{name} - {artists_str}", safe="")

        # search the `tracks page` if track_search is True
        if track_search:
            query += "/tracks"

        search_url = f"https://www.musixmatch.com/search/{query}"
        search_resp = requests.get(
            search_url,
            headers=self.headers,
            timeout=10,
            proxies=GlobalConfig.get_parameter("proxies"),
        )
        search_soup = BeautifulSoup(search_resp.text, "html.parser")
        song_url_tag = search_soup.select("a[href^='/lyrics/']")

        if not song_url_tag:
            # song_url_tag being None means no results were found on the
            # All Results page, therefore, we use `track_search` to
            # search the tracks page.

            # track_serach being True means we are already searching the tracks page.
            if track_search:
                return {}

            return self.get_results(name, artists, track_search=True)

        results: Dict[str, str] = {}
        for tag in song_url_tag:
            results[tag.get_text()] = "https://www.musixmatch.com" + str(
                tag.get("href", "")
            )

        return results



File: spotdl/providers/lyrics/synced.py

"""
Synced lyrics provider using the syncedlyrics library
"""

from typing import Dict, List, Optional

import requests
import syncedlyrics

from spotdl.providers.lyrics.base import LyricsProvider

__all__ = ["Synced"]


class Synced(LyricsProvider):
    """
    Lyrics provider for synced lyrics using the syncedlyrics library
    Currently supported websites: Deezer, NetEase
    """

    def get_results(self, name: str, artists: List[str], **kwargs) -> Dict[str, str]:
        """
        Returns the results for the given song.

        ### Arguments
        - name: The name of the song.
        - artists: The artists of the song.
        - kwargs: Additional arguments.

        ### Returns
        - A dictionary with the results. (The key is the title and the value is the url.)
        """

        raise NotImplementedError

    def extract_lyrics(self, url: str, **kwargs) -> Optional[str]:
        """
        Extracts the lyrics from the given url.

        ### Arguments
        - url: The url to extract the lyrics from.
        - kwargs: Additional arguments.

        ### Returns
        - The lyrics of the song or None if no lyrics were found.
        """

        raise NotImplementedError

    def get_lyrics(self, name: str, artists: List[str], **kwargs) -> Optional[str]:
        """
        Try to get lyrics using syncedlyrics

        ### Arguments
        - name: The name of the song.
        - artists: The artists of the song.
        - kwargs: Additional arguments.

        ### Returns
        - The lyrics of the song or None if no lyrics were found.
        """

        try:
            lyrics = syncedlyrics.search(
                f"{name} - {artists[0]}",
                synced_only=not kwargs.get("allow_plain_format", True),
            )
            return lyrics
        except requests.exceptions.SSLError:
            # Max retries reached
            return None
        except TypeError:
            # Error at syncedlyrics.providers.musixmatch L89 -
            #   Because `body` is occasionally an empty list instead of a dictionary.
            # We get this error when allow_plain_format is set to True,
            #   and there are no synced lyrics present
            # Because its empty, we know there are no lyrics
            return None



File: spotdl/providers/lyrics/__init__.py

"""
Lyrics providers for spotdl.
"""

from spotdl.providers.lyrics.azlyrics import AzLyrics
from spotdl.providers.lyrics.base import LyricsProvider
from spotdl.providers.lyrics.genius import Genius
from spotdl.providers.lyrics.musixmatch import MusixMatch
from spotdl.providers.lyrics.synced import Synced

__all__ = ["AzLyrics", "Genius", "MusixMatch", "Synced", "LyricsProvider"]



File: spotdl/providers/lyrics/base.py

"""
Base module for all other lyrics providers.
"""

import logging
from typing import Dict, List, Optional

from spotdl.utils.formatter import ratio, slugify
from spotdl.utils.matching import based_sort

__all__ = ["LyricsProvider"]
logger = logging.getLogger(__name__)


class LyricsProvider:
    """
    Base class for all other lyrics providers.
    """

    def __init__(self):
        """
        Init the lyrics provider searchand set headers.
        """

        self.headers = {
            "Connection": "keep-alive",
            "Pragma": "no-cache",
            "Cache-Control": "no-cache",
            "sec-ch-ua": '"Chromium";v="104", " Not A;Brand";v="99", "Google Chrome";v="104"',
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
            "(KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36",
            "Accept": "*/*",
            "Sec-Fetch-Site": "same-origin",
            "Sec-Fetch-Mode": "cors",
            "Sec-Fetch-Dest": "empty",
            "Accept-Language": "en-US;q=0.8,en;q=0.7",
        }

    def get_results(self, name: str, artists: List[str], **kwargs) -> Dict[str, str]:
        """
        Returns the results for the given song.

        ### Arguments
        - name: The name of the song.
        - artists: The artists of the song.
        - kwargs: Additional arguments.

        ### Returns
        - A dictionary with the results. (The key is the title and the value is the url.)
        """

        raise NotImplementedError

    def extract_lyrics(self, url: str, **kwargs) -> Optional[str]:
        """
        Extracts the lyrics from the given url.

        ### Arguments
        - url: The url to extract the lyrics from.
        - kwargs: Additional arguments.

        ### Returns
        - The lyrics of the song or None if no lyrics were found.
        """

        raise NotImplementedError

    def get_lyrics(self, name: str, artists: List[str], **kwargs) -> Optional[str]:
        """
        Returns the lyrics for the given song.

        ### Arguments
        - name: The name of the song.
        - artists: The artists of the song.
        - kwargs: Additional arguments.

        ### Returns
        - The lyrics of the song or None if no lyrics were found.
        """
        try:
            results = self.get_results(name, artists, **kwargs)
        except Exception as exc:
            logger.debug(
                "%s: Failed to get results for %s - %s: %s",
                self.name,
                name,
                ", ".join(artists),
                exc,
            )
            return None

        if not results:
            return None

        results_with_score = {}
        for title, url in results.items():
            result_title = slugify(title)
            match_title = slugify(f"{name} - {', '.join(artists)}")

            res_list, song_list = based_sort(
                result_title.split("-"), match_title.split("-")
            )
            result_title, match_title = "-".join(res_list), "-".join(song_list)

            score = ratio(result_title, match_title)
            results_with_score[score] = url

        if not results_with_score:
            return None

        # Get song url with highest title match
        score, url = max(results_with_score.items(), key=lambda x: x[0])

        # Only return lyrics if the title match is at least 55%
        if score < 55:
            return None

        try:
            return self.extract_lyrics(url, **kwargs)
        except Exception as exc:
            logger.debug(
                "%s: Failed to extract lyrics from %s: %s", self.name, url, exc
            )
            return None

    @property
    def name(self) -> str:
        """
        Returns the name of the lyrics provider.
        """

        return self.__class__.__name__



File: spotdl/providers/lyrics/azlyrics.py

"""
AZLyrics lyrics module.
"""

from typing import Dict, List, Optional

import requests
from bs4 import BeautifulSoup

from spotdl.providers.lyrics.base import LyricsProvider

__all__ = ["AzLyrics"]


class AzLyrics(LyricsProvider):
    """
    AZLyrics lyrics provider class.
    """

    def __init__(self):
        super().__init__()

        self.session = requests.Session()
        self.session.headers.update(self.headers)

        self.x_code = self.get_x_code()

    def get_results(self, name: str, artists: List[str], **_) -> Dict[str, str]:
        """
        Returns the results for the given song.

        ### Arguments
        - name: The name of the song.
        - artists: The artists of the song.
        - kwargs: Additional arguments.

        ### Returns
        - A dictionary with the results. (The key is the title and the value is the url.)
        """

        if self.x_code is None:
            self.x_code = self.get_x_code()

        if self.x_code is None:
            return {}

        # Join every artist by comma in artists
        artist_str = ", ".join(artist for artist in artists if artist)

        params = {
            "q": f"{artist_str} - {name}",
            "x": self.x_code,
        }

        counter = 0
        soup = None
        while counter < 4:
            try:
                response = self.session.get(
                    "https://search.azlyrics.com/search.php", params=params
                )
            except requests.ConnectionError:
                continue

            if not response.ok:
                counter += 1
                continue

            soup = BeautifulSoup(response.content, "html.parser")
            break

        if soup is None:
            return {}

        td_tags = soup.find_all("td")
        if len(td_tags) == 0:
            return {}

        results = {}
        for td_tag in td_tags:
            a_tags = td_tag.find_all("a", href=True)
            if len(a_tags) == 0:
                continue

            a_tag = a_tags[0]
            url = a_tag["href"].strip()
            if url == "":
                continue

            title = td_tag.find("span").get_text().strip()
            artist = td_tag.find("b").get_text().strip()

            results[f"{artist} - {title}"] = url

        return results

    def extract_lyrics(self, url: str, **_) -> Optional[str]:
        """
        Extracts the lyrics from the given url.

        ### Arguments
        - url: The url to extract the lyrics from.
        - kwargs: Additional arguments.

        ### Returns
        - The lyrics of the song or None if no lyrics were found.
        """

        response = self.session.get(url)
        soup = BeautifulSoup(response.content, "html.parser")

        # Find all divs that don't have a class
        div_tags = soup.find_all("div", class_=False, id_=False)

        # Find the div with the longest text
        lyrics_div = sorted(div_tags, key=lambda x: len(x.text))[-1]

        # extract lyrics from div and clean it up
        lyrics = lyrics_div.get_text().strip()

        return lyrics

    def get_x_code(self) -> Optional[str]:
        """
        Returns the x_code used by AZLyrics.

        ### Returns
        - The x_code used by AZLyrics or None if it couldn't be retrieved.
        """

        x_code = None

        try:
            self.session.get("https://www.azlyrics.com/")

            resp = self.session.get("https://www.azlyrics.com/geo.js")

            # extract value from js code
            js_code = resp.text
            start_index = js_code.find('value"') + 9
            end_index = js_code[start_index:].find('");')

            x_code = js_code[start_index : start_index + end_index]
        except requests.ConnectionError:
            pass

        return x_code



File: spotdl/providers/audio/bandcamp.py

"""
BandCamp module for downloading and searching songs.
"""

import logging
from typing import Any, Dict, List, Optional, Tuple

import requests

from spotdl.providers.audio.base import AudioProvider
from spotdl.types.result import Result
from spotdl.utils.config import GlobalConfig

__all__ = ["BandCamp"]

logger = logging.getLogger(__name__)


class BandCampTrack:
    """
    BandCamp track class based on the bandcamp_api library
    """

    def __init__(self, artist_id: int, track_id: int):
        # object info
        self.type = "track"

        # track information
        self.track_id: int = 0
        self.track_title: str = ""
        self.track_number: int = 0
        self.track_duration_seconds: float = 0.00
        self.track_streamable: Optional[bool] = None
        self.has_lyrics: Optional[bool] = None
        self.lyrics: str = ""
        self.is_price_set: Optional[bool] = None
        self.price: dict = {}
        self.require_email: Optional[bool] = None
        self.is_purchasable: Optional[bool] = None
        self.is_free: Optional[bool] = None
        self.is_preorder: Optional[bool] = None
        self.tags: list = []
        self.track_url: str = ""

        # art
        self.art_id: int = 0
        self.art_url: str = ""

        # artist information
        self.artist_id: int = 0
        self.artist_title: str = ""

        # album information
        self.album_id: int = 0
        self.album_title: str = ""

        # label
        self.label_id: int = 0
        self.label_title: str = ""

        # about
        self.about: str = ""
        self.credits: str = ""
        self.date_released_unix: int = 0

        # advanced
        self.date_last_modified_unix: int = 0
        self.date_published_unix: int = 0
        self.supporters: list = []

        response = requests.get(
            url="https://bandcamp.com/api/mobile/25/tralbum_details?band_id="
            + str(artist_id)
            + "&tralbum_id="
            + str(track_id)
            + "&tralbum_type=t",
            timeout=10,
            proxies=GlobalConfig.get_parameter("proxies"),
        )
        result = response.json()
        self.track_id = result["id"]
        self.track_title = result["title"]
        self.track_number = result["tracks"][0]["track_num"]
        self.track_duration_seconds = result["tracks"][0]["duration"]
        self.track_streamable = result["tracks"][0]["is_streamable"]
        self.has_lyrics = result["tracks"][0]["has_lyrics"]

        # getting lyrics, if there is any
        if self.has_lyrics is True:
            resp = requests.get(
                "https://bandcamp.com/api/mobile/25/tralbum_lyrics?tralbum_id="
                + str(self.track_id)
                + "&tralbum_type=t",
                timeout=10,
                proxies=GlobalConfig.get_parameter("proxies"),
            )
            rjson = resp.json()
            self.lyrics = rjson["lyrics"][str(self.track_id)]

        self.is_price_set = result["is_set_price"]
        self.price = {"currency": result["currency"], "amount": result["price"]}
        self.require_email = result["require_email"]
        self.is_purchasable = result["is_purchasable"]
        self.is_free = result["free_download"]
        self.is_preorder = result["is_preorder"]

        for tag in result["tags"]:
            self.tags.append(tag["name"])

        self.art_id = result["art_id"]
        self.art_url = "https://f4.bcbits.com/img/a" + str(self.art_id) + "_0.jpg"

        self.artist_id = result["band"]["band_id"]
        self.artist_title = result["band"]["name"]

        self.album_id = result["album_id"]
        self.album_title = result["album_title"]

        self.label_id = result["label_id"]
        self.label_title = result["label"]

        self.about = result["about"]
        self.credits = result["credits"]

        self.date_released_unix = result["release_date"]

        self.track_url = result["bandcamp_url"]


def search(search_string: str = ""):
    """
    I got this api url from the iOS app
    needs a way of removing characters
    that will screw up an url
    keep url safe characters

    ### Arguments
    - search_string: The search term to search for.

    ### Returns
    - A list of artist and track ids if found
    """

    response = requests.get(
        "https://bandcamp.com/api/fuzzysearch/2/app_autocomplete?q="
        + search_string
        + "&param_with_locations=true",
        timeout=10,
        proxies=GlobalConfig.get_parameter("proxies"),
    )

    results = response.json()["results"]

    return_results: List[Tuple[str, str]] = []

    for item in results:
        if item["type"] == "t":
            return_results.append((item["band_id"], item["id"]))

    return return_results


class BandCamp(AudioProvider):
    """
    SoundCloud audio provider class
    """

    SUPPORTS_ISRC = False
    GET_RESULTS_OPTS: List[Dict[str, Any]] = [{}]

    def get_results(self, search_term: str, *_args, **_kwargs) -> List[Result]:
        """
        Get results from slider.kz

        ### Arguments
        - search_term: The search term to search for.
        - args: Unused.
        - kwargs: Unused.

        ### Returns
        - A list of slider.kz results if found, None otherwise.
        """

        try:
            results = search(search_term)
        except KeyError:
            return []
        except Exception as exc:
            logger.error("Failed to get results from BandCamp", exc_info=exc)
            return []

        simplified_results: List[Result] = []
        for result in results:
            track = BandCampTrack(int(result[0]), int(result[1]))

            simplified_results.append(
                Result(
                    source="bandcamp",
                    url=track.track_url,
                    verified=False,
                    name=track.track_title,
                    duration=track.track_duration_seconds,
                    author=track.artist_title,
                    result_id=track.track_url,
                    search_query=search_term,
                    album=track.album_title,
                    artists=tuple(track.artist_title.split(", ")),
                )
            )

        return simplified_results



File: spotdl/providers/audio/youtube.py

"""
Youtube module for downloading and searching songs.
"""

from typing import Any, Dict, List, Optional

from pytube import Search
from pytube import YouTube as PyTube

from spotdl.providers.audio.base import AudioProvider
from spotdl.types.result import Result

__all__ = ["YouTube"]


class YouTube(AudioProvider):
    """
    YouTube audio provider class
    """

    SUPPORTS_ISRC = False
    GET_RESULTS_OPTS: List[Dict[str, Any]] = [{}]

    def get_results(
        self, search_term: str, *_args, **_kwargs
    ) -> List[Result]:  # pylint: disable=W0221
        """
        Get results from YouTube

        ### Arguments
        - search_term: The search term to search for.
        - args: Unused.
        - kwargs: Unused.

        ### Returns
        - A list of YouTube results if found, None otherwise.
        """

        search_results: Optional[List[PyTube]] = Search(search_term).results

        if not search_results:
            return []

        results = []
        for result in search_results:
            if result.watch_url:
                try:
                    duration = result.length
                except Exception:
                    duration = 0

                try:
                    views = result.views
                except Exception:
                    views = 0

                results.append(
                    Result(
                        source=self.name,
                        url=result.watch_url,
                        verified=False,
                        name=result.title,
                        duration=duration,
                        author=result.author,
                        search_query=search_term,
                        views=views,
                        result_id=result.video_id,
                    )
                )

        return results



File: spotdl/providers/audio/sliderkz.py

"""
SliderKZ module for downloading and searching songs.
"""

import logging
from typing import Any, Dict, List

import requests

from spotdl.providers.audio.base import AudioProvider
from spotdl.types.result import Result
from spotdl.utils.config import GlobalConfig

__all__ = ["SliderKZ"]

logger = logging.getLogger(__name__)

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/111.0"
}


class SliderKZ(AudioProvider):
    """
    Slider.kz audio provider class
    """

    SUPPORTS_ISRC = False
    GET_RESULTS_OPTS: List[Dict[str, Any]] = [{}]

    def get_results(self, search_term: str, *_args, **_kwargs) -> List[Result]:
        """
        Get results from slider.kz

        ### Arguments
        - search_term: The search term to search for.
        - args: Unused.
        - kwargs: Unused.

        ### Returns
        - A list of slider.kz results if found, None otherwise.
        """

        search_results = None
        max_retries = 0

        while not search_results and max_retries < 3:
            try:
                search_response = requests.get(
                    url="https://hayqbhgr.slider.kz/vk_auth.php?q=" + search_term,
                    headers=HEADERS,
                    timeout=5,
                    proxies=GlobalConfig.get_parameter("proxies"),
                )

                # Check if the response is valid
                if len(search_response.text) > 30:
                    # Set the search results to the json response
                    # effectively breaking out of the loop
                    search_results = search_response.json()

            except Exception as exc:
                logger.debug(
                    "Slider.kz search failed for query %s with error: %s. Retrying...",
                    search_term,
                    exc,
                )

            max_retries += 1

        if not search_results:
            logger.debug("Slider.kz search failed for query %s", search_term)
            return []

        results = []
        for result in search_results["audios"][""]:
            # urls from slider.kz sometimes are relative, so we need to add the domain
            if "https://" not in result["url"]:
                result["url"] = "https://hayqbhgr.slider.kz/" + result["url"]

            results.append(
                Result(
                    source="slider.kz",
                    url=result.get("url"),
                    verified=False,
                    name=result.get("tit_art"),
                    duration=int(result.get("duration", -9999)),
                    author="slider.kz",
                    result_id=result.get("id"),
                    views=1,
                )
            )

        return results



File: spotdl/providers/audio/__init__.py

"""
Audio providers for spotdl.
"""

from spotdl.providers.audio.bandcamp import BandCamp
from spotdl.providers.audio.base import (
    ISRC_REGEX,
    AudioProvider,
    AudioProviderError,
    YTDLLogger,
)
from spotdl.providers.audio.piped import Piped
from spotdl.providers.audio.sliderkz import SliderKZ
from spotdl.providers.audio.soundcloud import SoundCloud
from spotdl.providers.audio.youtube import YouTube
from spotdl.providers.audio.ytmusic import YouTubeMusic

__all__ = [
    "YouTube",
    "YouTubeMusic",
    "SliderKZ",
    "SoundCloud",
    "BandCamp",
    "Piped",
    "AudioProvider",
    "AudioProviderError",
    "YTDLLogger",
    "ISRC_REGEX",
]



File: spotdl/providers/audio/soundcloud.py

"""
SoundCloud module for downloading and searching songs.
"""

import logging
import re
from itertools import islice
from typing import Any, Dict, List

from soundcloud import SoundCloud as SoundCloudClient
from soundcloud.resource.track import Track

from spotdl.providers.audio.base import AudioProvider
from spotdl.types.result import Result

__all__ = ["SoundCloud"]

logger = logging.getLogger(__name__)


class SoundCloud(AudioProvider):
    """
    SoundCloud audio provider class
    """

    SUPPORTS_ISRC = False
    GET_RESULTS_OPTS: List[Dict[str, Any]] = [{}]

    def __init__(self, *args: Any, **kwargs: Any) -> None:
        """
        Initialize the SoundCloud API

        ### Arguments
        - args: Arguments passed to the `AudioProvider` class.
        - kwargs: Keyword arguments passed to the `AudioProvider` class.
        """

        super().__init__(*args, **kwargs)
        self.client = SoundCloudClient()

    def get_results(self, search_term: str, *_args, **_kwargs) -> List[Result]:
        """
        Get results from slider.kz

        ### Arguments
        - search_term: The search term to search for.
        - args: Unused.
        - kwargs: Unused.

        ### Returns
        - A list of slider.kz results if found, None otherwise.
        """

        results = list(islice(self.client.search(search_term), 20))
        regex = r"^(.+?)-|(\(\w+[\s\S]*\))"
        # Because anyone can post on soundcloud, we do another search with an edited search
        # The regex removes anything in brackets and the artist(s)'s name(s) if in the name
        edited_search_term = re.sub(regex, "", search_term)
        results.extend(list(islice(self.client.search(edited_search_term), 20)))

        # Simplify results
        simplified_results = []
        for result in results:
            if not isinstance(result, Track):
                continue

            # Ignore results that are not playable
            if "/preview/" in result.media.transcodings[0].url:
                continue

            album = self.client.get_track_albums(result.id)

            try:
                album_name = next(album).title
            except StopIteration:
                album_name = None

            simplified_results.append(
                Result(
                    source="soundcloud",
                    url=result.permalink_url,
                    name=result.title,
                    verified=result.user.verified,
                    duration=result.full_duration,
                    author=result.user.username,
                    result_id=str(result.id),
                    isrc_search=False,
                    search_query=search_term,
                    views=result.playback_count,
                    explicit=False,
                    album=album_name,
                )
            )

        return simplified_results



File: spotdl/providers/audio/ytmusic.py

"""
YTMusic module for downloading and searching songs.
"""

from typing import Any, Dict, List

from ytmusicapi import YTMusic

from spotdl.providers.audio.base import ISRC_REGEX, AudioProvider
from spotdl.types.result import Result
from spotdl.utils.formatter import parse_duration

__all__ = ["YouTubeMusic"]


class YouTubeMusic(AudioProvider):
    """
    YouTube Music audio provider class
    """

    SUPPORTS_ISRC = True
    GET_RESULTS_OPTS: List[Dict[str, Any]] = [
        {"filter": "songs", "ignore_spelling": True, "limit": 50},
        {"filter": "videos", "ignore_spelling": True, "limit": 50},
    ]

    def __init__(self, *args: Any, **kwargs: Any) -> None:
        """
        Initialize the YouTube Music API

        ### Arguments
        - args: Arguments passed to the `AudioProvider` class.
        - kwargs: Keyword arguments passed to the `AudioProvider` class.
        """

        super().__init__(*args, **kwargs)

        self.client = YTMusic(language="de")

    def get_results(self, search_term: str, **kwargs) -> List[Result]:
        """
        Get results from YouTube Music API and simplify them

        ### Arguments
        - search_term: The search term to search for.
        - kwargs: other keyword arguments passed to the `YTMusic.search` method.

        ### Returns
        - A list of simplified results (dicts)
        """

        is_isrc_result = ISRC_REGEX.search(search_term) is not None
        # if is_isrc_result:
        #     print("FORCEFULLY SETTING FILTER TO SONGS")
        #     kwargs["filter"] = "songs"

        search_results = self.client.search(search_term, **kwargs)

        # Simplify results
        results = []
        for result in search_results:
            if (
                result is None
                or result.get("videoId") is None
                or result.get("artists") in [[], None]
            ):
                continue

            results.append(
                Result(
                    source=self.name,
                    url=(
                        f'https://{"music" if result["resultType"] == "song" else "www"}'
                        f".youtube.com/watch?v={result['videoId']}"
                    ),
                    verified=result.get("resultType") == "song",
                    name=result["title"],
                    result_id=result["videoId"],
                    author=result["artists"][0]["name"],
                    artists=tuple(map(lambda a: a["name"], result["artists"])),
                    duration=parse_duration(result.get("duration")),
                    isrc_search=is_isrc_result,
                    search_query=search_term,
                    explicit=result.get("isExplicit"),
                    album=(
                        result.get("album", {}).get("name")
                        if result.get("album")
                        else None
                    ),
                )
            )

        return results



File: spotdl/providers/audio/base.py

"""
Base audio provider module.
"""

import logging
import re
import shlex
from typing import Any, Dict, List, Optional, Tuple

from yt_dlp import YoutubeDL

from spotdl.types.result import Result
from spotdl.types.song import Song
from spotdl.utils.config import get_temp_path
from spotdl.utils.formatter import (
    args_to_ytdlp_options,
    create_search_query,
    create_song_title,
)
from spotdl.utils.matching import get_best_matches, order_results

__all__ = ["AudioProviderError", "AudioProvider", "ISRC_REGEX", "YTDLLogger"]

logger = logging.getLogger(__name__)


class AudioProviderError(Exception):
    """
    Base class for all exceptions related to audio searching/downloading.
    """


class YTDLLogger:
    """
    Custom YT-dlp logger.
    """

    def debug(self, msg):
        """
        YTDL uses this to print debug messages.
        """

        pass  # pylint: disable=W0107

    def warning(self, msg):
        """
        YTDL uses this to print warnings.
        """

        pass  # pylint: disable=W0107

    def error(self, msg):
        """
        YTDL uses this to print errors.
        """

        raise AudioProviderError(msg)


ISRC_REGEX = re.compile(r"^[A-Z]{2}-?\w{3}-?\d{2}-?\d{5}$")


class AudioProvider:
    """
    Base class for all other providers. Provides some common functionality.
    Handles the yt-dlp audio handler.
    """

    SUPPORTS_ISRC: bool
    GET_RESULTS_OPTS: List[Dict[str, Any]]

    def __init__(
        self,
        output_format: str = "mp3",
        cookie_file: Optional[str] = None,
        search_query: Optional[str] = None,
        filter_results: bool = True,
        yt_dlp_args: Optional[str] = None,
    ) -> None:
        """
        Base class for audio providers.

        ### Arguments
        - output_directory: The directory to save the downloaded songs to.
        - output_format: The format to save the downloaded songs in.
        - cookie_file: The path to a file containing cookies to be used by YTDL.
        - search_query: The query to use when searching for songs.
        - filter_results: Whether to filter results.
        """

        self.output_format = output_format
        self.cookie_file = cookie_file
        self.search_query = search_query
        self.filter_results = filter_results

        if self.output_format == "m4a":
            ytdl_format = "bestaudio[ext=m4a]/bestaudio/best"
        elif self.output_format == "opus":
            ytdl_format = "bestaudio[ext=webm]/bestaudio/best"
        else:
            ytdl_format = "bestaudio"

        yt_dlp_options = {
            "format": ytdl_format,
            "quiet": True,
            "no_warnings": True,
            "encoding": "UTF-8",
            "logger": YTDLLogger(),
            "cookiefile": self.cookie_file,
            "outtmpl": str((get_temp_path() / "%(id)s.%(ext)s").resolve()),
            "retries": 5,
        }

        if yt_dlp_args:
            yt_dlp_options = args_to_ytdlp_options(
                shlex.split(yt_dlp_args), yt_dlp_options
            )

        self.audio_handler = YoutubeDL(yt_dlp_options)

    def get_results(self, search_term: str, **kwargs) -> List[Result]:
        """
        Get results from audio provider.

        ### Arguments
        - search_term: The search term to use.
        - kwargs: Additional arguments.

        ### Returns
        - A list of results.
        """

        raise NotImplementedError

    def get_views(self, url: str) -> int:
        """
        Get the number of views for a video.

        ### Arguments
        - url: The url of the video.

        ### Returns
        - The number of views.
        """

        data = self.get_download_metadata(url)

        return data["view_count"]

    def search(self, song: Song, only_verified: bool = False) -> Optional[str]:
        """
        Search for a song and return best match.

        ### Arguments
        - song: The song to search for.

        ### Returns
        - The url of the best match or None if no match was found.
        """

        # Create initial search query
        search_query = create_song_title(song.name, song.artists).lower()
        if self.search_query:
            search_query = create_search_query(
                song, self.search_query, False, None, True
            )

        logger.debug("[%s] Searching for %s", song.song_id, search_query)

        isrc_urls: List[str] = []

        # search for song using isrc if it's available
        if song.isrc and self.SUPPORTS_ISRC and not self.search_query:
            isrc_results = self.get_results(song.isrc)

            if only_verified:
                isrc_results = [result for result in isrc_results if result.verified]
                logger.debug(
                    "[%s] Filtered to %s verified ISRC results",
                    song.song_id,
                    len(isrc_results),
                )

            isrc_urls = [result.url for result in isrc_results]
            logger.debug(
                "[%s] Found %s results for ISRC %s",
                song.song_id,
                len(isrc_results),
                song.isrc,
            )

            if len(isrc_results) == 1 and isrc_results[0].verified:
                # If we only have one verified result, return it
                # What's the chance of it being wrong?
                logger.debug(
                    "[%s] Returning only ISRC result %s",
                    song.song_id,
                    isrc_results[0].url,
                )

                return isrc_results[0].url

            if len(isrc_results) > 0:
                sorted_isrc_results = order_results(
                    isrc_results, song, self.search_query
                )

                # get the best result, if the score is above 80 return it
                best_isrc_results = sorted(
                    sorted_isrc_results.items(), key=lambda x: x[1], reverse=True
                )

                logger.debug(
                    "[%s] Filtered to %s ISRC results",
                    song.song_id,
                    len(best_isrc_results),
                )

                if len(best_isrc_results) > 0:
                    best_isrc = best_isrc_results[0]
                    if best_isrc[1] > 80.0:
                        logger.debug(
                            "[%s] Best ISRC result is %s with score %s",
                            song.song_id,
                            best_isrc[0].url,
                            best_isrc[1],
                        )

                        return best_isrc[0].url

        results: Dict[Result, float] = {}
        for options in self.GET_RESULTS_OPTS:
            # Query YTM by songs only first, this way if we get correct result on the first try
            # we don't have to make another request
            search_results = self.get_results(search_query, **options)

            if only_verified:
                search_results = [
                    result for result in search_results if result.verified
                ]

            logger.debug(
                "[%s] Found %s results for search query %s with options %s",
                song.song_id,
                len(search_results),
                search_query,
                options,
            )

            # Check if any of the search results is in the
            # first isrc results, since they are not hashable we have to check
            # by name
            isrc_result = next(
                (result for result in search_results if result.url in isrc_urls),
                None,
            )

            if isrc_result:
                logger.debug(
                    "[%s] Best ISRC result is %s", song.song_id, isrc_result.url
                )

                return isrc_result.url

            logger.debug(
                "[%s] Have to filter results: %s", song.song_id, self.filter_results
            )

            if self.filter_results:
                # Order results
                new_results = order_results(search_results, song, self.search_query)
            else:
                new_results = {}
                if len(search_results) > 0:
                    new_results = {search_results[0]: 100.0}

            logger.debug("[%s] Filtered to %s results", song.song_id, len(new_results))

            # song type results are always more accurate than video type,
            # so if we get score of 80 or above
            # we are almost 100% sure that this is the correct link
            if len(new_results) != 0:
                # get the result with highest score
                best_result, best_score = self.get_best_result(new_results)
                logger.debug(
                    "[%s] Best result is %s with score %s",
                    song.song_id,
                    best_result.url,
                    best_score,
                )

                if best_score >= 80 and best_result.verified:
                    logger.debug(
                        "[%s] Returning verified best result %s with score %s",
                        song.song_id,
                        best_result.url,
                        best_score,
                    )

                    return best_result.url

                # Update final results with new results
                results.update(new_results)

        # No matches found
        if not results:
            logger.debug("[%s] No results found", song.song_id)
            return None

        # get the result with highest score
        best_result, best_score = self.get_best_result(results)
        logger.debug(
            "[%s] Returning best result %s with score %s",
            song.song_id,
            best_result.url,
            best_score,
        )

        return best_result.url

    def get_best_result(self, results: Dict[Result, float]) -> Tuple[Result, float]:
        """
        Get the best match from the results
        using views and average match

        ### Arguments
        - results: A dictionary of results and their scores

        ### Returns
        - The best match URL and its score
        """

        best_results = get_best_matches(results, 8)

        # If we have only one result, return it
        if len(best_results) == 1:
            return best_results[0][0], best_results[0][1]

        # Initial best result based on the average match
        best_result = best_results[0]

        # If the best result has a score higher than 80%
        # and it's a isrc search, return it
        if best_result[1] > 80 and best_result[0].isrc_search:
            return best_result[0], best_result[1]

        # If we have more than one result,
        # return the one with the highest score
        # and most views
        if len(best_results) > 1:
            views: List[int] = []
            for best_result in best_results:
                if best_result[0].views:
                    views.append(best_result[0].views)
                else:
                    views.append(self.get_views(best_result[0].url))

            highest_views = max(views)
            lowest_views = min(views)

            if highest_views in (0, lowest_views):
                return best_result[0], best_result[1]

            weighted_results: List[Tuple[Result, float]] = []
            for index, best_result in enumerate(best_results):
                result_views = views[index]
                views_score = (
                    (result_views - lowest_views) / (highest_views - lowest_views)
                ) * 15
                score = min(best_result[1] + views_score, 100)
                weighted_results.append((best_result[0], score))

            # Now we return the result with the highest score
            return max(weighted_results, key=lambda x: x[1])

        return best_result[0], best_result[1]

    def get_download_metadata(self, url: str, download: bool = False) -> Dict:
        """
        Get metadata for a download using yt-dlp.

        ### Arguments
        - url: The url to get metadata for.

        ### Returns
        - A dictionary containing the metadata.
        """

        try:
            data = self.audio_handler.extract_info(url, download=download)

            if data:
                return data
        except Exception as exception:
            logger.debug(exception)
            raise AudioProviderError(f"YT-DLP download error - {url}") from exception

        raise AudioProviderError(f"No metadata found for the provided url {url}")

    @property
    def name(self) -> str:
        """
        Get the name of the provider.

        ### Returns
        - The name of the provider.
        """

        return self.__class__.__name__



File: spotdl/providers/audio/piped.py

"""
Piped module for downloading and searching songs.
"""

import logging
import shlex
from typing import Any, Dict, List, Optional

import requests
from yt_dlp import YoutubeDL

from spotdl.providers.audio.base import (
    ISRC_REGEX,
    AudioProvider,
    AudioProviderError,
    YTDLLogger,
)
from spotdl.types.result import Result
from spotdl.utils.config import GlobalConfig, get_temp_path
from spotdl.utils.formatter import args_to_ytdlp_options

__all__ = ["Piped"]
logger = logging.getLogger(__name__)

HEADERS = {
    "accept": "*/*",
}


class Piped(AudioProvider):
    """
    YouTube Music audio provider class
    """

    SUPPORTS_ISRC = True
    GET_RESULTS_OPTS: List[Dict[str, Any]] = [
        {"filter": "music_songs"},
        {"filter": "music_videos"},
    ]

    def __init__(  # pylint: disable=super-init-not-called
        self,
        output_format: str = "mp3",
        cookie_file: Optional[str] = None,
        search_query: Optional[str] = None,
        filter_results: bool = True,
        yt_dlp_args: Optional[str] = None,
    ) -> None:
        """
        Pipe audio provider class

        ### Arguments
        - output_directory: The directory to save the downloaded songs to.
        - output_format: The format to save the downloaded songs in.
        - cookie_file: The path to a file containing cookies to be used by YTDL.
        - search_query: The query to use when searching for songs.
        - filter_results: Whether to filter results.
        """

        self.output_format = output_format
        self.cookie_file = cookie_file
        self.search_query = search_query
        self.filter_results = filter_results

        if self.output_format == "m4a":
            ytdl_format = "best[ext=m4a]/best"
        elif self.output_format == "opus":
            ytdl_format = "best[ext=webm]/best"
        else:
            ytdl_format = "best"

        yt_dlp_options = {
            "format": ytdl_format,
            "quiet": True,
            "no_warnings": True,
            "encoding": "UTF-8",
            "logger": YTDLLogger(),
            "cookiefile": self.cookie_file,
            "outtmpl": f"{get_temp_path()}/%(id)s.%(ext)s",
            "retries": 5,
        }

        if yt_dlp_args:
            user_options = args_to_ytdlp_options(shlex.split(yt_dlp_args))
            yt_dlp_options.update(user_options)

        self.audio_handler = YoutubeDL(yt_dlp_options)
        self.session = requests.Session()

    def get_results(self, search_term: str, **kwargs) -> List[Result]:
        """
        Get results from YouTube Music API and simplify them

        ### Arguments
        - search_term: The search term to search for.
        - kwargs: other keyword arguments passed to the `YTMusic.search` method.

        ### Returns
        - A list of simplified results (dicts)
        """

        if kwargs is None:
            kwargs = {}

        params = {"q": search_term, **kwargs}
        if params.get("filter") is None:
            params["filter"] = "music_videos"

        response = self.session.get(
            "https://piped.video/search",
            params=params,
            headers=HEADERS,
            timeout=20,
        )

        if response.status_code != 200:
            raise AudioProviderError(
                f"Failed to get results for {search_term} from Piped: {response.text}"
            )

        search_results = response.json()

        # Simplify results
        results = []
        for result in search_results["items"]:
            if result["type"] != "stream":
                continue

            isrc_result = ISRC_REGEX.search(search_term)

            results.append(
                Result(
                    source="piped",
                    url=f"https://piped.video{result['url']}",
                    verified=kwargs.get("filter") == "music_songs",
                    name=result["title"],
                    duration=result["duration"],
                    author=result["uploaderName"],
                    result_id=result["url"].split("?v=")[1],
                    artists=(
                        (result["uploaderName"],)
                        if kwargs.get("filter") == "music_songs"
                        else None
                    ),
                    isrc_search=isrc_result is not None,
                    search_query=search_term,
                )
            )

        return results

    def get_download_metadata(self, url: str, download: bool = False) -> Dict:
        """
        Get metadata for a download using yt-dlp.

        ### Arguments
        - url: The url to get metadata for.

        ### Returns
        - A dictionary containing the metadata.
        """

        url_id = url.split("?v=")[1]
        piped_response = requests.get(
            f"https://piped.video/streams/{url_id}",
            timeout=10,
            proxies=GlobalConfig.get_parameter("proxies"),
        )

        if piped_response.status_code != 200:
            raise AudioProviderError(
                f"Failed to get metadata for {url} from Piped: {piped_response.text}"
            )

        piped_data = piped_response.json()

        yt_dlp_json = {
            "title": piped_data["title"],
            "id": url_id,
            "view_count": piped_data["views"],
            "extractor": "Generic",
            "formats": [],
        }

        for audio_stream in piped_data["audioStreams"]:
            yt_dlp_json["formats"].append(
                {
                    "url": audio_stream["url"],
                    "ext": "webm" if audio_stream["codec"] == "opus" else "m4a",
                    "abr": audio_stream["quality"].split(" ")[0],
                    "filesize": audio_stream["contentLength"],
                }
            )

        return self.audio_handler.process_video_result(yt_dlp_json, download=download)



File: spotdl/utils/logging.py

"""
Module for logging
"""

import logging
from typing import Optional

from rich import get_console
from rich.console import ConsoleRenderable
from rich.logging import RichHandler
from rich.markup import escape
from rich.text import Text
from rich.theme import Theme
from rich.traceback import install

__all__ = [
    "CRITICAL",
    "FATAL",
    "ERROR",
    "WARNING",
    "WARN",
    "INFO",
    "DEBUG",
    "MATCH",
    "NOTSET",
    "init_logging",
    "SpotdlFormatter",
    "LEVEL_TO_NAME",
    "NAME_TO_LEVEL",
]

# https://github.com/python/cpython/blob/3.10/Lib/logging/__init__.py
CRITICAL = 50
FATAL = CRITICAL
ERROR = 40
WARNING = 30
WARN = WARNING
INFO = 20
DEBUG = 10
MATCH = 5
NOTSET = 0

LEVEL_TO_NAME = {
    CRITICAL: "CRITICAL",
    ERROR: "ERROR",
    WARNING: "WARNING",
    INFO: "INFO",
    MATCH: "MATCH",
    DEBUG: "DEBUG",
    NOTSET: "NOTSET",
}

NAME_TO_LEVEL = {
    "CRITICAL": CRITICAL,
    "FATAL": FATAL,
    "ERROR": ERROR,
    "WARN": WARNING,
    "WARNING": WARNING,
    "INFO": INFO,
    "MATCH": MATCH,
    "DEBUG": DEBUG,
    "NOTSET": NOTSET,
}

THEME = Theme(
    {
        "bar.back": "grey23",
        "bar.complete": "rgb(165,66,129)",
        "bar.finished": "rgb(114,156,31)",
        "bar.pulse": "rgb(165,66,129)",
        "general": "green",
        "nonimportant": "rgb(40,100,40)",
        "progress.data.speed": "red",
        "progress.description": "none",
        "progress.download": "green",
        "progress.filesize": "green",
        "progress.filesize.total": "green",
        "progress.percentage": "green",
        "progress.remaining": "rgb(40,100,40)",
        "logging.level.debug": "blue",
        "logging.level.info": "green",
        "logging.level.warning": "yellow",
        "logging.level.error": "red",
        "logging.level.critical": "bold red",
    }
)


class SpotdlFormatter(logging.Formatter):
    """
    A custom logger for spotdl.
    """

    def format(self, record: logging.LogRecord) -> str:
        """
        Format a log record.
        """

        result = escape(super().format(record))

        msg = result
        if record.levelno == DEBUG:
            msg = f"[blue]{result}"

        if record.levelno == MATCH:
            msg = f"[magenta]{result}"

        if record.levelno == INFO:
            msg = f"[green]{result}"

        if record.levelno == WARNING:
            msg = f"[yellow]{result}"

        if record.levelno == ERROR:
            msg = f"[red]{result}"

        if record.levelno == CRITICAL:
            msg = f"[bold red]{result}"

        return msg


class SpotdlHandler(RichHandler):
    """
    A custom logging handler for spotdl.
    In this case, it's just a wrapper around the rich handler.
    To not highlight keywords in info messages
    """

    def render_message(
        self, record: logging.LogRecord, message: str
    ) -> "ConsoleRenderable":
        """Render message text in to Text.

        ### Arguments
        - record: logging Record.
        - message: String containing log message.

        ### Returns
        - ConsoleRenderable: Renderable to display log message.
        """

        use_markup = getattr(record, "markup", self.markup)
        message_text = Text.from_markup(message) if use_markup else Text(message)

        highlighter = getattr(record, "highlighter", self.highlighter)

        # Don't highlight info messages
        if highlighter and record.levelno != INFO:
            message_text = highlighter(message_text)

        if not hasattr(self, "keywords"):
            self.keywords = self.KEYWORDS

        # Don't highlight keywords in info messages
        if self.keywords and record.levelno != INFO:
            message_text.highlight_words(self.keywords, "logging.keyword")

        return message_text


def init_logging(log_level: str, log_format: Optional[str] = None):
    """
    Initialize logging for spotdl.

    ### Arguments
    - `console`: The console to use.
    - `log_level`: The log level to use.
    """

    # Don't log too much
    logging.getLogger("requests").setLevel(logging.WARNING)
    logging.getLogger("urllib3").setLevel(logging.WARNING)
    logging.getLogger("spotipy").setLevel(logging.WARNING)
    logging.getLogger("asyncio").setLevel(logging.WARNING)
    logging.getLogger("syncedlyrics").setLevel(logging.WARNING)
    logging.getLogger("bandcamp_api").setLevel(logging.WARNING)
    logging.getLogger("beautifulsoup4").setLevel(logging.WARNING)
    logging.getLogger("pytube").setLevel(logging.ERROR)

    # Create console
    console = get_console()
    console.push_theme(THEME)

    # Add matching level loggers
    logging.addLevelName(MATCH, "MATCH")

    # Create a rich handler
    rich_handler = SpotdlHandler(
        show_time=log_level == "DEBUG",
        log_time_format="[%X]",
        omit_repeated_times=False,
        console=console,
        level=log_level,
        markup=True,
        show_path=log_level == "DEBUG",
        show_level=log_level == "DEBUG",
        rich_tracebacks=True,
    )

    msg_format = "%(message)s"
    if log_format is None:
        if log_level == "DEBUG":
            msg_format = "%(threadName)s - %(message)s"
    else:
        msg_format = log_format

    # Add rich handler to spotdl logger
    rich_handler.setFormatter(SpotdlFormatter(msg_format))

    # Create spotdl logger
    spotdl_logger = logging.getLogger("spotdl")

    # Setup spotdl logger
    spotdl_logger.setLevel(log_level)
    spotdl_logger.addHandler(rich_handler)

    # Install rich traceback handler
    install(show_locals=False, extra_lines=1, console=console)



File: spotdl/utils/console.py

"""
Module for holding console related actions.
"""

import json
import sys

from spotdl.utils.config import DEFAULT_CONFIG, get_config_file
from spotdl.utils.ffmpeg import download_ffmpeg as ffmpeg_download
from spotdl.utils.ffmpeg import get_local_ffmpeg, is_ffmpeg_installed
from spotdl.utils.github import check_for_updates as get_update_status

__all__ = [
    "is_frozen",
    "is_executable",
    "generate_initial_config",
    "generate_config",
    "check_for_updates",
    "download_ffmpeg",
    "ACTIONS",
]


def is_frozen():
    """
    Check if the application is frozen.

    ### Returns
    - `True` if the application is frozen, `False` otherwise.
    """

    return getattr(sys, "frozen", False)


def is_executable():
    """
    Check if the application is an prebuilt executable.
    And has been launched with double click.

    ### Returns
    - `True` if the application is an prebuilt executable, `False` otherwise.
    """

    return is_frozen() and len(sys.argv) == 1


def generate_initial_config():
    """
    Generate the initial config file if it doesn't exist.
    """

    if get_config_file().is_file() is False:
        config_path = get_config_file()
        with open(config_path, "w", encoding="utf-8") as config_file:
            json.dump(DEFAULT_CONFIG, config_file, indent=4)


def generate_config():
    """
    Generate the config file if it doesn't exist
    This is done before the argument parser so it doesn't requires `operation`
    and `query` to be passed.
    """

    config_path = get_config_file()
    if config_path.exists():
        overwrite_config = input("Config file already exists. Overwrite? (y/N): ")

        if overwrite_config.lower() != "y":
            print("Exiting...")
            return None

    with open(config_path, "w", encoding="utf-8") as config_file:
        json.dump(DEFAULT_CONFIG, config_file, indent=4)

    print(f"Config file generated at {config_path}")

    return None


def check_for_updates():
    """
    Check for updates to the current version.
    """

    version_message = get_update_status()

    print(version_message)


def download_ffmpeg():
    """
    Handle ffmpeg download process and print the result.
    """

    if get_local_ffmpeg() is not None or is_ffmpeg_installed():
        overwrite_ffmpeg = input(
            "FFmpeg is already installed. Do you want to overwrite it? (y/N): "
        )

        if overwrite_ffmpeg.lower() == "y":
            local_ffmpeg = ffmpeg_download()

            if local_ffmpeg.is_file():
                print(f"FFmpeg successfully downloaded to {local_ffmpeg.absolute()}")
            else:
                print("FFmpeg download failed")
    else:
        print("Downloading FFmpeg...")
        download_path = ffmpeg_download()

        if download_path.is_file():
            print(f"FFmpeg successfully downloaded to {download_path.absolute()}")
        else:
            print("FFmpeg download failed")


ACTIONS = {
    "--generate-config": generate_config,
    "--check-for-updates": check_for_updates,
    "--download-ffmpeg": download_ffmpeg,
}



File: spotdl/utils/formatter.py

"""
Module for formatting songs into strings.
Contains functions to create search queries and song titles
and file names.
"""

import copy
import logging
import re
from functools import lru_cache
from pathlib import Path
from typing import Any, Dict, List, Optional
from unicodedata import normalize

import pykakasi
from rapidfuzz import fuzz
from slugify import slugify as py_slugify
from yt_dlp.options import create_parser
from yt_dlp.options import optparse as yt_dlp_optparse
from yt_dlp.utils import sanitize_filename

from spotdl.types.song import Song

__all__ = [
    "VARS",
    "JAP_REGEX",
    "DISALLOWED_REGEX",
    "create_song_title",
    "sanitize_string",
    "slugify",
    "format_query",
    "create_search_query",
    "create_file_name",
    "parse_duration",
    "to_ms",
    "restrict_filename",
    "ratio",
    "smart_split",
    "create_path_object",
    "args_to_ytdlp_options",
]

VARS = [
    "{title}",
    "{artists}",
    "{artist}",
    "{album}",
    "{album-artist}",
    "{genre}",
    "{disc-number}",
    "{disc-count}",
    "{duration}",
    "{year}",
    "{original-date}",
    "{track-number}",
    "{tracks-count}",
    "{isrc}",
    "{track-id}",
    "{publisher}",
    "{list-length}",
    "{list-position}",
    "{list-name}",
    "{output-ext}",
]

KKS = pykakasi.kakasi()

JAP_REGEX = re.compile(
    "[\u3000-\u303f\u3040-\u309f\u30a0-\u30ff\uff00-\uff9f\u4e00-\u9faf\u3400-\u4dbf]"
)

DISALLOWED_REGEX = re.compile(r"[^-a-zA-Z0-9\!\@\$]+")
YT_DLP_PARSER = create_parser()

logger = logging.getLogger(__name__)


def create_song_title(song_name: str, song_artists: List[str]) -> str:
    """
    Create the song title.

    ### Arguments
    - song_name: the name of the song
    - song_artists: the list of artists of the song

    ### Returns
    - the song title

    ### Notes
    - Example: "Artist1, Artist2 - Song Name"

    """

    joined_artists = ", ".join(song_artists)
    if len(song_artists) >= 1:
        return f"{joined_artists} - {song_name}"

    return song_name


def sanitize_string(string: str) -> str:
    """
    Sanitize the filename to be used in the file system.

    ### Arguments
    - string: the string to sanitize

    ### Returns
    - the sanitized string
    """

    output = string

    # this is windows specific (disallowed chars)
    output = "".join(char for char in output if char not in "/?\\*|<>")

    # double quotes (") and semi-colons (:) are also disallowed characters but we would
    # like to retain their equivalents, so they aren't removed in the prior loop
    output = output.replace('"', "'").replace(":", "-")

    return output


@lru_cache()
def slugify(string: str) -> str:
    """
    Slugify the string.

    ### Arguments
    - string: the string to slugify

    ### Returns
    - the slugified string
    """

    # Replace ambiguous characters
    if not JAP_REGEX.search(string):
        # If string doesn't have japanese characters
        # return early
        return py_slugify(string, regex_pattern=DISALLOWED_REGEX.pattern)

    # Workaround for japanese characters
    # because slugify incorrectly converts them
    # to latin characters
    normal_slug = py_slugify(
        string,
        regex_pattern=JAP_REGEX.pattern,
    )

    results = KKS.convert(normal_slug)

    result = ""
    for index, item in enumerate(results):
        result += item["hepburn"]
        if not (
            item["kana"] == item["hepburn"]
            or item["kana"] == item["hepburn"]
            or (
                item == results[-1]
                or results[index + 1]["kana"] == results[index + 1]["hepburn"]
            )
        ):
            result += "-"

    return py_slugify(result, regex_pattern=DISALLOWED_REGEX.pattern)


def format_query(
    song: Song,
    template: str,
    santitize: bool,
    file_extension: Optional[str] = None,
    short: bool = False,
) -> str:
    """
    Replace template variables with the actual values.

    ### Arguments
    - song: the song object
    - template: the template string
    - santitize: whether to sanitize the string
    - file_extension: the file extension to use
    - short: whether to use the short version of the template

    ### Returns
    - the formatted string
    """

    if "{output-ext}" in template and file_extension is None:
        raise ValueError("file_extension is None, but template contains {output-ext}")

    for key, val in [
        ("{list-length}", song.list_length),
        ("{list-position}", song.list_position),
        ("{list-name}", song.list_name),
    ]:
        if not (key in template and val is None):
            continue

        logger.warning(
            "Template contains %s, but it's value is None. Replacing with empty string.",
            key,
        )

        template = template.replace(key, "")
        template = template.replace(r"//", r"/")

    # If template has only {output-ext}, fix it
    if template in ["/.{output-ext}", ".{output-ext}"]:
        template = "{artists} - {title}.{output-ext}"

    # Remove artists from the list that are already in the title
    if short:
        artists = [
            artist
            for artist in song.artists
            if slugify(artist) not in slugify(song.name)
        ]

        # Add the main artist again to the list
        if len(artists) == 0 or artists[0] != song.artists[0]:
            artists.insert(0, song.artists[0])
    else:
        artists = song.artists

    artists_str = ", ".join(artists)

    # the code below is valid, song_list is actually checked for None
    formats = {
        "{title}": song.name,
        "{artists}": song.artists[0] if short is True else artists_str,
        "{artist}": song.artists[0],
        "{album}": song.album_name,
        "{album-artist}": song.album_artist,
        "{genre}": song.genres[0] if song.genres else "",
        "{disc-number}": song.disc_number,
        "{disc-count}": song.disc_count,
        "{duration}": song.duration,
        "{year}": song.year,
        "{original-date}": song.date,
        "{track-number}": f"{int(song.track_number):02d}" if song.track_number else "",
        "{tracks-count}": song.tracks_count,
        "{isrc}": song.isrc,
        "{track-id}": song.song_id,
        "{publisher}": song.publisher,
        "{output-ext}": file_extension,
        "{list-name}": song.list_name,
        "{list-position}": str(song.list_position).zfill(len(str(song.list_length))),
        "{list-length}": song.list_length,
    }

    if santitize:
        # sanitize the values in formats dict
        for key, value in formats.items():
            if value is None:
                continue

            formats[key] = sanitize_string(str(value))

    # Replace all the keys with the values
    for key, value in formats.items():
        template = template.replace(key, str(value))

    return template


def create_search_query(
    song: Song,
    template: str,
    santitize: bool,
    file_extension: Optional[str] = None,
    short: bool = False,
) -> str:
    """
    Create the search query for the song.

    ### Arguments
    - song: the song object
    - template: the template string
    - santitize: whether to sanitize the string
    - file_extension: the file extension to use
    - short: whether to use the short version of the template

    ### Returns
    - the formatted string
    """

    # If template does not contain any of the keys,
    # append {artist} - {title} at the beginning of the template
    if not any(key in template for key in VARS):
        template = "{artist} - {title}" + template

    return format_query(song, template, santitize, file_extension, short=short)


def create_file_name(
    song: Song,
    template: str,
    file_extension: str,
    restrict: Optional[str] = None,
    short: bool = False,
    file_name_length: Optional[int] = None,
) -> Path:
    """
    Create the file name for the song, by replacing template variables with the actual values.

    ### Arguments
    - song: the song object
    - template: the template string
    - file_extension: the file extension to use
    - restrict: sanitization to apply to the filename
    - short: whether to use the short version of the template
    - file_name_length: the maximum length of the file name

    ### Returns
    - the formatted string as a Path object
    """

    temp_song = copy.deepcopy(song)

    # If template does not contain any of the keys,
    # append {artists} - {title}.{output-ext} to it
    if not any(key in template for key in VARS) and template != "":
        template += "/{artists} - {title}.{output-ext}"

    if template == "":
        template = "{artists} - {title}.{output-ext}"

    # If template ends with a slash. Does not have a file name with extension
    # at the end of the template, append {artists} - {title}.{output-ext} to it
    if template.endswith("/") or template.endswith(r"\\") or template.endswith("\\\\"):
        template += "/{artists} - {title}.{output-ext}"

    # If template does not end with {output-ext}, append it to the end of the template
    if not template.endswith(".{output-ext}"):
        template += ".{output-ext}"

    formatted_string = format_query(
        song=song,
        template=template,
        santitize=True,
        file_extension=file_extension,
        short=short,
    )

    file = create_path_object(formatted_string)

    length_limit = file_name_length or 255

    # Check if the file name length is greater than the limit
    if len(file.name) < length_limit:
        # Restrict the filename if needed
        if restrict and restrict != "none":
            return restrict_filename(file, restrict == "strict")

        return file

    if short is False:
        return create_file_name(
            song,
            template,
            file_extension,
            restrict=restrict,
            short=True,
            file_name_length=length_limit,
        )

    non_template_chars = re.findall(r"(?<!{)[^{}]+(?![^{}]*})", template)
    half_length = int((length_limit * 0.50) - (len("".join(non_template_chars)) / 2))

    # Path template is already short, but we still can't create a file
    # so we reduce it even further
    is_long_artist = len(temp_song.artist) > half_length
    is_long_title = len(temp_song.name) > half_length

    path_separator = "/" if "/" in template else "\\"
    name_template_parts = template.rsplit(path_separator, 1)
    name_template = (
        name_template_parts[1]
        if len(name_template_parts) > 1
        else name_template_parts[0]
    )

    if is_long_artist:
        logger.warning(
            "%s: Song artist is too long. Using only part of song artist.",
            temp_song.display_name,
        )

        temp_song.artist = smart_split(temp_song.artist, half_length, None)
        temp_song.artists = [temp_song.artist]

    if is_long_title:
        logger.warning(
            "%s: File name is too long. Using only part of the song title.",
            temp_song.display_name,
        )

        temp_song.name = smart_split(temp_song.name, half_length, None)

    new_file = create_path_object(
        format_query(
            song=temp_song,
            template=name_template,
            santitize=True,
            file_extension=file_extension,
            short=short,
        )
    )

    if len(new_file.name) > length_limit:
        logger.warning(
            "File name is still too long. "
            "Using default file name with shortened artist and title."
        )

        if template == "{artist} - {title}.{output-ext}":
            raise ValueError(
                "File name is still too long, "
                "but the template is already short. "
                "Please try other template, "
                "increase the file name length limit."
            )

        return create_file_name(
            temp_song,
            "{artist} - {title}.{output-ext}",
            file_extension,
            restrict=restrict,
            short=True,
            file_name_length=length_limit,
        )

    return new_file


def parse_duration(duration: Optional[str]) -> float:
    """
    Convert string value of time (duration: "25:36:59") to a float value of seconds (92219.0)

    ### Arguments
    - duration: the string value of time

    ### Returns
    - the float value of seconds
    """

    if duration is None:
        return 0.0

    try:
        # {(1, "s"), (60, "m"), (3600, "h")}
        mapped_increments = zip([1, 60, 3600], reversed(duration.split(":")))
        seconds = sum(multiplier * int(time) for multiplier, time in mapped_increments)
        return float(seconds)

    # This usually occurs when the wrong string is mistaken for the duration
    except (ValueError, TypeError, AttributeError):
        return 0.0


def to_ms(
    string: Optional[str] = None, precision: Optional[int] = None, **kwargs
) -> float:
    """
    Convert a string to milliseconds.

    ### Arguments
    - string: the string to convert
    - precision: the number of decimals to round to
    - kwargs: the keyword args to convert

    ### Returns
    - the milliseconds

    ### Notes
    - You can either pass a string,
    - or a set of keyword args ("hour", "min", "sec", "ms") to convert.
    - If "precision" is set, the result is rounded to the number of decimals given.
    - From: https://gist.github.com/Hellowlol/5f8545e999259b4371c91ac223409209
    """

    if string:
        hour = int(string[0:2])
        minute = int(string[3:5])
        sec = int(string[6:8])
        milliseconds = int(string[10:11])
    else:
        hour = int(kwargs.get("hour", 0))
        minute = int(kwargs.get("min", 0))
        sec = int(kwargs.get("sec", 0))
        milliseconds = int(kwargs.get("ms", 0))

    result = (
        (hour * 60 * 60 * 1000) + (minute * 60 * 1000) + (sec * 1000) + milliseconds
    )

    if precision and isinstance(precision, int):
        return round(result, precision)

    return result


def restrict_filename(pathobj: Path, strict: bool = True) -> Path:
    """
    Sanitizes the filename part of a Path object. Returns modified object.

    ### Arguments
    - pathobj: the Path object to sanitize
    - strict: whether sanitization should be strict

    ### Returns
    - the modified Path object

    ### Notes
    - Based on the `sanitize_filename` function from yt-dlp
    """
    if strict:
        result = sanitize_filename(pathobj.name, True, False)  # type: ignore
        result = result.replace("_-_", "-")
    else:
        result = (
            normalize("NFKD", pathobj.name).encode("ascii", "ignore").decode("utf-8")
        )

    if not result:
        result = "_"

    return pathobj.with_name(result)


@lru_cache()
def ratio(string1: str, string2: str) -> float:
    """
    Wrapper for fuzz.ratio
    with lru_cache

    ### Arguments
    - string1: the first string
    - string2: the second string

    ### Returns
    - the ratio
    """

    return fuzz.ratio(string1, string2)


def smart_split(
    string: str, max_length: int, separators: Optional[List[str]] = None
) -> str:
    """
    Split a string into a list of strings
    with a maximum length of max_length.
    Stops at the first separator that produces a string
    with a length less than max_length.

    ### Arguments
    - string: the string to split
    - max_length: the maximum length of string
    - separators: the separators to split the string with

    ### Returns
    - the new string
    """

    if separators is None:
        separators = ["-", ",", " ", ""]

    for separator in separators:
        parts = string.split(separator if separator != "" else None)
        new_string = separator.join(parts[:1])
        for part in parts[1:]:
            if len(new_string) + len(separator) + len(part) > max_length:
                break
            new_string += separator + part

        if len(new_string) <= max_length:
            return new_string

    return string[:max_length]


def create_path_object(string: str) -> Path:
    """
    Create a Path object from a string.
    Sanitizes the filename part of the Path object.

    ### Arguments
    - string: the string to convert

    ### Returns
    - the Path object
    """

    # Parse template as Path object
    file = Path(string)

    santitized_parts = []
    for part in file.parts:
        match = re.search(r"[^\.*](.*)[^\.*$]", part)
        if match and part != ".spotdl":
            santitized_parts.append(match.group(0))
        else:
            santitized_parts.append(part)

    # Join the parts of the path
    return Path(*santitized_parts)


def args_to_ytdlp_options(
    argument_list: List[str], defaults: Optional[Dict[str, Any]] = None
) -> Dict[str, Any]:
    """
    Convert a list of arguments to a dictionary of options.

    ### Arguments
    - argument_list: the list of arguments
    - defaults: the default options

    ### Returns
    - the dictionary of options
    """

    new_args = YT_DLP_PARSER.parse_args(argument_list, yt_dlp_optparse.Values(defaults))

    return vars(new_args[0])



File: spotdl/utils/web.py

"""
Module which contains the web server related function
FastAPI routes/classes etc.
"""

import argparse
import asyncio
import logging
import mimetypes
import os
import shutil
from argparse import Namespace
from pathlib import Path
from typing import Any, Dict, List, Optional, Union

from fastapi import (
    APIRouter,
    Depends,
    FastAPI,
    HTTPException,
    Query,
    Response,
    WebSocket,
    WebSocketDisconnect,
)
from fastapi.responses import FileResponse
from fastapi.staticfiles import StaticFiles
from starlette.types import Scope
from uvicorn import Server

from spotdl._version import __version__
from spotdl.download.downloader import Downloader
from spotdl.download.progress_handler import ProgressHandler, SongTracker
from spotdl.types.album import Album
from spotdl.types.artist import Artist
from spotdl.types.options import (
    DownloaderOptionalOptions,
    DownloaderOptions,
    WebOptions,
)
from spotdl.types.playlist import Playlist
from spotdl.types.song import Song
from spotdl.utils.arguments import create_parser
from spotdl.utils.config import (
    DOWNLOADER_OPTIONS,
    create_settings_type,
    get_spotdl_path,
)
from spotdl.utils.github import RateLimitError, get_latest_version, get_status
from spotdl.utils.search import get_search_results

__all__ = [
    "ALLOWED_ORIGINS",
    "SPAStaticFiles",
    "Client",
    "ApplicationState",
    "router",
    "app_state",
    "get_current_state",
    "get_client",
    "websocket_endpoint",
    "song_from_url",
    "query_search",
    "download_url",
    "download_file",
    "get_settings",
    "update_settings",
    "fix_mime_types",
]

ALLOWED_ORIGINS = [
    "http://localhost:8800",
    "http://127.0.0.1:8800",
    "https://localhost:8800",
    "https://127.0.0.1:8800",
]


class SPAStaticFiles(StaticFiles):
    """
    Override the static files to serve the index.html and other assets.
    """

    async def get_response(self, path: str, scope: Scope) -> Response:
        """
        Serve static files from the SPA.

        ### Arguments
        - path: The path to the file.
        - scope: The scope of the request.

        ### Returns
        - returns the response.
        """

        response = await super().get_response(path, scope)
        if response.status_code == 404:
            response = await super().get_response(".", scope)

        response.headers.setdefault(
            "Cache-Control", "max-age=0, no-cache, no-store, , must-revalidate"
        )
        response.headers.setdefault("Pragma", "no-cache")
        response.headers.setdefault("Expires", "0")

        return response


class Client:
    """
    Holds the client's state.
    """

    def __init__(
        self,
        websocket: WebSocket,
        client_id: str,
    ):
        """
        Initialize the WebSocket handler.
        ### Arguments
        - websocket: The WebSocket instance.
        - client_id: The client's ID.
        - downloader_settings: The downloader settings.
        """

        self.downloader_settings = DownloaderOptions(
            **create_settings_type(
                Namespace(config=False),
                dict(app_state.downloader_settings),
                DOWNLOADER_OPTIONS,
            )  # type: ignore
        )

        self.websocket = websocket
        self.client_id = client_id
        self.downloader = Downloader(
            settings=self.downloader_settings, loop=app_state.loop
        )

        self.downloader.progress_handler.web_ui = True

    async def connect(self):
        """
        Called when a new client connects to the websocket.
        """

        await self.websocket.accept()

        # Add the connection to the list of connections
        app_state.clients[self.client_id] = self
        app_state.logger.info("Client %s connected", self.client_id)

    async def send_update(self, update: Dict[str, Any]):
        """
        Send an update to the client.

        ### Arguments
        - update: The update to send.
        """

        await self.websocket.send_json(update)

    def song_update(self, progress_handler: SongTracker, message: str):
        """
        Called when a song updates.

        ### Arguments
        - progress_handler: The progress handler.
        - message: The message to send.
        """

        update_message = {
            "song": progress_handler.song.json,
            "progress": progress_handler.progress,
            "message": message,
        }

        asyncio.run_coroutine_threadsafe(
            self.send_update(update_message), app_state.loop
        )

    @classmethod
    def get_instance(cls, client_id: str) -> Optional["Client"]:
        """
        Get the WebSocket instance for a client.

        ### Arguments
        - client_id: The client's ID.

        ### Returns
        - returns the WebSocket instance.
        """

        instance = app_state.clients.get(client_id)
        if instance:
            return instance

        app_state.logger.error("Client %s not found", client_id)

        return None


class ApplicationState:
    """
    Class that holds the application state.
    """

    api: FastAPI
    server: Server
    loop: asyncio.AbstractEventLoop
    web_settings: WebOptions
    downloader_settings: DownloaderOptions
    clients: Dict[str, Client] = {}
    logger: logging.Logger


router = APIRouter()
app_state: ApplicationState = ApplicationState()


def get_current_state() -> ApplicationState:
    """
    Get the current state of the application.

    ### Returns
    - returns the application state.
    """

    return app_state


def get_client(client_id: Union[str, None] = Query(default=None)) -> Client:
    """
    Get the client's state.

    ### Arguments
    - client_id: The client's ID.

    ### Returns
    - returns the client's state.
    """

    if client_id is None:
        raise HTTPException(status_code=400, detail="client_id is required")

    instance = Client.get_instance(client_id)
    if instance is None:
        raise HTTPException(status_code=404, detail="client not found")

    return instance


@router.websocket("/api/ws")
async def websocket_endpoint(websocket: WebSocket, client_id: str):
    """
    Websocket endpoint.

    ### Arguments
    - websocket: The WebSocket instance.
    """

    await Client(websocket, client_id).connect()

    try:
        while True:
            await websocket.receive_json()
    except WebSocketDisconnect:
        app_state.clients.pop(client_id, None)

        if (
            len(app_state.clients) == 0
            and app_state.web_settings["keep_alive"] is False
        ):
            app_state.logger.debug(
                "No active connections, waiting 1s before shutting down"
            )

            await asyncio.sleep(1)

            # Wait 1 second before shutting down
            # This is to prevent the server from shutting down when a client
            # disconnects and reconnects quickly (e.g. when refreshing the page)
            if len(app_state.clients) == 0:
                # Perform a clean exit
                app_state.logger.info("Shutting down server, no active connections")
                app_state.server.force_exit = True
                app_state.server.should_exit = True
                await app_state.server.shutdown()


# Deprecated
@router.get("/api/song/url", response_model=None)
def song_from_url(url: str) -> Song:
    """
    Search for a song on spotify using url.

    ### Arguments
    - url: The url to search.

    ### Returns
    - returns the first result as a Song object.
    """

    return Song.from_url(url)


@router.get("/api/url", response_model=None)
def songs_from_url(url: str) -> List[Song]:
    """
    Search for a song, playlist, artist or album on spotify using url.

    ### Arguments
    - url: The url to search.

    ### Returns
    - returns a list with Song objects to be downloaded.
    """

    if "playlist" in url:
        playlist = Playlist.from_url(url)
        return list(map(Song.from_url, playlist.urls))
    if "album" in url:
        album = Album.from_url(url)
        return list(map(Song.from_url, album.urls))
    if "artist" in url:
        artist = Artist.from_url(url)
        return list(map(Song.from_url, artist.urls))

    return [Song.from_url(url)]


@router.get("/api/version", response_model=None)
def version() -> str:
    """
    Get the current version
    This method is created to ensure backward compatibility of the web app,
    as the web app is updated with the latest regardless of the backend version

    ### Returns
    -  returns the version of the app
    """

    return __version__


@router.on_event("shutdown")
async def shutdown_event():
    """
    Called when the server is shutting down.
    """

    if (
        not app_state.web_settings["keep_sessions"]
        and not app_state.web_settings["web_use_output_dir"]
    ):
        app_state.logger.info("Removing sessions directories")
        sessions_dir = Path(get_spotdl_path(), "web/sessions")
        if sessions_dir.exists():
            shutil.rmtree(sessions_dir)


@router.get("/api/songs/search", response_model=None)
def query_search(query: str) -> List[Song]:
    """
    Parse search term and return list of Song objects.

    ### Arguments
    - query: The query to parse.

    ### Returns
    - returns a list of Song objects.
    """

    return get_search_results(query)


@router.post("/api/download/url")
async def download_url(
    url: str,
    client: Client = Depends(get_client),
    state: ApplicationState = Depends(get_current_state),
) -> Optional[str]:
    """
    Download songs using Song url.

    ### Arguments
    - url: The url to download.

    ### Returns
    - returns the file path if the song was downloaded.
    """

    if state.web_settings.get("web_use_output_dir", False):
        client.downloader.settings["output"] = client.downloader_settings["output"]
    else:
        client.downloader.settings["output"] = str(
            (get_spotdl_path() / f"web/sessions/{client.client_id}").absolute()
        )

    client.downloader.progress_handler = ProgressHandler(
        simple_tui=True,
        update_callback=client.song_update,
    )

    try:
        # Fetch song metadata
        song = Song.from_url(url)

        # Download Song
        _, path = await client.downloader.pool_download(song)

        if path is None:
            state.logger.error(f"Failure downloading {song.name}")

            raise HTTPException(
                status_code=500, detail=f"Error downloading: {song.name}"
            )

        return str(path.absolute())

    except Exception as exception:
        state.logger.error(f"Error downloading! {exception}")

        raise HTTPException(
            status_code=500, detail=f"Error downloading: {exception}"
        ) from exception


@router.get("/api/download/file")
async def download_file(
    file: str,
    client: Client = Depends(get_client),
    state: ApplicationState = Depends(get_current_state),
):
    """
    Download file using path.

    ### Arguments
    - file: The file path.
    - client: The client's state.

    ### Returns
    - returns the file response, filename specified to return as attachment.
    """

    expected_path = str((get_spotdl_path() / "web/sessions").absolute())
    if state.web_settings.get("web_use_output_dir", False):
        expected_path = str(
            Path(client.downloader_settings["output"].split("{", 1)[0]).absolute()
        )

    if (not file.endswith(client.downloader_settings["format"])) or (
        not file.startswith(expected_path)
    ):
        raise HTTPException(status_code=400, detail="Invalid download path.")

    return FileResponse(
        file,
        filename=os.path.basename(file),
    )


@router.get("/api/settings")
def get_settings(
    client: Client = Depends(get_client),
) -> DownloaderOptions:
    """
    Get client settings.

    ### Arguments
    - client: The client's state.

    ### Returns
    - returns the settings.
    """

    return client.downloader_settings


@router.post("/api/settings/update")
def update_settings(
    settings: DownloaderOptionalOptions,
    client: Client = Depends(get_client),
    state: ApplicationState = Depends(get_current_state),
) -> DownloaderOptions:
    """
    Update client settings, and re-initialize downloader.

    ### Arguments
    - settings: The settings to change.
    - client: The client's state.
    - state: The application state.

    ### Returns
    - returns True if the settings were changed.
    """

    # Create shallow copy of settings
    settings_cpy = client.downloader_settings.copy()

    # Update settings with new settings that are not None
    settings_cpy.update({k: v for k, v in settings.items() if v is not None})  # type: ignore

    state.logger.info(f"Applying settings: {settings_cpy}")

    new_settings = DownloaderOptions(**settings_cpy)  # type: ignore

    # Re-initialize downloader
    client.downloader_settings = new_settings
    client.downloader = Downloader(
        new_settings,
        loop=state.loop,
    )

    return new_settings


@router.get("/api/check_update")
def check_update() -> bool:
    """
    Check for update.

    ### Returns
    - returns True if there is an update.
    """

    try:
        _, ahead, _ = get_status(__version__, "master")
        if ahead > 0:
            return True
    except RuntimeError:
        latest_version = get_latest_version()
        latest_tuple = tuple(latest_version.replace("v", "").split("."))
        current_tuple = tuple(__version__.split("."))
        if latest_tuple > current_tuple:
            return True
    except RateLimitError:
        return False

    return False


@router.get("/api/options_model")
def get_options() -> Dict[str, Any]:
    """
    Get options model (possible settings).

    ### Returns
    - returns the options.
    """

    parser = create_parser()

    # Forbidden actions
    forbidden_actions = [
        "help",
        "operation",
        "version",
        "config",
        "user_auth",
        "client_id",
        "client_secret",
        "auth_token",
        "cache_path",
        "no_cache",
        "cookie_file",
        "ffmpeg",
        "archive",
        "host",
        "port",
        "keep_alive",
        "enable_tls",
        "key_file",
        "cert_file",
        "ca_file",
        "allowed_origins",
        "web_use_output_dir",
        "keep_sessions",
        "log_level",
        "simple_tui",
        "headless",
        "download_ffmpeg",
        "generate_config",
        "check_for_updates",
        "profile",
        "version",
    ]

    options = {}
    for action in parser._actions:  # pylint: disable=protected-access
        if action.dest in forbidden_actions:
            continue

        default = app_state.downloader_settings.get(action.dest, None)
        choices = list(action.choices) if action.choices else None

        type_name = ""
        if action.type is not None:
            if hasattr(action.type, "__objclass__"):
                type_name: str = action.type.__objclass__.__name__  # type: ignore
            else:
                type_name: str = action.type.__name__  # type: ignore

        if isinstance(
            action, argparse._StoreConstAction  # pylint: disable=protected-access
        ):
            type_name = "bool"

        if choices is not None and action.nargs == "*":
            type_name = "list"

        options[action.dest] = {
            "type": type_name,
            "choices": choices,
            "default": default,
            "help": action.help,
        }

    return options


def fix_mime_types():
    """Fix incorrect entries in the `mimetypes` registry.
    On Windows, the Python standard library's `mimetypes` reads in
    mappings from file extension to MIME type from the Windows
    registry. Other applications can and do write incorrect values
    to this registry, which causes `mimetypes.guess_type` to return
    incorrect values, which causes spotDL to fail to render on
    the frontend.
    This method hard-codes the correct mappings for certain MIME
    types that are known to be either used by TensorBoard or
    problematic in general.
    """

    # Known to be problematic when Visual Studio is installed:
    # <https://github.com/tensorflow/tensorboard/issues/3120>
    # https://github.com/spotDL/spotify-downloader/issues/1540
    mimetypes.add_type("application/javascript", ".js")

    # Not known to be problematic, but used by spotDL:
    mimetypes.add_type("text/css", ".css")
    mimetypes.add_type("image/svg+xml", ".svg")
    mimetypes.add_type("text/html", ".html")



File: spotdl/utils/config.py

"""
Module related to managing reading and writing to the config file.

Default config - spotdl.utils.config.DEFAULT_CONFIG
"""

import json
import logging
import os
import platform
from argparse import Namespace
from pathlib import Path
from typing import Any, Dict, Tuple, Union

import platformdirs

from spotdl.types.options import (
    DownloaderOptions,
    SpotDLOptions,
    SpotifyOptions,
    WebOptions,
)

__all__ = [
    "ConfigError",
    "get_spotdl_path",
    "get_config_file",
    "get_cache_path",
    "get_temp_path",
    "get_errors_path",
    "get_web_ui_path",
    "get_config",
    "create_settings_type",
    "create_settings",
    "SPOTIFY_OPTIONS",
    "DOWNLOADER_OPTIONS",
    "WEB_OPTIONS",
    "DEFAULT_CONFIG",
]

logger = logging.getLogger(__name__)


class ConfigError(Exception):
    """
    Base class for all exceptions related to config.
    """


def get_spotdl_path() -> Path:
    """
    Get the path to the spotdl folder.

    ### Returns
    - The path to the spotdl folder.

    ### Notes
    - If the spotdl directory does not exist, it will be created.
    """

    # Check if os is linux
    if platform.system() == "Linux":
        # if platform is linux, and XDG DATA HOME spotdl folder exists, use it
        user_data_dir = Path(platformdirs.user_data_dir("spotdl", "spotDL"))
        if user_data_dir.exists():
            return user_data_dir

    spotdl_path = Path(os.path.expanduser("~"), ".spotdl")
    if not spotdl_path.exists():
        os.mkdir(spotdl_path)

    return spotdl_path


def get_config_file() -> Path:
    """
    Get config file path

    ### Returns
    - The path to the config file.
    """

    return get_spotdl_path() / "config.json"


def get_cache_path() -> Path:
    """
    Get the path to the cache folder.

    ### Returns
    - The path to the spotipy cache file.
    """

    return get_spotdl_path() / ".spotipy"


def get_spotify_cache_path() -> Path:
    """
    Get the path to the spotify cache folder.

    ### Returns
    - The path to the spotipy cache file.
    """

    return get_spotdl_path() / ".spotify_cache"


def get_temp_path() -> Path:
    """
    Get the path to the temp folder.

    ### Returns
    - The path to the temp folder.
    """

    temp_path = get_spotdl_path() / "temp"
    if not temp_path.exists():
        os.mkdir(temp_path)

    return temp_path


def get_errors_path() -> Path:
    """
    Get the path to the errors folder.

    ### Returns
    - The path to the errors folder.

    ### Notes
    - If the errors directory does not exist, it will be created.
    """

    errors_path = get_spotdl_path() / "errors"

    if not errors_path.exists():
        os.mkdir(errors_path)

    return errors_path


def get_web_ui_path() -> Path:
    """
    Get the path to the web-ui folder.

    ### Returns
    - The path to the web-ui folder.

    ### Notes
    - If the web-ui directory does not exist, it will be created.
    """

    web_ui_path = get_spotdl_path() / "web-ui"

    if not web_ui_path.exists():
        os.mkdir(web_ui_path)

    return web_ui_path


def get_config() -> Dict[str, Any]:
    """
    Get the config.

    ### Returns
    - The dictionary with the config.

    ### Errors
    - ConfigError: If the config file does not exist.
    """

    config_path = get_config_file()

    if not config_path.exists():
        raise ConfigError(
            "Config file not found."
            "Please run `spotdl --generate-config` to create a config file."
        )

    with open(config_path, "r", encoding="utf-8") as config_file:
        return json.load(config_file)


def create_settings_type(
    arguments: Namespace,
    config: Dict[str, Any],
    default: Union[SpotifyOptions, DownloaderOptions, WebOptions],
) -> Dict[str, Any]:
    """
    Create settings dict
    Argument value has always the priority, then the config file
    value, and if neither are set, use default value

    ### Arguments
    - arguments: Namespace from argparse
    - default: dict

    ### Returns
    - settings: dict
    """

    settings = {}
    for key, default_value in default.items():
        argument_val = arguments.__dict__.get(key)
        config_val = config.get(key)

        if argument_val is not None:
            settings[key] = argument_val
        elif config_val is not None:
            settings[key] = config_val
        else:
            settings[key] = default_value

    return settings


def create_settings(
    arguments: Namespace,
) -> Tuple[SpotifyOptions, DownloaderOptions, WebOptions]:
    """
    Create settings dicts for Spotify, Downloader and Web
    based on the arguments and config file (if enabled)

    ### Arguments
    - arguments: Namespace from argparse

    ### Returns
    - spotify_options: SpotifyOptions
    - downloader_options: DownloaderOptions
    - web_options: WebOptions
    """

    # Get the config file
    # It will automatically load if the `load_config` is set to True
    # in the config file
    config = {}
    if arguments.config or (
        get_config_file().exists() and get_config().get("load_config")
    ):
        config = get_config()

    # Type: ignore because of the issues below
    # https://github.com/python/mypy/issues/8890
    # https://github.com/python/mypy/issues/5382
    spotify_options = SpotifyOptions(
        **create_settings_type(arguments, config, SPOTIFY_OPTIONS)  # type: ignore
    )
    downloader_options = DownloaderOptions(
        **create_settings_type(arguments, config, DOWNLOADER_OPTIONS)  # type: ignore
    )
    web_options = WebOptions(**create_settings_type(arguments, config, WEB_OPTIONS))  # type: ignore

    return spotify_options, downloader_options, web_options


def modernize_settings(options: DownloaderOptions):
    """Handle deprecated values in config file.

    ### Arguments
    - options: DownloaderOptions to modernize
    """

    warning_msg = "Deprecated '%s' value found for '%s' setting in config file. Using '%s' instead."

    # Respect backward compatibility with old boolean --restrict flag
    if options["restrict"] is True:
        logger.warning(warning_msg, True, "restrict", "strict")
        options["restrict"] = "strict"


class GlobalConfig:
    """
    Class to store global configuration
    """

    parameters: Dict[str, Any] = {}

    @classmethod
    def set_parameter(cls, key, value):
        """
        Set a parameter for the download config
        """

        cls.parameters[key] = value

    @classmethod
    def get_parameter(cls, key):
        """
        Get a parameter from the download config
        """

        return cls.parameters.get(key, None)


SPOTIFY_OPTIONS: SpotifyOptions = {
    "client_id": "5f573c9620494bae87890c0f08a60293",
    "client_secret": "212476d9b0f3472eaa762d90b19b0ba8",
    "auth_token": None,
    "user_auth": False,
    "headless": False,
    "cache_path": str(get_cache_path()),
    "no_cache": False,
    "max_retries": 3,
    "use_cache_file": False,
}

DOWNLOADER_OPTIONS: DownloaderOptions = {
    "audio_providers": ["youtube-music"],
    "lyrics_providers": ["genius", "azlyrics", "musixmatch"],
    "genius_token": "alXXDbPZtK1m2RrZ8I4k2Hn8Ahsd0Gh_o076HYvcdlBvmc0ULL1H8Z8xRlew5qaG",
    "playlist_numbering": False,
    "scan_for_songs": False,
    "m3u": None,
    "output": "{artists} - {title}.{output-ext}",
    "overwrite": "skip",
    "search_query": None,
    "ffmpeg": "ffmpeg",
    "bitrate": None,
    "ffmpeg_args": None,
    "format": "mp3",
    "save_file": None,
    "filter_results": True,
    "album_type": None,
    "threads": 4,
    "cookie_file": None,
    "restrict": None,
    "print_errors": False,
    "sponsor_block": False,
    "preload": False,
    "archive": None,
    "load_config": True,
    "log_level": "INFO",
    "simple_tui": False,
    "fetch_albums": False,
    "id3_separator": "/",
    "ytm_data": False,
    "add_unavailable": False,
    "generate_lrc": False,
    "force_update_metadata": False,
    "only_verified_results": False,
    "sync_without_deleting": False,
    "max_filename_length": None,
    "yt_dlp_args": None,
    "detect_formats": None,
    "save_errors": None,
    "ignore_albums": None,
    "proxy": None,
    "skip_explicit": False,
    "log_format": None,
    "redownload": False,
    "skip_album_art": False,
    "create_skip_file": False,
    "respect_skip_file": False,
    "sync_remove_lrc": False,
}

WEB_OPTIONS: WebOptions = {
    "web_use_output_dir": False,
    "port": 8800,
    "host": "localhost",
    "keep_alive": False,
    "enable_tls": False,
    "key_file": None,
    "cert_file": None,
    "ca_file": None,
    "allowed_origins": None,
    "keep_sessions": False,
    "force_update_gui": False,
    "web_gui_repo": None,
    "web_gui_location": None,
}

# Type: ignore because of the issues above
DEFAULT_CONFIG: SpotDLOptions = {
    **SPOTIFY_OPTIONS,  # type: ignore
    **DOWNLOADER_OPTIONS,  # type: ignore
    **WEB_OPTIONS,  # type: ignore
}



File: spotdl/utils/metadata.py

"""
Module for embedding metadata into audio files using Mutagen.

```python
embed_metadata(
    output_file=Path("test.mp3"),
    song=song_object,
    file_format="mp3",
)
```
"""

import base64
import logging
import re
from pathlib import Path
from typing import Any, Dict, Optional

import requests
from mutagen._file import File
from mutagen.flac import Picture
from mutagen.id3 import ID3
from mutagen.id3._frames import (
    APIC,
    COMM,
    POPM,
    SYLT,
    TALB,
    TCOM,
    TCON,
    TCOP,
    TDRC,
    TIT2,
    TPE1,
    TRCK,
    TSRC,
    TYER,
    USLT,
    WOAS,
)
from mutagen.id3._specs import Encoding
from mutagen.mp4 import MP4Cover
from mutagen.wave import WAVE

from spotdl.types.song import Song
from spotdl.utils.config import GlobalConfig
from spotdl.utils.formatter import to_ms
from spotdl.utils.lrc import remomve_lrc

logger = logging.getLogger(__name__)

__all__ = [
    "MetadataError",
    "M4A_TAG_PRESET",
    "MP3_TAG_PRESET",
    "TAG_PRESET",
    "TAG_TO_SONG",
    "M4A_TO_SONG",
    "MP3_TO_SONG",
    "LRC_REGEX",
    "embed_metadata",
    "embed_cover",
    "embed_lyrics",
    "get_file_metadata",
]


class MetadataError(Exception):
    """
    Base class for all exceptions related to metadata and id3 embedding.
    """


# Apple has specific tags - see mutagen docs -
# http://mutagen.readthedocs.io/en/latest/api/mp4.html
M4A_TAG_PRESET = {
    "album": "\xa9alb",
    "artist": "\xa9ART",
    "date": "\xa9day",
    "title": "\xa9nam",
    "year": "\xa9day",
    "comment": "\xa9cmt",
    "group": "\xa9grp",
    "writer": "\xa9wrt",
    "genre": "\xa9gen",
    "tracknumber": "trkn",
    "trackcount": "trkn",
    "albumartist": "aART",
    "discnumber": "disk",
    "disccount": "disk",
    "cpil": "cpil",
    "albumart": "covr",
    "encodedby": "\xa9too",
    "copyright": "cprt",
    "tempo": "tmpo",
    "lyrics": "\xa9lyr",
    "explicit": "rtng",
    "woas": "----:spotdl:WOAS",
    "isrc": "----:spotdl:ISRC",
}

MP3_TAG_PRESET = {
    "album": "TALB",
    "artist": "TPE1",
    "date": "TDRC",
    "title": "TIT2",
    "year": "TDRC",
    "comment": "COMM::XXX",
    "group": "TIT1",
    "writer": "TEXT",
    "genre": "TCON",
    "tracknumber": "TRCK",
    "trackcount": "TRCK",
    "albumartist": "TPE2",
    "discnumber": "TPOS",
    "disccount": "TPOS",
    "cpil": "TCMP",
    "albumart": "APIC",
    "encodedby": "TENC",
    "copyright": "TCOP",
    "tempo": "TBPM",
    "lyrics": "USLT::XXX",
    "woas": "WOAS",
    "isrc": "TSRC",
    "explicit": "NULL",
}

TAG_PRESET = {key: key for key in M4A_TAG_PRESET}

TAG_TO_SONG = {
    "title": "name",
    "artist": "artists",
    "album": "album_name",
    "albumartist": "album_artist",
    "genre": "genres",
    "discnumber": "disc_number",
    "year": "year",
    "date": "date",
    "tracknumber": "track_number",
    "encodedby": "publisher",
    "woas": "url",
    "comment": "download_url",
    "isrc": "isrc",
    "copyright": "copyright_text",
    "lyrics": "lyrics",
    "albumart": "album_art",
}

M4A_TO_SONG = {
    value: TAG_TO_SONG.get(key)
    for key, value in M4A_TAG_PRESET.items()
    if TAG_TO_SONG.get(key)
}
MP3_TO_SONG = {
    value: TAG_TO_SONG.get(key)
    for key, value in MP3_TAG_PRESET.items()
    if TAG_TO_SONG.get(key)
}

LRC_REGEX = re.compile(r"(\[\d{2}:\d{2}.\d{2,3}\])")


def embed_metadata(
    output_file: Path,
    song: Song,
    id3_separator: str = "/",
    skip_album_art: Optional[bool] = False,
):
    """
    Set ID3 tags for generic files (FLAC, OPUS, OGG)

    ### Arguments
    - output_file: Path to the output file.
    - song: Song object.
    - id3_separator: The separator used for the id3 tags.
    - skip_album_art: Boolean to skip album art embedding.
    """

    # Get the file extension for the output file
    encoding = output_file.suffix[1:]

    if encoding == "wav":
        embed_wav_file(output_file, song)
        return

    # Get the tag preset for the file extension
    tag_preset = TAG_PRESET if encoding != "m4a" else M4A_TAG_PRESET

    try:
        audio_file = File(str(output_file.resolve()), easy=encoding == "mp3")

        if audio_file is None:
            raise MetadataError(
                f"Unrecognized file format for {output_file} from {song.url}"
            )
    except Exception as exc:
        raise MetadataError("Unable to load file.") from exc

    # Embed basic metadata
    audio_file[tag_preset["artist"]] = song.artists
    audio_file[tag_preset["albumartist"]] = (
        song.album_artist if song.album_artist else song.artist
    )
    audio_file[tag_preset["title"]] = song.name
    audio_file[tag_preset["date"]] = song.date
    audio_file[tag_preset["encodedby"]] = song.publisher

    # Embed metadata that isn't always present
    album_name = song.album_name
    if album_name:
        audio_file[tag_preset["album"]] = album_name

    if song.genres:
        audio_file[tag_preset["genre"]] = song.genres[0].title()

    if song.copyright_text:
        audio_file[tag_preset["copyright"]] = song.copyright_text

    if song.download_url and encoding != "mp3":
        audio_file[tag_preset["comment"]] = song.download_url

    # Embed some metadata in format specific ways
    if encoding in ["flac", "ogg", "opus"]:
        audio_file["discnumber"] = str(song.disc_number)
        audio_file["disctotal"] = str(song.disc_count)
        audio_file["tracktotal"] = str(song.tracks_count)
        audio_file["tracknumber"] = str(song.track_number)
        audio_file["woas"] = song.url
        audio_file["isrc"] = song.isrc
    elif encoding == "m4a":
        audio_file[tag_preset["discnumber"]] = [(song.disc_number, song.disc_count)]
        audio_file[tag_preset["tracknumber"]] = [(song.track_number, song.tracks_count)]
        audio_file[tag_preset["explicit"]] = (4 if song.explicit is True else 2,)
        audio_file[tag_preset["woas"]] = song.url.encode("utf-8")
    elif encoding == "mp3":
        audio_file["tracknumber"] = f"{str(song.track_number)}/{str(song.tracks_count)}"
        audio_file["discnumber"] = f"{str(song.disc_number)}/{str(song.disc_count)}"
        audio_file["isrc"] = song.isrc

    # Mp3 specific encoding
    if encoding == "mp3":
        if id3_separator != "/":
            audio_file.save(v23_sep=id3_separator, v2_version=3)
        else:
            audio_file.save(v2_version=3)

        audio_file = ID3(str(output_file.resolve()))

        audio_file.add(WOAS(encoding=3, url=song.url))

        if song.download_url:
            audio_file.add(COMM(encoding=3, text=song.download_url))

        if song.popularity:
            audio_file.add(
                POPM(
                    rating=int(song.popularity * 255 / 100),
                )
            )

        if song.year:
            audio_file.add(TYER(encoding=3, text=str(song.year)))

    if not skip_album_art:
        # Embed album art
        audio_file = embed_cover(audio_file, song, encoding)

    # Embed lyrics
    audio_file = embed_lyrics(audio_file, song, encoding)

    # Mp3 specific encoding
    if encoding == "mp3":
        audio_file.save(v23_sep=id3_separator, v2_version=3)
    else:
        audio_file.save()


def embed_cover(audio_file, song: Song, encoding: str):
    """
    Embed the album art in the audio file.

    ### Arguments
    - audio_file: Audio file object.
    - song: Song object.
    """

    if not song.cover_url:
        return audio_file

    # Try to download the cover art
    try:
        cover_data = requests.get(
            song.cover_url,
            timeout=10,
            proxies=GlobalConfig.get_parameter("proxies"),
        ).content
    except Exception:
        return audio_file

    # Create the image object for the file type
    if encoding in ["flac", "ogg", "opus"]:
        picture = Picture()
        picture.type = 3
        picture.desc = "Cover"
        picture.mime = "image/jpeg"
        picture.data = cover_data

        if encoding in ["ogg", "opus"]:
            image_data = picture.write()
            encoded_data = base64.b64encode(image_data)
            vcomment_value = encoded_data.decode("ascii")
            if "metadata_block_picture" in audio_file.keys():
                audio_file.pop("metadata_block_picture")
            audio_file["metadata_block_picture"] = [vcomment_value]
        elif encoding == "flac":
            if audio_file.pictures:
                audio_file.clear_pictures()
            audio_file.add_picture(picture)
    elif encoding == "m4a":
        if M4A_TAG_PRESET["albumart"] in audio_file.keys():
            audio_file.pop(M4A_TAG_PRESET["albumart"])
        audio_file[M4A_TAG_PRESET["albumart"]] = [
            MP4Cover(
                cover_data,
                imageformat=MP4Cover.FORMAT_JPEG,
            )
        ]
    elif encoding == "mp3":
        if "APIC:Cover" in audio_file.keys():
            audio_file.pop("APIC:Cover")
        audio_file["APIC"] = APIC(
            encoding=3,
            mime="image/jpeg",
            type=3,
            desc="Cover",
            data=cover_data,
        )

    return audio_file


def embed_lyrics(audio_file, song: Song, encoding: str):
    """
    Detect lyrics type (lrc or txt) and embed them in the audio file.

    ### Arguments
    - audio_file: Audio file object.
    - song: Song object.
    - encoding: Encoding type.
    """

    lyrics = song.lyrics
    if not lyrics:
        return audio_file

    tag_preset = TAG_PRESET if encoding != "m4a" else M4A_TAG_PRESET

    # Check if the lyrics are in lrc format
    # using regex on the first 5 lines
    lrc_lines = lyrics.splitlines()[:5]
    lrc_lines = [line for line in lrc_lines if line and LRC_REGEX.match(line)]

    if len(lrc_lines) == 0:
        # Lyrics are not in lrc format
        # Embed them normally
        if encoding == "mp3":
            audio_file.add(USLT(encoding=Encoding.UTF8, text=song.lyrics))
        else:
            audio_file[tag_preset["lyrics"]] = song.lyrics
    else:
        # Lyrics are in lrc format
        # Embed them as SYLT id3 tag
        clean_lyrics = remomve_lrc(lyrics)
        if encoding == "mp3":
            lrc_data = []
            for line in lyrics.splitlines():
                time_tag = line.split("]", 1)[0] + "]"
                text = line.replace(time_tag, "")

                time_tag = time_tag.replace("[", "")
                time_tag = time_tag.replace("]", "")
                time_tag = time_tag.replace(".", ":")
                time_tag_vals = time_tag.split(":")
                if len(time_tag_vals) != 3 or any(
                    not isinstance(tag, int) for tag in time_tag_vals
                ):
                    continue

                minute, sec, millisecond = time_tag_vals
                time = to_ms(min=minute, sec=sec, ms=millisecond)
                lrc_data.append((text, time))

            audio_file.add(USLT(encoding=3, text=clean_lyrics))
            audio_file.add(SYLT(encoding=3, text=lrc_data, format=2, type=1))
        else:
            audio_file[tag_preset["lyrics"]] = song.lyrics

    return audio_file


def get_file_metadata(path: Path, id3_separator: str = "/") -> Optional[Dict[str, Any]]:
    """
    Get song metadata.

    ### Arguments
    - path: Path to the song.

    ### Returns
    - Dict of song metadata.

    ### Raises
    - OSError: If the file is not found.
    - MetadataError: If the file is not a valid audio file.
    """

    if path.exists() is False:
        raise OSError(f"File not found: {path}")

    audio_file = File(str(path.resolve()))

    if audio_file is None or audio_file == {}:
        return None

    song_meta: Dict[str, Any] = {}
    for key in TAG_PRESET:
        if path.suffix == ".m4a":
            val = audio_file.get(M4A_TAG_PRESET[key])
        elif path.suffix == ".mp3":
            val = audio_file.get(MP3_TAG_PRESET[key])
        else:
            val = audio_file.get(key)

        # Cover art is a special case and
        # has to be handled before checking the val
        # M4A is handled in the m4a section since it
        # has data in the val variable
        if key == "albumart":
            if path.suffix == ".mp3":
                cover = audio_file.get("APIC:Cover")
                if cover:
                    song_meta["album_art"] = cover.data
                else:
                    song_meta["album_art"] = None

                continue

            if path.suffix == ".flac":
                song_meta["album_art"] = audio_file.pictures[0].data
                continue

            if path.suffix in [".ogg", ".opus"]:
                pictures = audio_file.get("metadata_block_picture")
                if pictures and pictures[0]:
                    song_meta["album_art"] = pictures[0]
                else:
                    song_meta["album_art"] = None

                continue

        # If the tag is empty, skip it
        if val is None:
            # If the tag is empty but it's key is in the
            # song object, set it to None
            empty_key = TAG_TO_SONG.get(key)
            if empty_key:
                song_meta[empty_key] = None

            continue

        # MP3 specific decoding
        if path.suffix == ".mp3":
            if key == "woas":
                song_meta["url"] = val.url
            elif key == "comment":
                song_meta["download_url"] = val.text[0]
            elif key == "year":
                song_meta["year"] = int(str(val.text[0])[:4])
            elif key == "date":
                song_meta["date"] = str(val.text[0])
            elif key in ["tracknumber", "trackcount"]:
                count = val.text[0].split(id3_separator)
                if len(count) == 2:
                    song_meta["track_number"] = int(count[0])
                    song_meta["tracks_count"] = int(count[1])
                else:
                    song_meta["track_number"] = val.text[0]
            elif key in ["discnumber", "disccount"]:
                count = val.text[0].split(id3_separator)
                if len(count) == 2:
                    song_meta["disc_number"] = int(count[0])
                    song_meta["disc_count"] = int(count[1])
                else:
                    song_meta["disc_number"] = val.text[0]
            elif key == "artist":
                artists_val: str = (
                    val.text[0] if isinstance(val.text, list) else val.text
                )
                song_meta["artists"] = artists_val.split(id3_separator)
            else:
                meta_key = TAG_TO_SONG.get(key)
                if meta_key and song_meta.get(meta_key) is None:
                    song_meta[meta_key] = (
                        val.text[0]
                        if isinstance(val.text, list) and len(val.text) == 1
                        else val.text
                    )

        # M4A specific decoding
        elif path.suffix == ".m4a":
            if key == "artist":
                song_meta["artists"] = val
            elif key == "woas":
                song_meta["url"] = val[0].decode("utf-8")
            elif key == "explicit":
                song_meta["explicit"] = val == [4] if val else None
            elif key == "year":
                song_meta["year"] = int(str(val[0])[:4])
            elif key == "discnumber":
                song_meta["disc_number"] = val[0][0]
                song_meta["disc_count"] = val[0][1]
            elif key == "tracknumber":
                song_meta["track_number"] = val[0][0]
                song_meta["tracks_count"] = val[0][1]
            else:
                meta_key = TAG_TO_SONG.get(key)
                if meta_key:
                    song_meta[meta_key] = (
                        val[0] if isinstance(val, list) and len(val) == 1 else val
                    )

        # FLAC, OGG, OPUS specific decoding
        else:
            if key == "artist":
                song_meta["artists"] = val
            elif key == "tracknumber":
                song_meta["track_number"] = int(val[0])
            elif key == "discnumber":
                song_meta["disc_count"] = int(val[0])
                song_meta["disc_number"] = int(val[0])
            else:
                meta_key = TAG_TO_SONG.get(key)
                if meta_key:
                    song_meta[meta_key] = (
                        val[0] if isinstance(val, list) and len(val) == 1 else val
                    )

    # Make sure that artists is a list
    if isinstance(song_meta["artists"], str):
        song_meta["artists"] = [song_meta["artists"]]
    elif song_meta["artists"] is not None:
        song_meta["artists"] = list(song_meta["artists"])
    else:
        song_meta["artists"] = []

    # Make sure that genres is a list
    if isinstance(song_meta["genres"], str):
        song_meta["genres"] = [song_meta["genres"]]

    # Add main artist to the song meta object
    if song_meta["artists"]:
        song_meta["artist"] = song_meta["artists"][0]
    else:
        song_meta["artist"] = None

    return song_meta


def embed_wav_file(output_file: Path, song: Song):
    """
    Embeds the song metadata into the wav file

    ### Arguments
    - output_file: The output file path
    - song: The song object
    - id3_separator: The separator used for the id3 tags
    """
    audio = WAVE(output_file)
    if audio is None:
        raise ValueError("Invalid audio file")

    if audio.tags:
        audio.tags.clear()

    audio.add_tags()

    audio.tags.add(TIT2(encoding=3, text=song.name))  # type: ignore
    audio.tags.add(TPE1(encoding=3, text=song.artists))  # type: ignore
    audio.tags.add(TALB(encoding=3, text=song.album_name))  # type: ignore
    audio.tags.add(TCOM(encoding=3, text=song.publisher))  # type: ignore
    audio.tags.add(TCON(encoding=3, text=song.genres))  # type: ignore
    audio.tags.add(TDRC(encoding=3, text=song.date))  # type: ignore
    audio.tags.add(  # type: ignore
        TRCK(encoding=3, text=f"{song.track_number}/{song.tracks_count}")  # type: ignore
    )
    audio.tags.add(TDRC(encoding=3, text=song.date))  # type: ignore
    audio.tags.add(WOAS(encoding=3, text=song.url))  # type: ignore
    audio.tags.add(TSRC(encoding=3, text=song.isrc))  # type: ignore

    if song.download_url:
        audio.tags.add(COMM(encoding=3, text=song.download_url))  # type: ignore

    if song.copyright_text:
        audio.tags.add(TCOP(encoding=3, text=song.copyright_text))  # type: ignore

    if song.popularity:
        audio.tags.add(  # type: ignore
            COMM(
                encoding=3,
                lang="eng",
                text="Spotify Popularity: " + str(song.popularity),
            )
        )

    if song.cover_url:
        try:
            cover_data = requests.get(song.cover_url, timeout=10).content
            audio.tags.add(  # type: ignore
                APIC(
                    encoding=3, mime="image/jpeg", type=3, desc="Cover", data=cover_data
                )
            )
        except Exception:
            pass

    if song.lyrics:
        # Check if the lyrics are in lrc format
        # using regex on the first 5 lines
        lrc_lines = song.lyrics.splitlines()[:5]
        lrc_lines = [line for line in lrc_lines if line and LRC_REGEX.match(line)]

        if len(lrc_lines) == 0:
            audio.tags.add(USLT(encoding=Encoding.UTF8, text=song.lyrics))  # type: ignore
        else:
            lrc_data = []
            clean_lyrics = remomve_lrc(song.lyrics)
            for line in song.lyrics.splitlines():
                time_tag = line.split("]", 1)[0] + "]"
                text = line.replace(time_tag, "")

                time_tag = time_tag.replace("[", "")
                time_tag = time_tag.replace("]", "")
                time_tag = time_tag.replace(".", ":")
                time_tag_vals = time_tag.split(":")
                if len(time_tag_vals) != 3 or any(
                    not isinstance(tag, int) for tag in time_tag_vals
                ):
                    continue

                minute, sec, millisecond = time_tag_vals
                time = to_ms(min=minute, sec=sec, ms=millisecond)
                lrc_data.append((text, time))

            audio.tags.add(USLT(encoding=3, text=clean_lyrics))  # type: ignore
            audio.tags.add(SYLT(encoding=3, text=lrc_data, format=2, type=1))  # type: ignore

    audio.save()



File: spotdl/utils/downloader.py

"""
Module for functions related to downloading songs.
"""

from spotdl.providers.audio import YouTubeMusic

__all__ = ["check_ytmusic_connection"]


def check_ytmusic_connection() -> bool:
    """
    Check if we can connect to YouTube Music API

    ### Returns
    - `True` if we can connect to YouTube Music API
    - `False` if we can't connect to YouTube Music API
    """

    # Check if we are getting results from YouTube Music
    ytm = YouTubeMusic()
    test_results = ytm.get_results("a")
    if len(test_results) == 0:
        return False

    return True



File: spotdl/utils/arguments.py

"""
Module that handles the command line arguments.
"""

import argparse
import sys
import textwrap
from argparse import ArgumentParser, Namespace, _ArgumentGroup
from typing import List

from spotdl import _version
from spotdl.download.downloader import AUDIO_PROVIDERS, LYRICS_PROVIDERS
from spotdl.utils.ffmpeg import FFMPEG_FORMATS
from spotdl.utils.formatter import VARS
from spotdl.utils.logging import NAME_TO_LEVEL

__all__ = ["OPERATIONS", "SmartFormatter", "parse_arguments"]

OPERATIONS = ["download", "save", "web", "sync", "meta", "url"]


class SmartFormatter(argparse.HelpFormatter):
    """
    Class that overrides the default help formatter.
    """

    def _split_lines(self, text: str, width: int) -> List[str]:
        """
        Split the text in multiple lines if a line starts
        with a N|
        """

        if text.startswith("N|"):
            return text[2:].splitlines()

        text = self._whitespace_matcher.sub(" ", text).strip()

        return textwrap.wrap(text, width)


def parse_main_options(parser: _ArgumentGroup):
    """
    Parse main options from the command line.

    ### Arguments
    - parser: The argument parser to add the options to.
    """

    # Add operation argument
    operation = parser.add_argument(
        "operation",
        choices=OPERATIONS,
        default="download",
        const="download",
        nargs="?",
        help=(
            "N|The operation to perform.\n"
            "download: Download the songs to the disk and embed metadata.\n"
            "save: Saves the songs metadata to a file for further use.\n"
            "web: Starts a web interface to simplify the download process.\n"
            "sync: Removes songs that are no longer present, downloads new ones\n"
            "meta: Update your audio files with metadata\n"
            "url: Get the download URL for songs\n\n"
        ),
    )

    # Add query argument
    query = parser.add_argument(
        "query",
        nargs="+",
        type=str,
        help=(
            "N|Spotify/YouTube URL for a song/playlist/album/artist/etc. to download.\n\n"
            "For album/playlist/artist searching, include 'album:', 'playlist:', 'artist:' \n"
            "(ie. 'album:the album name' you can mix these options to get more accurate results)"
            ".\n\n"
            "To download liked songs use 'saved' as the query, to download all user playlists\n"
            "use 'all-user-playlists, to download playlists that the user has created\n"
            "use 'all-saved-playlists', to download all user liked playlists\n"
            "use 'all-user-followed-artists', to download all user saved albums "
            "use 'all-user-saved-albums' \n\n"
            "For manual audio matching, you can use the format 'YouTubeURL|SpotifyURL'\n"
            "You can only use album/playlist/tracks urls when "
            "downloading/matching youtube urls.\n"
            "When using youtube url without spotify url, "
            "you won't be able to use `--fetch-albums` option.\n\n"
        ),
    )

    try:
        is_web = sys.argv[1] == "web"
    except IndexError:
        is_web = False

    is_frozen = getattr(sys, "frozen", False)

    # If the program is frozen, we and user didn't pass any arguments,
    # or if the user is using the web interface, we don't need to parse
    # the query
    if (is_frozen and len(sys.argv) < 2) or (len(sys.argv) > 1 and is_web):
        # If we are running the web interface
        # or we are in the frozen env and not running web interface
        # don't remove the operation from the arg parser
        if not is_web or (is_frozen and not is_web):
            parser._remove_action(operation)  # pylint: disable=protected-access

        parser._remove_action(query)  # pylint: disable=protected-access

    # Audio provider argument
    parser.add_argument(
        "--audio",
        dest="audio_providers",
        nargs="*",
        choices=AUDIO_PROVIDERS,
        help="The audio provider to use. You can provide more than one for fallback.",
    )

    # Lyrics provider argument
    parser.add_argument(
        "--lyrics",
        dest="lyrics_providers",
        nargs="*",
        choices=LYRICS_PROVIDERS.keys(),
        help=(
            "The lyrics provider to use. You can provide more than one for fallback. "
            "Synced lyrics might not work correctly with some music players. "
            "For such cases it's better to use `--generate-lrc` option."
        ),
    )

    parser.add_argument(
        "--genius-access-token",
        dest="genius_token",
        help="Lets you choose your own Genius access token.",
    )

    # Add config argument
    parser.add_argument(
        "--config",
        action="store_true",
        help=(
            "Use the config file to download songs. "
            "It's located under C:\\Users\\user\\.spotdl\\config.json "
            "or ~/.spotdl/config.json under linux"
        ),
    )

    # Add search query argument
    parser.add_argument(
        "--search-query",
        help=f"The search query to use, available variables: {', '.join(VARS)}",
        type=str,
    )

    # Add don't filter results argument
    parser.add_argument(
        "--dont-filter-results",
        dest="filter_results",
        action="store_const",
        const=False,
        help="Disable filtering results.",
    )

    # Add use only verified results argument
    parser.add_argument(
        "--album-type",
        choices={"album", "single"},
        help="Type of the album to search for. (album, single)",
        type=str,
    )

    # Add use only verified results argument
    parser.add_argument(
        "--only-verified-results",
        action="store_const",
        const=True,
        help="Use only verified results. (Not all providers support this)",
    )


def parse_spotify_options(parser: _ArgumentGroup):
    """
    Parse spotify options from the command line.

    ### Arguments
    - parser: The argument parser to add the options to.
    """

    # Add login argument
    parser.add_argument(
        "--user-auth",
        action="store_const",
        const=True,
        help="Login to Spotify using OAuth.",
    )

    # Add client id argument
    parser.add_argument(
        "--client-id",
        help="The client id to use when logging in to Spotify.",
        type=str,
    )

    # Add client secret argument
    parser.add_argument(
        "--client-secret",
        help="The client secret to use when logging in to Spotify.",
        type=str,
    )

    # Add auth token argument
    parser.add_argument(
        "--auth-token",
        help="The authorization token to use directly to log in to Spotify.",
        type=str,
    )

    # Add cache path argument
    parser.add_argument(
        "--cache-path",
        type=str,
        help="The path where spotipy cache file will be stored.",
    )

    # Add no cache argument
    parser.add_argument(
        "--no-cache",
        action="store_const",
        const=True,
        help="Disable caching (both requests and token).",
    )

    # Add max retries argument
    parser.add_argument(
        "--max-retries",
        type=int,
        help="The maximum number of retries to perform when getting metadata.",
    )

    # Add headless argument
    parser.add_argument(
        "--headless",
        action="store_const",
        const=True,
        help="Run in headless mode.",
    )

    # Add use cache file argument
    parser.add_argument(
        "--use-cache-file",
        action="store_const",
        const=True,
        help=(
            "Use the cache file to get metadata. "
            "It's located under C:\\Users\\user\\.spotdl\\.spotify_cache "
            "or ~/.spotdl/.spotify_cache under linux. "
            "It only caches tracks and "
            "gets updated whenever spotDL gets metadata from Spotify. "
            "(It may provide outdated metadata use with caution)"
        ),
    )


def parse_ffmpeg_options(parser: _ArgumentGroup):
    """
    Parse ffmpeg options from the command line.

    ### Arguments
    - parser: The argument parser to add the options to.
    """

    # Add ffmpeg executable argument
    parser.add_argument(
        "--ffmpeg",
        help="The ffmpeg executable to use.",
        type=str,
    )

    # Add search threads argument
    parser.add_argument(
        "--threads",
        type=int,
        help="The number of threads to use when downloading songs.",
    )

    # Add constant bit rate argument
    parser.add_argument(
        "--bitrate",
        choices=[
            "auto",
            "disable",
            "8k",
            "16k",
            "24k",
            "32k",
            "40k",
            "48k",
            "64k",
            "80k",
            "96k",
            "112k",
            "128k",
            "160k",
            "192k",
            "224k",
            "256k",
            "320k",
        ]
        + list(map(str, range(0, 10))),
        type=str.lower,
        help=(
            "The constant/variable bitrate to use for the output file. "
            "Values from 0 to 9 are variable bitrates. "
            "Auto will use the bitrate of the original file. "
            "Disable will disable the bitrate option. "
            "(In case of m4a and opus files, auto and disable will skip the conversion)"
        ),
    )

    # Additional ffmpeg arguments
    parser.add_argument(
        "--ffmpeg-args",
        type=str,
        help="Additional ffmpeg arguments passed as a string.",
    )


def parse_output_options(parser: _ArgumentGroup):
    """
    Parse output options from the command line.

    ### Arguments
    - parser: The argument parser to add the options to.
    """

    # Add output format argument
    parser.add_argument(
        "--format",
        choices=FFMPEG_FORMATS.keys(),
        help="The format to download the song in.",
        type=str,
    )

    # Add save file argument
    parser.add_argument(
        "--save-file",
        type=str,
        help=(
            "The file to save/load the songs data from/to. "
            "It has to end with .spotdl. "
            "If combined with the download operation, it will save the songs data to the file. "
            "Required for save/sync (use - to print to stdout when using save). "
        ),
        required=len(sys.argv) > 1 and sys.argv[1] in ["save"],
    )

    # Add preload argument
    parser.add_argument(
        "--preload",
        action="store_const",
        const=True,
        help="Preload the download url to speed up the download process.",
    )

    # Add name format argument
    parser.add_argument(
        "--output",
        type=str,
        help=f"Specify the downloaded file name format, available variables: {', '.join(VARS)}",
    )

    # Add m3u argument
    parser.add_argument(
        "--m3u",
        type=str,
        nargs="?",
        help=(
            "Name of the m3u file to save the songs to. "
            "Defaults to {list[0]}.m3u8 "
            "If you want to generate a m3u for each list in the query use {list}, "
            "If you want to generate a m3u file based on the first list in the query use {list[0]}"
            ", (0 is the first list in the query, 1 is the second, etc. "
            "songs don't count towards the list number) "
        ),
        const="{list[0]}.m3u8",
    )

    # Add cookie file argument
    parser.add_argument(
        "--cookie-file",
        help="Path to cookies file.",
        type=str,
    )

    # Add overwrite argument
    parser.add_argument(
        "--overwrite",
        choices={"force", "skip", "metadata"},
        help=(
            "How to handle existing/duplicate files. "
            "(When combined with --scan-for-songs force will remove "
            "all duplicates, and metadata will only apply metadata to the "
            "latest song and will remove the rest. )"
        ),
        type=str,
    )

    # Option to increase compatibility of filenames and easier handling in the shell
    parser.add_argument(
        "--restrict",
        choices={"strict", "ascii", "none"},
        const="strict",
        nargs="?",
        help="Restrict filenames to a sanitized set of characters for better compatibility",
        type=str,
    )

    # Option to print errors on exit, useful for long playlist
    parser.add_argument(
        "--print-errors",
        action="store_const",
        const=True,
        help="Print errors (wrong songs, failed downloads etc) on exit, useful for long playlist",
    )

    # Option to save errors to a file
    parser.add_argument(
        "--save-errors",
        type=str,
        help="Save errors (wrong songs, failed downloads etc) to a file",
    )

    # Option to use sponsor block
    parser.add_argument(
        "--sponsor-block",
        action="store_const",
        const=True,
        help="Use the sponsor block to download songs from yt/ytm.",
    )

    # Add archive_file argument
    parser.add_argument(
        "--archive",
        type=str,
        help="Specify the file name for an archive of already downloaded songs",
    )

    # Option to set the track number & album of tracks in a playlist to their index in the playlist
    # & the name of playlist respectively.
    parser.add_argument(
        "--playlist-numbering",
        action="store_const",
        dest="playlist_numbering",
        const=True,
        help="Sets each track in a playlist to have the playlist's name as its album,\
            and album art as the playlist's icon",
    )

    # Option to scan the output directory for existing files
    parser.add_argument(
        "--scan-for-songs",
        action="store_const",
        const=True,
        help=(
            "Scan the output directory for existing files. "
            "This option should be combined with the --overwrite option "
            "to control how existing files are handled. (Output directory is the last directory "
            "that is not a template variable in the output template)"
        ),
    )

    # Option to fetch all albums from songs in query
    parser.add_argument(
        "--fetch-albums",
        action="store_const",
        const=True,
        help="Fetch all albums from songs in query",
    )

    # Option to change the id3 separator
    parser.add_argument(
        "--id3-separator",
        type=str,
        help="Change the separator used in the id3 tags. Only supported for mp3 files.",
    )

    # Option to use ytm data instead of spotify data
    # when downloading using ytm link
    parser.add_argument(
        "--ytm-data",
        action="store_const",
        const=True,
        help="Use ytm data instead of spotify data when downloading using ytm link.",
    )

    # Option whether to add unavailable songs to the m3u file
    parser.add_argument(
        "--add-unavailable",
        action="store_const",
        const=True,
        help="Add unavailable songs to the m3u/archive files when downloading",
    )

    # Generate lrc files
    parser.add_argument(
        "--generate-lrc",
        action="store_const",
        const=True,
        help=(
            "Generate lrc files for downloaded songs. "
            "Requires `synced` provider to be present in the lyrics providers list."
        ),
    )

    # Force update metadata
    parser.add_argument(
        "--force-update-metadata",
        action="store_const",
        const=True,
        help="Force update metadata for songs that already have metadata.",
    )

    # Sync without deleting
    parser.add_argument(
        "--sync-without-deleting",
        action="store_const",
        const=True,
        help="Sync without deleting songs that are not in the query.",
    )

    # Max file name length
    parser.add_argument(
        "--max-filename-length",
        type=int,
        help=(
            "Max file name length. "
            "(This won't override the max file name length enforced by the OS)"
        ),
    )

    # YT-DlP options
    parser.add_argument(
        "--yt-dlp-args",
        type=str,
        help="Arguments to pass to yt-dlp",
    )

    # Detect formats option
    parser.add_argument(
        "--detect-formats",
        type=str,
        nargs="*",
        help=(
            "Detect already downloaded songs with file format different from the --format option "
            "(When combined with --m3u option, "
            "only first detected format will be added to m3u file)"
        ),
        choices=FFMPEG_FORMATS.keys(),
    )

    # download song in meta operation
    parser.add_argument(
        "--redownload",
        action="store_const",
        const=True,
        help="to redownload the local song in diffrent format using --format for meta operation",
    )

    # skip album art for meta operation
    parser.add_argument(
        "--skip-album-art",
        action="store_const",
        const=True,
        help="skip downloading album art for meta operation",
    )

    # Ignore songs from a paticular album
    parser.add_argument(
        "--ignore-albums",
        type=str,
        nargs="*",
        help="ignores the song of the given albums",
    )

    # Skip explicit songs options
    parser.add_argument(
        "--skip-explicit", action="store_const", const=True, help="Skip explicit songs"
    )

    parser.add_argument(
        "--proxy",
        help="Http(s) proxy server for download song. Example: http://host:port",
    )

    # Skip songs having a skip flag file
    parser.add_argument(
        "--create-skip-file",
        action="store_const",
        const=True,
        help="Create skip file for successfully downloaded file",
    )

    # Skip songs having a skip flag file
    parser.add_argument(
        "--respect-skip-file",
        action="store_const",
        const=True,
        help="If a file with the extension .skip exists, skip download",
    )

    # Sync remove lrc files
    parser.add_argument(
        "--sync-remove-lrc",
        action="store_const",
        const=True,
        help="Remove lrc files when using sync operation when downloading songs",
    )


def parse_web_options(parser: _ArgumentGroup):
    """
    Parse web options from the command line.

    ### Arguments
    - parser: The argument parser to add the options to.
    """

    # Add host argument
    parser.add_argument(
        "--host",
        type=str,
        help="The host to use for the web server.",
    )

    # Add port argument
    parser.add_argument(
        "--port",
        type=int,
        help="The port to run the web server on.",
    )

    # Add keep alive argument
    parser.add_argument(
        "--keep-alive",
        action="store_const",
        const=True,
        help="Keep the web server alive even when no clients are connected.",
    )

    # Add allowed origins argument
    parser.add_argument(
        "--allowed-origins",
        nargs="*",
        help="The allowed origins for the web server.",
    )

    # Add use output directory argument
    parser.add_argument(
        "--web-use-output-dir",
        action="store_const",
        const=True,
        help=(
            "Use the output directory instead of the session directory for downloads. ("
            "This might cause issues if you have multiple users using the web-ui at the same time)"
        ),
    )

    # Add keep sessions argument
    parser.add_argument(
        "--keep-sessions",
        action="store_const",
        const=True,
        help="Keep the session directory after the web server is closed.",
    )

    # Add keep sessions argument
    parser.add_argument(
        "--force-update-gui",
        action="store_const",
        const=True,
        default=False,
        help="Refresh the web server directory with a fresh git checkout",
    )

    # Add custom web gui repo
    parser.add_argument(
        "--web-gui-repo",
        type=str,
        help=(
            "Custom web gui repo to use for the web server. "
            "Example: https://github.com/spotdl/web-ui/tree/master/dist"
        ),
    )

    # Add custom web gui repo
    parser.add_argument(
        "--web-gui-location",
        type=str,
        help="Path to the web gui directory to use for the web server.",
    )

    # Enable TLS for the web server
    parser.add_argument(
        "--enable-tls",
        action="store_const",
        const=True,
        help="Enable TLS on the web server.",
    )

    # Add File Location of the TLS Certificate file (Pem Format)
    parser.add_argument(
        "--cert-file", type=str, help="File Path to the TLS Certificate (PEM format)."
    )

    # Add File Location of the TLS Private Key file (Pem Format)
    parser.add_argument(
        "--key-file", type=str, help="File Path to the TLS Private Key (PEM format)."
    )

    # Add File Location of the TLS Certificate Authority file (Pem Format)
    parser.add_argument(
        "--ca-file",
        type=str,
        help="File Path to the TLS Certificate Authority File (PEM format).",
    )


def parse_misc_options(parser: _ArgumentGroup):
    """
    Parse misc options from the command line.

    ### Arguments
    - parser: The argument parser to add the options to.
    """

    # Add verbose argument
    parser.add_argument(
        "--log-level",
        choices=NAME_TO_LEVEL.keys(),
        help="Select log level.",
    )

    # Add simple tui argument
    parser.add_argument(
        "--simple-tui",
        action="store_const",
        const=True,
        help="Use a simple tui.",
    )

    # Add log format argument
    parser.add_argument(
        "--log-format",
        help=(
            "Custom logging format to use. More info: "
            "https://docs.python.org/3/library/logging.html#logrecord-attributes"
        ),
    )


def parse_other_options(parser: _ArgumentGroup):
    """
    Parse other options from the command line.

    ### Arguments
    - parser: The argument parser to add the options to.
    """

    parser.add_argument(
        "--download-ffmpeg",
        action="store_true",
        help="Download ffmpeg to spotdl directory.",
    )

    parser.add_argument(
        "--generate-config",
        action="store_true",
        help="Generate a config file. This will overwrite current config if present.",
    )

    parser.add_argument(
        "--check-for-updates", action="store_true", help="Check for new version."
    )

    parser.add_argument(
        "--profile",
        action="store_true",
        help="Run in profile mode. Useful for debugging.",
    )

    parser.add_argument(
        "--version",
        "-v",
        action="version",
        help="Show the version number and exit.",
        version=_version.__version__,
    )


def create_parser() -> ArgumentParser:
    """
    Parse arguments from the command line.

    ### Returns
    - A Namespace object containing the parsed arguments.
    """

    # Initialize argument parser
    parser = ArgumentParser(
        prog="spotdl",
        description="Download your Spotify playlists and songs along with album art and metadata",
        formatter_class=SmartFormatter,
        epilog=(
            "For more information, visit http://spotdl.rtfd.io/ "
            "or join our Discord server: https://discord.com/invite/xCa23pwJWY"
        ),
    )

    # Parse main options
    main_options = parser.add_argument_group("Main options")
    parse_main_options(main_options)

    # Parse spotify options
    spotify_options = parser.add_argument_group("Spotify options")
    parse_spotify_options(spotify_options)

    # Parse ffmpeg options
    ffmpeg_options = parser.add_argument_group("FFmpeg options")
    parse_ffmpeg_options(ffmpeg_options)

    # Parse output options
    output_options = parser.add_argument_group("Output options")
    parse_output_options(output_options)

    # Parse web options
    web_options = parser.add_argument_group("Web options")
    parse_web_options(web_options)

    # Parse misc options
    misc_options = parser.add_argument_group("Misc options")
    parse_misc_options(misc_options)

    # Parse other options
    other_options = parser.add_argument_group("Other options")
    parse_other_options(other_options)

    return parser


def parse_arguments() -> Namespace:
    """
    Parse arguments from the command line.

    ### Arguments
    - parser: The argument parser to parse the arguments from.

    ### Returns
    - A Namespace object containing the parsed arguments.
    """

    # Create parser
    parser = create_parser()

    # Parse arguments
    return parser.parse_args()



File: spotdl/utils/ffmpeg.py

"""
Module for converting audio files to different formats
and checking for ffmpeg binary, and downloading it if not found.
"""

import os
import platform
import re
import shlex
import shutil
import stat
import subprocess
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Tuple, Union

import requests

from spotdl.utils.config import get_spotdl_path
from spotdl.utils.formatter import to_ms

__all__ = [
    "FFMPEG_URLS",
    "FFMPEG_FORMATS",
    "DUR_REGEX",
    "TIME_REGEX",
    "VERSION_REGEX",
    "YEAR_REGEX",
    "FFmpegError",
    "is_ffmpeg_installed",
    "get_ffmpeg_path",
    "get_ffmpeg_version",
    "get_local_ffmpeg",
    "download_ffmpeg",
    "convert",
]

FFMPEG_URLS = {
    "windows": {
        "amd64": "https://github.com/eugeneware/ffmpeg-static/releases/download/b4.4/win32-x64",
        "i686": "https://github.com/eugeneware/ffmpeg-static/releases/download/b4.4/win32-ia32",
    },
    "linux": {
        "x86_64": "https://github.com/eugeneware/ffmpeg-static/releases/download/b4.4/linux-x64",
        "x86": "https://github.com/eugeneware/ffmpeg-static/releases/download/b4.4/linux-ia32",
        "arm32": "https://github.com/eugeneware/ffmpeg-static/releases/download/b4.4/linux-arm",
        "aarch64": "https://github.com/eugeneware/ffmpeg-static/releases/download/b4.4/linux-arm64",
    },
    "darwin": {
        "x86_64": "https://github.com/eugeneware/ffmpeg-static/releases/download/b4.4/darwin-x64",
        "arm64": "https://github.com/eugeneware/ffmpeg-static/releases/download/b4.4/darwin-arm64",
    },
}

FFMPEG_FORMATS = {
    "mp3": ["-codec:a", "libmp3lame"],
    "flac": ["-codec:a", "flac", "-sample_fmt", "s16"],
    "ogg": ["-codec:a", "libvorbis"],
    "opus": ["-codec:a", "libopus"],
    "m4a": ["-codec:a", "aac"],
    "wav": ["-codec:a", "pcm_s16le"],
}

DUR_REGEX = re.compile(
    r"Duration: (?P<hour>\d{2}):(?P<min>\d{2}):(?P<sec>\d{2})\.(?P<ms>\d{2})"
)
TIME_REGEX = re.compile(
    r"out_time=(?P<hour>\d{2}):(?P<min>\d{2}):(?P<sec>\d{2})\.(?P<ms>\d{2})"
)
VERSION_REGEX = re.compile(r"ffmpeg version \w?(\d+\.)?(\d+)")
YEAR_REGEX = re.compile(r"Copyright \(c\) \d\d\d\d\-\d\d\d\d")


class FFmpegError(Exception):
    """
    Base class for all exceptions related to FFmpeg.
    """


def is_ffmpeg_installed(ffmpeg: str = "ffmpeg") -> bool:
    """
    Check if ffmpeg is installed.

    ### Arguments
    - ffmpeg: ffmpeg executable to check

    ### Returns
    - True if ffmpeg is installed, False otherwise.
    """

    if ffmpeg == "ffmpeg":
        global_ffmpeg = shutil.which("ffmpeg")
        if global_ffmpeg is None:
            ffmpeg_path = get_ffmpeg_path()
        else:
            ffmpeg_path = Path(global_ffmpeg)
    else:
        ffmpeg_path = Path(ffmpeg)

    if ffmpeg_path is None:
        return False

    # else check if path to ffmpeg is valid
    # and if ffmpeg has the correct access rights
    return ffmpeg_path.exists() and os.access(ffmpeg_path, os.X_OK)


def get_ffmpeg_path() -> Optional[Path]:
    """
    Get path to global ffmpeg binary or a local ffmpeg binary.

    ### Returns
    - Path to ffmpeg binary or None if not found.
    """

    # Check if ffmpeg is installed
    global_ffmpeg = shutil.which("ffmpeg")
    if global_ffmpeg:
        return Path(global_ffmpeg)

    # Get local ffmpeg path
    return get_local_ffmpeg()


def get_ffmpeg_version(ffmpeg: str = "ffmpeg") -> Tuple[Optional[float], Optional[int]]:
    """
    Get ffmpeg version.

    ### Arguments
    - ffmpeg: ffmpeg executable to check

    ### Returns
    - Tuple of optional version and optional year.

    ### Errors
    - FFmpegError if ffmpeg is not installed.
    - FFmpegError if ffmpeg version is not found.
    """

    # Check if ffmpeg is installed
    if not is_ffmpeg_installed(ffmpeg):
        if ffmpeg == "ffmpeg":
            raise FFmpegError("ffmpeg is not installed.")

        raise FFmpegError(f"{ffmpeg} is not a valid ffmpeg executable.")

    with subprocess.Popen(
        [ffmpeg, "-version"],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        encoding="utf-8",
    ) as process:
        output = "".join(process.communicate())

    # Search for version and build year in output
    version_result = VERSION_REGEX.search(output)
    year_result = YEAR_REGEX.search(output)

    build_year = None
    version = None

    if version_result is not None:
        # remove all non numeric characters from string example: n4.3
        version_str = re.sub(r"[a-zA-Z]", "", version_result.group(0))

        # parse version string to float
        version = float(version_str) if version_str else None

    if year_result is not None:
        # get build years from string example: Copyright (c) 2019-2020
        build_years = [
            int(
                re.sub(r"[^0-9]", "", year)
            )  # remove all non numeric characters from string
            for year in year_result.group(0).split(
                "-"
            )  # split string into list of years
        ]

        # get the highest build year
        build_year = max(build_years)

    return (version, build_year)


def get_local_ffmpeg() -> Optional[Path]:
    """
    Get local ffmpeg binary path.

    ### Returns
    - Path to ffmpeg binary or None if not found.
    """

    ffmpeg_path = Path(get_spotdl_path()) / (
        "ffmpeg" + (".exe" if platform.system() == "Windows" else "")
    )

    if ffmpeg_path.is_file():
        return ffmpeg_path

    return None


def download_ffmpeg() -> Path:
    """
    Download ffmpeg binary to spotdl directory.

    ### Returns
    - Path to ffmpeg binary.

    ### Notes
    - ffmpeg is downloaded from github releases
        for current platform and architecture.
    - executable permission is set for ffmpeg binary.
    """

    os_name = platform.system().lower()
    os_arch = platform.machine().lower()
    ffmpeg_url: Optional[str] = None

    # if platform.system() == "Darwin" and (
    #     platform.processor() == "arm"
    #     or subprocess.run(["sysctl", "-n", "sysctl.proc_translated"], check=False)
    # ):
    #     ffmpeg_url = FFMPEG_URLS["darwin"]["arm"]
    # else:
    #     ffmpeg_url = FFMPEG_URLS.get(os_name, {}).get(os_arch)

    ffmpeg_url = FFMPEG_URLS.get(os_name, {}).get(os_arch)

    if ffmpeg_url is None:
        raise FFmpegError("FFmpeg binary is not available for your system.")

    ffmpeg_path = Path(
        os.path.join(
            get_spotdl_path(), "ffmpeg" + (".exe" if os_name == "windows" else "")
        )
    )

    # Download binary and save it to a file in spotdl directory
    ffmpeg_binary = requests.get(ffmpeg_url, allow_redirects=True, timeout=10).content
    with open(ffmpeg_path, "wb") as ffmpeg_file:
        ffmpeg_file.write(ffmpeg_binary)

    # Set executable permission on linux and mac
    if os_name in ["linux", "darwin"]:
        ffmpeg_path.chmod(ffmpeg_path.stat().st_mode | stat.S_IEXEC)

    return ffmpeg_path


def convert(
    input_file: Union[Path, Tuple[str, str]],
    output_file: Path,
    ffmpeg: str = "ffmpeg",
    output_format: str = "mp3",
    bitrate: Optional[str] = None,
    ffmpeg_args: Optional[str] = None,
    progress_handler: Optional[Callable[[int], None]] = None,
) -> Tuple[bool, Optional[Dict[str, Any]]]:
    """
    Convert the input file to the output file synchronously with progress handler.

    ### Arguments
    - input_file: Path to input file or tuple of (url, file_format).
    - output_file: Path to output file.
    - ffmpeg: ffmpeg executable to use.
    - output_format: output format.
    - bitrate: constant/variable bitrate.
    - ffmpeg_args: ffmpeg arguments.
    - progress_handler: progress handler, has to accept an integer as argument.

    ### Returns
    - Tuple of conversion status and error dictionary.

    ### Notes
    - Make sure to check if ffmpeg is installed before calling this function.
    """

    # Initialize ffmpeg command
    # -i is the input file
    arguments: List[str] = [
        "-nostdin",
        "-y",
        "-i",
        str(input_file.resolve()) if isinstance(input_file, Path) else input_file[0],
        "-movflags",
        "+faststart",
        "-v",
        "debug",
        "-progress",
        "-",
        "-nostats",
    ]

    file_format = (
        str(input_file.suffix).split(".")[1]
        if isinstance(input_file, Path)
        else input_file[1]
    )

    # Add output format to command
    # -c:a is used if the file is not an matroska container
    # and we want to convert to opus
    # otherwise we use arguments from FFMPEG_FORMATS
    if output_format == "opus" and file_format != "webm":
        arguments.extend(["-c:a", "libopus"])
    else:
        if (
            (output_format == "opus" and file_format == "webm")
            or (output_format == "m4a" and file_format == "m4a")
            and not (bitrate or ffmpeg_args)
        ):
            # Copy the audio stream to the output file
            arguments.extend(["-vn", "-c:a", "copy"])
        else:
            arguments.extend(FFMPEG_FORMATS[output_format])

    # Add bitrate if specified
    if bitrate:
        # Check if bitrate is an integer
        # if it is then use it as variable bitrate
        if bitrate.isdigit():
            arguments.extend(["-q:a", bitrate])
        else:
            arguments.extend(["-b:a", bitrate])

    # Add other ffmpeg arguments if specified
    if ffmpeg_args:
        arguments.extend(shlex.split(ffmpeg_args))

    # Add output file at the end
    arguments.append(str(output_file.resolve()))

    # Run ffmpeg
    with subprocess.Popen(
        [ffmpeg, *arguments],
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        universal_newlines=False,
    ) as process:
        if not progress_handler:
            # Wait for process to finish
            proc_out = process.communicate()

            if process.returncode != 0:
                # get version and build year
                version = get_ffmpeg_version(ffmpeg)

                # join stdout and stderr and decode to utf-8
                message = b"".join([out for out in proc_out if out]).decode("utf-8")

                # return error dictionary
                return False, {
                    "return_code": process.returncode,
                    "arguments": arguments,
                    "ffmpeg": ffmpeg,
                    "version": version[0],
                    "build_year": version[1],
                    "error": message,
                }

            return True, None

        progress_handler(0)

        out_buffer = []
        total_dur = None
        while True:
            if process.stdout is None:
                continue

            out_line = (
                process.stdout.readline().decode("utf-8", errors="replace").strip()
            )

            if out_line == "" and process.poll() is not None:
                break

            out_buffer.append(out_line.strip())

            total_dur_match = DUR_REGEX.search(out_line)
            if total_dur is None and total_dur_match:
                total_dur = to_ms(**total_dur_match.groupdict())  # type: ignore
                continue
            if total_dur:
                progress_time = TIME_REGEX.search(out_line)
                if progress_time:
                    elapsed_time = to_ms(**progress_time.groupdict())  # type: ignore
                    progress_handler(int(elapsed_time / total_dur * 100))  # type: ignore

        if process.returncode != 0:
            # get version and build year
            version = get_ffmpeg_version(ffmpeg)

            return False, {
                "return_code": process.returncode,
                "arguments": arguments,
                "ffmpeg": ffmpeg,
                "version": version[0],
                "build_year": version[1],
                "error": "\n".join(out_buffer),
            }

        progress_handler(100)

        return True, None



File: spotdl/utils/__init__.py

"""
Utility functions for spotdl. These functions are used in every stage of the
download process.
"""



File: spotdl/utils/matching.py

"""
Module for all things matching related
"""

import logging
from itertools import product, zip_longest
from math import exp
from typing import Dict, List, Optional, Tuple

from spotdl.types.result import Result
from spotdl.types.song import Song
from spotdl.utils.formatter import (
    create_search_query,
    create_song_title,
    ratio,
    slugify,
)
from spotdl.utils.logging import MATCH

__all__ = [
    "FORBIDDEN_WORDS",
    "fill_string",
    "create_clean_string",
    "sort_string",
    "based_sort",
    "check_common_word",
    "check_forbidden_words",
    "create_match_strings",
    "get_best_matches",
    "calc_main_artist_match",
    "calc_artists_match",
    "artists_match_fixup1",
    "artists_match_fixup2",
    "artists_match_fixup3",
    "calc_name_match",
    "calc_time_match",
    "calc_album_match",
]

logger = logging.getLogger(__name__)

FORBIDDEN_WORDS = [
    "bassboosted",
    "remix",
    "remastered",
    "remaster",
    "reverb",
    "bassboost",
    "live",
    "acoustic",
    "8daudio",
    "concert",
    "live",
    "acapella",
    "slowed",
    "instrumental",
    "remix",
    "cover",
    "reverb",
]


def debug(song_id: str, result_id: str, message: str) -> None:
    """
    Log a message with MATCH level

    ### Arguments
    - message: message to log
    """

    logger.log(MATCH, "[%s|%s] %s", song_id, result_id, message)


def fill_string(strings: List[str], main_string: str, string_to_check: str) -> str:
    """
    Create a string with strings from `strings` list
    if they are not yet present in main_string
    but are present in string_to_check

    ### Arguments
    - strings: strings to check
    - main_string: string to add strings to
    - string_to_check: string to check if strings are present in

    ### Returns
    - string with strings from `strings` list
    """

    final_str = main_string
    test_str = final_str.replace("-", "")
    simple_test_str = string_to_check.replace("-", "")
    for string in strings:
        slug_str = slugify(string).replace("-", "")

        if slug_str in simple_test_str and slug_str not in test_str:
            final_str += f"-{slug_str}"
            test_str += slug_str

    return final_str


def create_clean_string(
    words: List[str], string: str, sort: bool = False, join_str: str = "-"
) -> str:
    """
    Create a string with strings from `words` list
    if they are not yet present in `string`

    ### Arguments
    - words: strings to check
    - string: string to check if strings are present in
    - sort: sort strings in list
    - join_str: string to join strings with

    ### Returns
    - string with strings from `words` list
    """

    string = slugify(string).replace("-", "")

    final = []
    for word in words:
        word = slugify(word).replace("-", "")

        if word in string:
            continue

        final.append(word)

    if sort:
        return sort_string(final, join_str)

    return f"{join_str}".join(final)


def sort_string(strings: List[str], join_str: str) -> str:
    """
    Sort strings in list and join them with `join` string

    ### Arguments
    - strings: strings to sort
    - join: string to join strings with

    ### Returns
    - joined sorted string
    """

    final_str = strings
    final_str.sort()

    return f"{join_str}".join(final_str)


def based_sort(strings: List[str], based_on: List[str]) -> Tuple[List[str], List[str]]:
    """
    Sort strings in list based on the order of strings in `based_on` list

    ### Arguments
    - strings: strings to sort
    - based_on: strings to sort `strings` list based on

    ### Returns
    - sorted list of strings
    """

    strings.sort()
    based_on.sort()

    list_map = {value: index for index, value in enumerate(based_on)}

    strings = sorted(
        strings,
        key=lambda x: list_map.get(x, -1),
        reverse=True,
    )

    based_on.reverse()

    return strings, based_on


def check_common_word(song: Song, result: Result) -> bool:
    """
    Check if a word is present in a sentence

    ### Arguments
    - song: song to match
    - result: result to match

    ### Returns
    - True if word is present in sentence, False otherwise
    """

    sentence_words = slugify(song.name).split("-")
    to_check = slugify(result.name).replace("-", "")

    for word in sentence_words:
        if word != "" and word in to_check:
            return True

    return False


def check_forbidden_words(song: Song, result: Result) -> Tuple[bool, List[str]]:
    """
    Check if a forbidden word is present in the result name

    ### Arguments
    - song: song to match
    - result: result to match

    ### Returns
    - True if forbidden word is present in result name, False otherwise
    """

    song_name = slugify(song.name).replace("-", "")
    to_check = slugify(result.name).replace("-", "")

    words = []
    for word in FORBIDDEN_WORDS:
        if word in to_check and word not in song_name:
            words.append(word)

    return len(words) > 0, words


def create_match_strings(
    song: Song, result: Result, search_query: Optional[str] = None
) -> Tuple[str, str]:
    """
    Create strings based on song and result to match
    fill strings with missing artists

    ### Arguments
    - song: song to match
    - result: result to match

    ### Returns
    - tuple of strings to match
    """

    slug_song_name = slugify(song.name)
    slug_song_title = slugify(
        create_song_title(song.name, song.artists)
        if not search_query
        else create_search_query(song, search_query, False, None, True)
    )

    test_str1 = slugify(result.name)
    test_str2 = slug_song_name if result.verified else slug_song_title

    # Fill strings with missing artists
    test_str1 = fill_string(song.artists, test_str1, test_str2)
    test_str2 = fill_string(song.artists, test_str2, test_str1)

    # Sort both strings and then join them
    test_list1, test_list2 = based_sort(test_str1.split("-"), test_str2.split("-"))
    test_str1, test_str2 = "-".join(test_list1), "-".join(test_list2)

    return test_str1, test_str2


def get_best_matches(
    results: Dict[Result, float], score_threshold: float
) -> List[Tuple[Result, float]]:
    """
    Get best matches from a list of results

    ### Arguments
    - results: list of results to match
    - score_threshold: threshold to match results

    ### Returns
    - list of best matches
    """

    result_items = list(results.items())

    # Sort results by highest score
    sorted_results = sorted(result_items, key=lambda x: x[1], reverse=True)

    best_score = sorted_results[0][1]

    return [
        result
        for result in sorted_results
        if (best_score - result[1]) <= score_threshold
    ]


def calc_main_artist_match(song: Song, result: Result) -> float:
    """
    Check if main artist is present in list of artists

    ### Arguments
    - main_artist: main artist to check
    - artists: list of artists to check

    ### Returns
    - True if main artist is present in list of artists, False otherwise
    """

    main_artist_match = 0.0

    # Result has no artists, return 0.0
    if not result.artists:
        return main_artist_match

    song_artists, result_artists = list(map(slugify, song.artists)), list(
        map(slugify, result.artists)
    )
    sorted_song_artists, sorted_result_artists = based_sort(
        song_artists, result_artists
    )

    debug(song.song_id, result.result_id, f"Song artists: {sorted_song_artists}")
    debug(song.song_id, result.result_id, f"Result artists: {sorted_result_artists}")

    slug_song_main_artist = slugify(song.artists[0])
    slug_result_main_artist = sorted_result_artists[0]

    # Result has only one artist, but song has multiple artists
    # we can assume that other artists are in the main artist name
    if len(song.artists) > 1 and len(result.artists) == 1:
        for artist in map(slugify, song.artists[1:]):
            artist = sort_string(slugify(artist).split("-"), "-")

            res_main_artist = sort_string(slug_result_main_artist.split("-"), "-")

            if artist in res_main_artist:
                main_artist_match += 100 / len(song.artists)

        return main_artist_match

    # Match main result artist with main song artist
    main_artist_match = ratio(slug_song_main_artist, slug_result_main_artist)

    debug(
        song.song_id, result.result_id, f"First main artist match: {main_artist_match}"
    )

    # Use second artist from the sorted list to
    # calculate the match if the first artist match is too low
    if main_artist_match < 50 and len(song_artists) > 1:
        for song_artist, result_artist in product(
            song_artists[:2], sorted_result_artists[:2]
        ):
            new_artist_match = ratio(song_artist, result_artist)
            debug(
                song.song_id,
                result.result_id,
                f"Matched {song_artist} with {result_artist}: {new_artist_match}",
            )

            main_artist_match = max(main_artist_match, new_artist_match)

    return main_artist_match


def calc_artists_match(song: Song, result: Result) -> float:
    """
    Check if all artists are present in list of artists

    ### Arguments
    - song: song to match
    - result: result to match

    ### Returns
    - artists match percentage
    """

    artist_match_number = 0.0

    # Result has only one artist, return 0.0
    if len(song.artists) == 1 or not result.artists:
        return artist_match_number

    artist1_list, artist2_list = based_sort(
        list(map(slugify, song.artists)), list(map(slugify, result.artists))
    )

    # Remove main artist from the lists
    artist1_list, artist2_list = artist1_list[1:], artist2_list[1:]

    artists_match = 0.0
    for artist1, artist2 in zip_longest(artist1_list, artist2_list):
        artist12_match = ratio(artist1, artist2)
        artists_match += artist12_match

    artist_match_number = artists_match / len(artist1_list)

    return artist_match_number


def artists_match_fixup1(song: Song, result: Result, score: float) -> float:
    """
    Multiple fixes to the artists score for
    not verified results to improve the accuracy

    ### Arguments
    - song: song to match
    - result: result to match
    - score: current score

    ### Returns
    - new score
    """

    # If we have a verified result, we don't have to fix anything
    if result.verified or score > 50:
        return score

    # If we didn't find any artist match,
    # we fallback to channel name match
    channel_name_match = ratio(
        slugify(song.artist),
        slugify(", ".join(result.artists)) if result.artists else "",
    )

    score = max(score, channel_name_match)

    # If artist match is still too low,
    # we fallback to matching all song artist names
    # with the result's title
    if score <= 70:
        artist_title_match = 0.0
        result_name = slugify(result.name).replace("-", "")
        for artist in song.artists:
            slug_artist = slugify(artist).replace("-", "")

            if slug_artist in result_name:
                artist_title_match += 1.0

        artist_title_match = (artist_title_match / len(song.artists)) * 100

        score = max(score, artist_title_match)

    # If artist match is still too low,
    # we fallback to matching all song artist names
    # with the result's artists
    if score <= 70:
        # Song artists: ['charlie-moncler', 'fukaj', 'mata', 'pedro']
        # Result artists: ['fukaj-mata-charlie-moncler-und-pedro']

        # For artist_list1
        artist_list1 = []
        for artist in song.artists:
            artist_list1.extend(slugify(artist).split("-"))

        # For artist_list2
        artist_list2 = []
        if result.artists:
            for artist in result.artists:
                artist_list2.extend(slugify(artist).split("-"))

        artist_tuple1 = tuple(artist_list1)
        artist_tuple2 = tuple(artist_list2)

        artist_title_match = ratio(artist_tuple1, artist_tuple2)

        score = max(score, artist_title_match)

    return score


def artists_match_fixup2(
    song: Song, result: Result, score: float, search_query: Optional[str] = None
) -> float:
    """
    Multiple fixes to the artists score for
    verified results to improve the accuracy

    ### Arguments
    - song: song to match
    - result: result to match
    - score: current score

    ### Returns
    - new score
    """

    if score > 70 or not result.verified:
        # Don't fixup the score
        # if the artist match is already high
        # or if the result is not verified
        return score

    # Slugify some variables
    slug_song_name = slugify(song.name)
    slug_result_name = slugify(result.name)

    # # Check if the main artist is simlar
    has_main_artist = (score / (2 if len(song.artists) > 1 else 1)) > 50

    _, match_str2 = create_match_strings(song, result, search_query)

    # Check if other song artists are in the result name
    # if they are, we increase the artist match
    # (main artist is already checked, so we skip it)
    artists_to_check = song.artists[int(has_main_artist) :]
    for artist in artists_to_check:
        artist = slugify(artist).replace("-", "")
        if artist in match_str2.replace("-", ""):
            score += 5

    # if the artist match is still too low,
    # we fallback to matching all song artist names
    # with the result's artists
    if score <= 70:
        # Artists from song/result name without the song/result name words
        artist_list1 = create_clean_string(song.artists, slug_song_name, True)
        artist_list2 = create_clean_string(
            list(result.artists) if result.artists else [result.author],
            slug_result_name,
            True,
        )

        artist_title_match = ratio(artist_list1, artist_list2)

        score = max(score, artist_title_match)

    return score


def artists_match_fixup3(song: Song, result: Result, score: float) -> float:
    """
    Calculate match percentage based result's name
    and song's title if the result has exactly one artist
    and the song has more than one artist

    ### Arguments
    - song: song to match
    - result: result to match
    - score: current score

    ### Returns
    - new score
    """

    if (
        score > 70
        or not result.artists
        or len(result.artists) > 1
        or len(song.artists) == 1
    ):
        # Don't fixup the score
        # if the score is already high
        # or if the result has more than one artist
        # or if the song has only one artist
        return score

    artists_score_fixup = ratio(
        slugify(result.name),
        slugify(create_song_title(song.name, [song.artist])),
    )

    if artists_score_fixup >= 80:
        score = (score + artists_score_fixup) / 2

    # Make sure that the score is not higher than 100
    score = min(score, 100)

    return score


def calc_name_match(
    song: Song, result: Result, search_query: Optional[str] = None
) -> float:
    """
    Calculate name match percentage

    ### Arguments
    - song: song to match
    - result: result to match

    ### Returns
    - name match percentage
    """

    # Create match strings that will be used
    # to calculate name match value
    match_str1, match_str2 = create_match_strings(song, result, search_query)
    result_name, song_name = slugify(result.name), slugify(song.name)

    res_list, song_list = based_sort(result_name.split("-"), song_name.split("-"))
    result_name, song_name = "-".join(res_list), "-".join(song_list)

    # Calculate initial name match
    name_match = ratio(result_name, song_name)

    debug(song.song_id, result.result_id, f"MATCH STRINGS: {match_str1} - {match_str2}")
    debug(
        song.song_id,
        result.result_id,
        f"SLUG MATCH STRINGS: {song_name} - {result_name}",
    )
    debug(song.song_id, result.result_id, f"First name match: {name_match}")

    # If name match is lower than 60%,
    # we try to match using the test strings
    if name_match <= 75:
        second_name_match = ratio(
            match_str1,
            match_str2,
        )

        debug(
            song.song_id,
            result.result_id,
            f"Second name match: {second_name_match}",
        )

        name_match = max(name_match, second_name_match)

    return name_match


def calc_time_match(song: Song, result: Result) -> float:
    """
    Calculate time difference between song and result

    ### Arguments
    - song: song to match
    - result: result to match

    ### Returns
    - time difference between song and result
    """

    time_diff = abs(song.duration - result.duration)
    score = exp(-0.1 * time_diff)
    return score * 100


def calc_album_match(song: Song, result: Result) -> float:
    """
    Calculate album match percentage

    ### Arguments
    - song: song to match
    - result: result to match

    ### Returns
    - album match percentage
    """

    if not result.album:
        return 0.0

    return ratio(slugify(song.album_name), slugify(result.album))


def order_results(
    results: List[Result],
    song: Song,
    search_query: Optional[str] = None,
) -> Dict[Result, float]:
    """
    Order results.

    ### Arguments
    - results: The results to order.
    - song: The song to order for.
    - search_query: The search query.

    ### Returns
    - The ordered results.
    """

    # Assign an overall avg match value to each result
    links_with_match_value = {}

    # Iterate over all results
    for result in results:
        debug(
            song.song_id,
            result.result_id,
            f"Calculating match value for {result.url} - {result.json}",
        )

        # skip results that have no common words in their name
        if not check_common_word(song, result):
            debug(
                song.song_id, result.result_id, "Skipping result due to no common words"
            )

            continue

        # Calculate match value for main artist
        artists_match = calc_main_artist_match(song, result)
        debug(song.song_id, result.result_id, f"Main artist match: {artists_match}")

        # Calculate match value for all artists
        other_artists_match = calc_artists_match(song, result)
        debug(
            song.song_id,
            result.result_id,
            f"Other artists match: {other_artists_match}",
        )

        artists_match += other_artists_match

        # Calculate initial artist match value
        debug(song.song_id, result.result_id, f"Initial artists match: {artists_match}")
        artists_match = artists_match / (2 if len(song.artists) > 1 else 1)
        debug(song.song_id, result.result_id, f"First artists match: {artists_match}")

        # First attempt to fix artist match
        artists_match = artists_match_fixup1(song, result, artists_match)
        debug(
            song.song_id,
            result.result_id,
            f"Artists match after fixup1: {artists_match}",
        )

        # Second attempt to fix artist match
        artists_match = artists_match_fixup2(song, result, artists_match)
        debug(
            song.song_id,
            result.result_id,
            f"Artists match after fixup2: {artists_match}",
        )

        # Third attempt to fix artist match
        artists_match = artists_match_fixup3(song, result, artists_match)
        debug(
            song.song_id,
            result.result_id,
            f"Artists match after fixup3: {artists_match}",
        )

        debug(song.song_id, result.result_id, f"Final artists match: {artists_match}")

        # Calculate name match
        name_match = calc_name_match(song, result, search_query)
        debug(song.song_id, result.result_id, f"Initial name match: {name_match}")

        # Check if result contains forbidden words
        contains_fwords, found_fwords = check_forbidden_words(song, result)
        if contains_fwords:
            for _ in found_fwords:
                name_match -= 15

        debug(
            song.song_id,
            result.result_id,
            f"Contains forbidden words: {contains_fwords}, {found_fwords}",
        )
        debug(song.song_id, result.result_id, f"Final name match: {name_match}")

        # Calculate album match
        album_match = calc_album_match(song, result)
        debug(song.song_id, result.result_id, f"Final album match: {album_match}")

        # Calculate time match
        time_match = calc_time_match(song, result)
        debug(song.song_id, result.result_id, f"Final time match: {time_match}")

        # Ignore results with name match lower than 60%
        if name_match <= 60:
            debug(
                song.song_id,
                result.result_id,
                "Skipping result due to name match lower than 60%",
            )
            continue

        # Ignore results with artists match lower than 70%
        if artists_match < 70 and result.source != "slider.kz":
            debug(
                song.song_id,
                result.result_id,
                "Skipping result due to artists match lower than 70%",
            )
            continue

        # Calculate total match
        average_match = (artists_match + name_match) / 2
        debug(song.song_id, result.result_id, f"Average match: {average_match}")

        if (
            result.verified
            and not result.isrc_search
            and result.album
            and album_match <= 80
        ):
            # we are almost certain that this is the correct result
            # so we add the album match to the average match
            average_match = (average_match + album_match) / 2
            debug(
                song.song_id,
                result.result_id,
                f"Average match /w album match: {average_match}",
            )

        # Skip results with time match lower than 25%
        if time_match < 25:
            debug(
                song.song_id,
                result.result_id,
                "Skipping result due to time match lower than 25%",
            )
            continue

        # If the time match is lower than 50%
        # and the average match is lower than 75%
        # we skip the result
        if time_match < 50 and average_match < 75:
            debug(
                song.song_id,
                result.result_id,
                "Skipping result due to time match < 50% and average match < 75%",
            )
            continue

        if (
            (not result.isrc_search and average_match <= 85)
            or result.source == "slider.kz"
            or time_match < 0
        ):
            # Don't add time to avg match if average match is not the best
            # (lower than 85%), always include time match if result is from
            # slider.kz or if time match is lower than 0
            average_match = (average_match + time_match) / 2

            debug(
                song.song_id,
                result.result_id,
                f"Average match /w time match: {average_match}",
            )

            if (result.explicit is not None and song.explicit is not None) and (
                result.explicit != song.explicit
            ):
                debug(
                    song.song_id,
                    result.result_id,
                    "Lowering average match due to explicit mismatch",
                )

                average_match -= 5

        average_match = min(average_match, 100)
        debug(song.song_id, result.result_id, f"Final average match: {average_match}")

        # the results along with the avg Match
        links_with_match_value[result] = average_match

    return links_with_match_value



File: spotdl/utils/m3u.py

"""
Module for creating m3u content and writing it to a file.
"""

from pathlib import Path
from typing import Dict, List, Optional

from spotdl.types.song import Song
from spotdl.utils.formatter import create_file_name, sanitize_string

__all__ = [
    "create_m3u_content",
    "gen_m3u_files",
    "create_m3u_file",
]


def create_m3u_content(
    song_list: List[Song],
    template: str,
    file_extension: str,
    restrict: Optional[str] = None,
    short: bool = False,
    detect_formats: Optional[List[str]] = None,
) -> str:
    """
    Create m3u content and return it as a string.

    ### Arguments
    - song_list: the list of songs
    - template: the template to use
    - file_extension: the file extension to use
    - restrict: sanitization to apply to the filename
    - short: whether to use the short version of the template

    ### Returns
    - the m3u content as a string
    """

    text = ""
    for song in song_list:
        if not detect_formats:
            file_name = create_file_name(
                song, template, file_extension, restrict, short
            )

            text += str(file_name) + "\n"
        else:
            for file_ext in detect_formats:
                file_name = create_file_name(song, template, file_ext, restrict, short)
                if file_name.exists():
                    text += str(file_name) + "\n"
                    break
            else:
                file_name = create_file_name(
                    song, template, file_extension, restrict, short
                )

                text += str(file_name) + "\n"

    return text


def gen_m3u_files(
    songs: List[Song],
    file_name: Optional[str],
    template: str,
    file_extension: str,
    restrict: Optional[str] = None,
    short: bool = False,
    detect_formats: Optional[List[str]] = None,
):
    """
    Create an m3u8 filename from the query.

    ### Arguments
    - query: the query
    - file_name: the file name to use
    - song_list: the list of songs
    - template: the output file template to use
    - file_extension: the file extension to use
    - restrict: sanitization to apply to the filename
    - short: whether to use the short version of the template
    - detect_formats: the formats to detect
    """

    # If no file name is provided, use the first list's name
    if not file_name:
        file_name = "{list[0]}.m3u8"

    # If file_name ends with a slash. Does not have a m3u8 name with extension
    # at the end of the template, append `{list[0]}`` to it
    if (
        file_name.endswith("/")
        or file_name.endswith(r"\\")
        or file_name.endswith("\\\\")
    ):
        file_name += "/{list[0]}.m3u8"

    # Check if the file name ends with .m3u or .m3u8
    if not file_name.endswith(".m3u") and not file_name.endswith(".m3u8"):
        file_name += ".m3u8"

    lists_object: Dict[str, List[Song]] = {}
    for song in songs:
        if song.list_name is None:
            continue

        if song.list_name not in lists_object:
            lists_object[song.list_name] = []

        lists_object[song.list_name].append(song)

    if "{list}" in file_name:
        # Create multiple m3u files if there are multiple lists
        for list_name, song_list in lists_object.items():
            create_m3u_file(
                file_name.format(
                    list=list_name,
                ),
                song_list,
                template,
                file_extension,
                restrict,
                short,
                detect_formats,
            )
    elif "{list[" in file_name and "]}" in file_name:
        # Create a single m3u file for specified song list name
        create_m3u_file(
            file_name.format(list=list(lists_object.keys())),
            songs,
            template,
            file_extension,
            restrict,
            short,
            detect_formats,
        )
    else:
        # Use the provided file name
        create_m3u_file(
            file_name,
            songs,
            template,
            file_extension,
            restrict,
            short,
            detect_formats,
        )


def create_m3u_file(
    file_name: str,
    song_list: List[Song],
    template: str,
    file_extension: str,
    restrict: Optional[str] = None,
    short: bool = False,
    detect_formats: Optional[List[str]] = None,
) -> str:
    """
    Create the m3u file.

    ### Arguments
    - file_name: the file name to use
    - song_list: the list of songs
    - template: the template to use
    - file_extension: the file extension to use
    - restrict: sanitization to apply to the filename
    - short: whether to use the short version of the template
    - detect_formats: the formats to detect

    ### Returns
    - the m3u content as a string
    """

    m3u_content = create_m3u_content(
        song_list,
        template,
        file_extension,
        restrict,
        short,
        detect_formats,
    )

    file_path = Path(sanitize_string(file_name)).absolute()

    with open(file_path, "w", encoding="utf-8") as m3u_file:
        m3u_file.write(m3u_content)

    return m3u_content



File: spotdl/utils/static.py

"""
Module for holding static variables that are used throughout the project.
And are for the most part, not likely to change.
(Also holds really large variables, so pylint is disabled here)
"""

# pylint: skip-file

__all__ = [
    "AMBIGUOUS_CHARACTERS",
    "BAD_CHARS",
]

BAD_CHARS = [12441, 12442] + list(range(769, 880))

# Source: https://github.com/hediet/vscode-unicode-data/blob/main/out/ambiguous.json
AMBIGUOUS_CHARACTERS = {
    "8232": 32,
    "8233": 32,
    "5760": 32,
    "8192": 32,
    "8193": 32,
    "8194": 32,
    "8195": 32,
    "8196": 32,
    "8197": 32,
    "8198": 32,
    "8200": 32,
    "8201": 32,
    "8202": 32,
    "8287": 32,
    "8199": 32,
    "8239": 32,
    "2042": 95,
    "65101": 95,
    "65102": 95,
    "65103": 95,
    "8208": 45,
    "8209": 45,
    "8210": 45,
    "65112": 45,
    "1748": 45,
    "8259": 45,
    "727": 45,
    "8722": 45,
    "10134": 45,
    "11450": 45,
    "1549": 44,
    "1643": 44,
    "8218": 44,
    "184": 44,
    "42233": 44,
    "894": 59,
    "2307": 58,
    "2691": 58,
    "1417": 58,
    "1795": 58,
    "1796": 58,
    "5868": 58,
    "65072": 58,
    "6147": 58,
    "6153": 58,
    "8282": 58,
    "1475": 58,
    "760": 58,
    "42889": 58,
    "8758": 58,
    "720": 58,
    "42237": 58,
    "451": 33,
    "11601": 33,
    "660": 63,
    "577": 63,
    "2429": 63,
    "5038": 63,
    "42731": 63,
    "119149": 46,
    "8228": 46,
    "1793": 46,
    "1794": 46,
    "42510": 46,
    "68176": 46,
    "1632": 46,
    "1776": 46,
    "42232": 46,
    "1373": 96,
    "65287": 96,
    "8219": 96,
    "8242": 96,
    "1370": 96,
    "1523": 96,
    "8175": 96,
    "65344": 96,
    "900": 96,
    "8189": 96,
    "8125": 96,
    "8127": 96,
    "8190": 96,
    "697": 96,
    "884": 96,
    "712": 96,
    "714": 96,
    "715": 96,
    "756": 96,
    "699": 96,
    "701": 96,
    "700": 96,
    "702": 96,
    "42892": 96,
    "1497": 96,
    "2036": 96,
    "2037": 96,
    "5194": 96,
    "5836": 96,
    "94033": 96,
    "94034": 96,
    "65339": 91,
    "10088": 40,
    "10098": 40,
    "12308": 40,
    "64830": 40,
    "65341": 93,
    "10089": 41,
    "10099": 41,
    "12309": 41,
    "64831": 41,
    "10100": 123,
    "119060": 123,
    "10101": 125,
    "65342": 94,
    "8270": 42,
    "1645": 42,
    "8727": 42,
    "66335": 42,
    "5941": 47,
    "8257": 47,
    "8725": 47,
    "8260": 47,
    "9585": 47,
    "10187": 47,
    "10744": 47,
    "119354": 47,
    "12755": 47,
    "12339": 47,
    "11462": 47,
    "20031": 47,
    "12035": 47,
    "65340": 92,
    "65128": 92,
    "8726": 92,
    "10189": 92,
    "10741": 92,
    "10745": 92,
    "119311": 92,
    "119355": 92,
    "12756": 92,
    "20022": 92,
    "12034": 92,
    "42872": 38,
    "708": 94,
    "710": 94,
    "5869": 43,
    "10133": 43,
    "66203": 43,
    "8249": 60,
    "10094": 60,
    "706": 60,
    "119350": 60,
    "5176": 60,
    "5810": 60,
    "5120": 61,
    "11840": 61,
    "12448": 61,
    "42239": 61,
    "8250": 62,
    "10095": 62,
    "707": 62,
    "119351": 62,
    "5171": 62,
    "94015": 62,
    "8275": 126,
    "732": 126,
    "8128": 126,
    "8764": 126,
    "65372": 124,
    "65293": 45,
    "120784": 50,
    "120794": 50,
    "120804": 50,
    "120814": 50,
    "120824": 50,
    "130034": 50,
    "42842": 50,
    "423": 50,
    "1000": 50,
    "42564": 50,
    "5311": 50,
    "42735": 50,
    "119302": 51,
    "120785": 51,
    "120795": 51,
    "120805": 51,
    "120815": 51,
    "120825": 51,
    "130035": 51,
    "42923": 51,
    "540": 51,
    "439": 51,
    "42858": 51,
    "11468": 51,
    "1248": 51,
    "94011": 51,
    "71882": 51,
    "120786": 52,
    "120796": 52,
    "120806": 52,
    "120816": 52,
    "120826": 52,
    "130036": 52,
    "5070": 52,
    "71855": 52,
    "120787": 53,
    "120797": 53,
    "120807": 53,
    "120817": 53,
    "120827": 53,
    "130037": 53,
    "444": 53,
    "71867": 53,
    "120788": 54,
    "120798": 54,
    "120808": 54,
    "120818": 54,
    "120828": 54,
    "130038": 54,
    "11474": 54,
    "5102": 54,
    "71893": 54,
    "119314": 55,
    "120789": 55,
    "120799": 55,
    "120809": 55,
    "120819": 55,
    "120829": 55,
    "130039": 55,
    "66770": 55,
    "71878": 55,
    "2819": 56,
    "2538": 56,
    "2666": 56,
    "125131": 56,
    "120790": 56,
    "120800": 56,
    "120810": 56,
    "120820": 56,
    "120830": 56,
    "130040": 56,
    "547": 56,
    "546": 56,
    "66330": 56,
    "2663": 57,
    "2920": 57,
    "2541": 57,
    "3437": 57,
    "120791": 57,
    "120801": 57,
    "120811": 57,
    "120821": 57,
    "120831": 57,
    "130041": 57,
    "42862": 57,
    "11466": 57,
    "71884": 57,
    "71852": 57,
    "71894": 57,
    "9082": 97,
    "65345": 97,
    "119834": 97,
    "119886": 97,
    "119938": 97,
    "119990": 97,
    "120042": 97,
    "120094": 97,
    "120146": 97,
    "120198": 97,
    "120250": 97,
    "120302": 97,
    "120354": 97,
    "120406": 97,
    "120458": 97,
    "593": 97,
    "945": 97,
    "120514": 97,
    "120572": 97,
    "120630": 97,
    "120688": 97,
    "120746": 97,
    "65313": 65,
    "119808": 65,
    "119860": 65,
    "119912": 65,
    "119964": 65,
    "120016": 65,
    "120068": 65,
    "120120": 65,
    "120172": 65,
    "120224": 65,
    "120276": 65,
    "120328": 65,
    "120380": 65,
    "120432": 65,
    "913": 65,
    "120488": 65,
    "120546": 65,
    "120604": 65,
    "120662": 65,
    "120720": 65,
    "5034": 65,
    "5573": 65,
    "42222": 65,
    "94016": 65,
    "66208": 65,
    "119835": 98,
    "119887": 98,
    "119939": 98,
    "119991": 98,
    "120043": 98,
    "120095": 98,
    "120147": 98,
    "120199": 98,
    "120251": 98,
    "120303": 98,
    "120355": 98,
    "120407": 98,
    "120459": 98,
    "388": 98,
    "5071": 98,
    "5234": 98,
    "5551": 98,
    "65314": 66,
    "8492": 66,
    "119809": 66,
    "119861": 66,
    "119913": 66,
    "120017": 66,
    "120069": 66,
    "120121": 66,
    "120173": 66,
    "120225": 66,
    "120277": 66,
    "120329": 66,
    "120381": 66,
    "120433": 66,
    "42932": 66,
    "914": 66,
    "120489": 66,
    "120547": 66,
    "120605": 66,
    "120663": 66,
    "120721": 66,
    "5108": 66,
    "5623": 66,
    "42192": 66,
    "66178": 66,
    "66209": 66,
    "66305": 66,
    "65347": 99,
    "8573": 99,
    "119836": 99,
    "119888": 99,
    "119940": 99,
    "119992": 99,
    "120044": 99,
    "120096": 99,
    "120148": 99,
    "120200": 99,
    "120252": 99,
    "120304": 99,
    "120356": 99,
    "120408": 99,
    "120460": 99,
    "7428": 99,
    "1010": 99,
    "11429": 99,
    "43951": 99,
    "66621": 99,
    "128844": 67,
    "71922": 67,
    "71913": 67,
    "65315": 67,
    "8557": 67,
    "8450": 67,
    "8493": 67,
    "119810": 67,
    "119862": 67,
    "119914": 67,
    "119966": 67,
    "120018": 67,
    "120174": 67,
    "120226": 67,
    "120278": 67,
    "120330": 67,
    "120382": 67,
    "120434": 67,
    "1017": 67,
    "11428": 67,
    "5087": 67,
    "42202": 67,
    "66210": 67,
    "66306": 67,
    "66581": 67,
    "66844": 67,
    "8574": 100,
    "8518": 100,
    "119837": 100,
    "119889": 100,
    "119941": 100,
    "119993": 100,
    "120045": 100,
    "120097": 100,
    "120149": 100,
    "120201": 100,
    "120253": 100,
    "120305": 100,
    "120357": 100,
    "120409": 100,
    "120461": 100,
    "1281": 100,
    "5095": 100,
    "5231": 100,
    "42194": 100,
    "8558": 68,
    "8517": 68,
    "119811": 68,
    "119863": 68,
    "119915": 68,
    "119967": 68,
    "120019": 68,
    "120071": 68,
    "120123": 68,
    "120175": 68,
    "120227": 68,
    "120279": 68,
    "120331": 68,
    "120383": 68,
    "120435": 68,
    "5024": 68,
    "5598": 68,
    "5610": 68,
    "42195": 68,
    "8494": 101,
    "65349": 101,
    "8495": 101,
    "8519": 101,
    "119838": 101,
    "119890": 101,
    "119942": 101,
    "120046": 101,
    "120098": 101,
    "120150": 101,
    "120202": 101,
    "120254": 101,
    "120306": 101,
    "120358": 101,
    "120410": 101,
    "120462": 101,
    "43826": 101,
    "1213": 101,
    "8959": 69,
    "65317": 69,
    "8496": 69,
    "119812": 69,
    "119864": 69,
    "119916": 69,
    "120020": 69,
    "120072": 69,
    "120124": 69,
    "120176": 69,
    "120228": 69,
    "120280": 69,
    "120332": 69,
    "120384": 69,
    "120436": 69,
    "917": 69,
    "120492": 69,
    "120550": 69,
    "120608": 69,
    "120666": 69,
    "120724": 69,
    "11577": 69,
    "5036": 69,
    "42224": 69,
    "71846": 69,
    "71854": 69,
    "66182": 69,
    "119839": 102,
    "119891": 102,
    "119943": 102,
    "119995": 102,
    "120047": 102,
    "120099": 102,
    "120151": 102,
    "120203": 102,
    "120255": 102,
    "120307": 102,
    "120359": 102,
    "120411": 102,
    "120463": 102,
    "43829": 102,
    "42905": 102,
    "383": 102,
    "7837": 102,
    "1412": 102,
    "119315": 70,
    "8497": 70,
    "119813": 70,
    "119865": 70,
    "119917": 70,
    "120021": 70,
    "120073": 70,
    "120125": 70,
    "120177": 70,
    "120229": 70,
    "120281": 70,
    "120333": 70,
    "120385": 70,
    "120437": 70,
    "42904": 70,
    "988": 70,
    "120778": 70,
    "5556": 70,
    "42205": 70,
    "71874": 70,
    "71842": 70,
    "66183": 70,
    "66213": 70,
    "66853": 70,
    "65351": 103,
    "8458": 103,
    "119840": 103,
    "119892": 103,
    "119944": 103,
    "120048": 103,
    "120100": 103,
    "120152": 103,
    "120204": 103,
    "120256": 103,
    "120308": 103,
    "120360": 103,
    "120412": 103,
    "120464": 103,
    "609": 103,
    "7555": 103,
    "397": 103,
    "1409": 103,
    "119814": 71,
    "119866": 71,
    "119918": 71,
    "119970": 71,
    "120022": 71,
    "120074": 71,
    "120126": 71,
    "120178": 71,
    "120230": 71,
    "120282": 71,
    "120334": 71,
    "120386": 71,
    "120438": 71,
    "1292": 71,
    "5056": 71,
    "5107": 71,
    "42198": 71,
    "65352": 104,
    "8462": 104,
    "119841": 104,
    "119945": 104,
    "119997": 104,
    "120049": 104,
    "120101": 104,
    "120153": 104,
    "120205": 104,
    "120257": 104,
    "120309": 104,
    "120361": 104,
    "120413": 104,
    "120465": 104,
    "1211": 104,
    "1392": 104,
    "5058": 104,
    "65320": 72,
    "8459": 72,
    "8460": 72,
    "8461": 72,
    "119815": 72,
    "119867": 72,
    "119919": 72,
    "120023": 72,
    "120179": 72,
    "120231": 72,
    "120283": 72,
    "120335": 72,
    "120387": 72,
    "120439": 72,
    "919": 72,
    "120494": 72,
    "120552": 72,
    "120610": 72,
    "120668": 72,
    "120726": 72,
    "11406": 72,
    "5051": 72,
    "5500": 72,
    "42215": 72,
    "66255": 72,
    "731": 105,
    "9075": 105,
    "65353": 105,
    "8560": 105,
    "8505": 105,
    "8520": 105,
    "119842": 105,
    "119894": 105,
    "119946": 105,
    "119998": 105,
    "120050": 105,
    "120102": 105,
    "120154": 105,
    "120206": 105,
    "120258": 105,
    "120310": 105,
    "120362": 105,
    "120414": 105,
    "120466": 105,
    "120484": 105,
    "618": 105,
    "617": 105,
    "953": 105,
    "8126": 105,
    "890": 105,
    "120522": 105,
    "120580": 105,
    "120638": 105,
    "120696": 105,
    "120754": 105,
    "1110": 105,
    "42567": 105,
    "1231": 105,
    "43893": 105,
    "5029": 105,
    "71875": 105,
    "65354": 106,
    "8521": 106,
    "119843": 106,
    "119895": 106,
    "119947": 106,
    "119999": 106,
    "120051": 106,
    "120103": 106,
    "120155": 106,
    "120207": 106,
    "120259": 106,
    "120311": 106,
    "120363": 106,
    "120415": 106,
    "120467": 106,
    "1011": 106,
    "1112": 106,
    "65322": 74,
    "119817": 74,
    "119869": 74,
    "119921": 74,
    "119973": 74,
    "120025": 74,
    "120077": 74,
    "120129": 74,
    "120181": 74,
    "120233": 74,
    "120285": 74,
    "120337": 74,
    "120389": 74,
    "120441": 74,
    "42930": 74,
    "895": 74,
    "1032": 74,
    "5035": 74,
    "5261": 74,
    "42201": 74,
    "119844": 107,
    "119896": 107,
    "119948": 107,
    "120000": 107,
    "120052": 107,
    "120104": 107,
    "120156": 107,
    "120208": 107,
    "120260": 107,
    "120312": 107,
    "120364": 107,
    "120416": 107,
    "120468": 107,
    "8490": 75,
    "65323": 75,
    "119818": 75,
    "119870": 75,
    "119922": 75,
    "119974": 75,
    "120026": 75,
    "120078": 75,
    "120130": 75,
    "120182": 75,
    "120234": 75,
    "120286": 75,
    "120338": 75,
    "120390": 75,
    "120442": 75,
    "922": 75,
    "120497": 75,
    "120555": 75,
    "120613": 75,
    "120671": 75,
    "120729": 75,
    "11412": 75,
    "5094": 75,
    "5845": 75,
    "42199": 75,
    "66840": 75,
    "1472": 108,
    "8739": 73,
    "9213": 73,
    "65512": 73,
    "1633": 108,
    "1777": 73,
    "66336": 108,
    "125127": 108,
    "120783": 73,
    "120793": 73,
    "120803": 73,
    "120813": 73,
    "120823": 73,
    "130033": 73,
    "65321": 73,
    "8544": 73,
    "8464": 73,
    "8465": 73,
    "119816": 73,
    "119868": 73,
    "119920": 73,
    "120024": 73,
    "120128": 73,
    "120180": 73,
    "120232": 73,
    "120284": 73,
    "120336": 73,
    "120388": 73,
    "120440": 73,
    "65356": 108,
    "8572": 73,
    "8467": 108,
    "119845": 108,
    "119897": 108,
    "119949": 108,
    "120001": 108,
    "120053": 108,
    "120105": 73,
    "120157": 73,
    "120209": 73,
    "120261": 73,
    "120313": 73,
    "120365": 73,
    "120417": 73,
    "120469": 73,
    "448": 73,
    "120496": 73,
    "120554": 73,
    "120612": 73,
    "120670": 73,
    "120728": 73,
    "11410": 73,
    "1030": 73,
    "1216": 73,
    "1493": 108,
    "1503": 108,
    "1575": 108,
    "126464": 108,
    "126592": 108,
    "65166": 108,
    "65165": 108,
    "1994": 108,
    "11599": 73,
    "5825": 73,
    "42226": 73,
    "93992": 73,
    "66186": 124,
    "66313": 124,
    "119338": 76,
    "8556": 76,
    "8466": 76,
    "119819": 76,
    "119871": 76,
    "119923": 76,
    "120027": 76,
    "120079": 76,
    "120131": 76,
    "120183": 76,
    "120235": 76,
    "120287": 76,
    "120339": 76,
    "120391": 76,
    "120443": 76,
    "11472": 76,
    "5086": 76,
    "5290": 76,
    "42209": 76,
    "93974": 76,
    "71843": 76,
    "71858": 76,
    "66587": 76,
    "66854": 76,
    "65325": 77,
    "8559": 77,
    "8499": 77,
    "119820": 77,
    "119872": 77,
    "119924": 77,
    "120028": 77,
    "120080": 77,
    "120132": 77,
    "120184": 77,
    "120236": 77,
    "120288": 77,
    "120340": 77,
    "120392": 77,
    "120444": 77,
    "924": 77,
    "120499": 77,
    "120557": 77,
    "120615": 77,
    "120673": 77,
    "120731": 77,
    "1018": 77,
    "11416": 77,
    "5047": 77,
    "5616": 77,
    "5846": 77,
    "42207": 77,
    "66224": 77,
    "66321": 77,
    "119847": 110,
    "119899": 110,
    "119951": 110,
    "120003": 110,
    "120055": 110,
    "120107": 110,
    "120159": 110,
    "120211": 110,
    "120263": 110,
    "120315": 110,
    "120367": 110,
    "120419": 110,
    "120471": 110,
    "1400": 110,
    "1404": 110,
    "65326": 78,
    "8469": 78,
    "119821": 78,
    "119873": 78,
    "119925": 78,
    "119977": 78,
    "120029": 78,
    "120081": 78,
    "120185": 78,
    "120237": 78,
    "120289": 78,
    "120341": 78,
    "120393": 78,
    "120445": 78,
    "925": 78,
    "120500": 78,
    "120558": 78,
    "120616": 78,
    "120674": 78,
    "120732": 78,
    "11418": 78,
    "42208": 78,
    "66835": 78,
    "3074": 111,
    "3202": 111,
    "3330": 111,
    "3458": 111,
    "2406": 111,
    "2662": 111,
    "2790": 111,
    "3046": 111,
    "3174": 111,
    "3302": 111,
    "3430": 111,
    "3664": 111,
    "3792": 111,
    "4160": 111,
    "1637": 111,
    "1781": 111,
    "65359": 111,
    "8500": 111,
    "119848": 111,
    "119900": 111,
    "119952": 111,
    "120056": 111,
    "120108": 111,
    "120160": 111,
    "120212": 111,
    "120264": 111,
    "120316": 111,
    "120368": 111,
    "120420": 111,
    "120472": 111,
    "7439": 111,
    "7441": 111,
    "43837": 111,
    "959": 111,
    "120528": 111,
    "120586": 111,
    "120644": 111,
    "120702": 111,
    "120760": 111,
    "963": 111,
    "120532": 111,
    "120590": 111,
    "120648": 111,
    "120706": 111,
    "120764": 111,
    "11423": 111,
    "4351": 111,
    "1413": 111,
    "1505": 111,
    "1607": 111,
    "126500": 111,
    "126564": 111,
    "126596": 111,
    "65259": 111,
    "65260": 111,
    "65258": 111,
    "65257": 111,
    "1726": 111,
    "64428": 111,
    "64429": 111,
    "64427": 111,
    "64426": 111,
    "1729": 111,
    "64424": 111,
    "64425": 111,
    "64423": 111,
    "64422": 111,
    "1749": 111,
    "3360": 111,
    "4125": 111,
    "66794": 111,
    "71880": 111,
    "71895": 111,
    "66604": 111,
    "1984": 79,
    "2534": 79,
    "2918": 79,
    "12295": 79,
    "70864": 79,
    "71904": 79,
    "120782": 79,
    "120792": 79,
    "120802": 79,
    "120812": 79,
    "120822": 79,
    "130032": 79,
    "65327": 79,
    "119822": 79,
    "119874": 79,
    "119926": 79,
    "119978": 79,
    "120030": 79,
    "120082": 79,
    "120134": 79,
    "120186": 79,
    "120238": 79,
    "120290": 79,
    "120342": 79,
    "120394": 79,
    "120446": 79,
    "927": 79,
    "120502": 79,
    "120560": 79,
    "120618": 79,
    "120676": 79,
    "120734": 79,
    "11422": 79,
    "1365": 79,
    "11604": 79,
    "4816": 79,
    "2848": 79,
    "66754": 79,
    "42227": 79,
    "71861": 79,
    "66194": 79,
    "66219": 79,
    "66564": 79,
    "66838": 79,
    "9076": 112,
    "65360": 112,
    "119849": 112,
    "119901": 112,
    "119953": 112,
    "120005": 112,
    "120057": 112,
    "120109": 112,
    "120161": 112,
    "120213": 112,
    "120265": 112,
    "120317": 112,
    "120369": 112,
    "120421": 112,
    "120473": 112,
    "961": 112,
    "120530": 112,
    "120544": 112,
    "120588": 112,
    "120602": 112,
    "120646": 112,
    "120660": 112,
    "120704": 112,
    "120718": 112,
    "120762": 112,
    "120776": 112,
    "11427": 112,
    "65328": 80,
    "8473": 80,
    "119823": 80,
    "119875": 80,
    "119927": 80,
    "119979": 80,
    "120031": 80,
    "120083": 80,
    "120187": 80,
    "120239": 80,
    "120291": 80,
    "120343": 80,
    "120395": 80,
    "120447": 80,
    "929": 80,
    "120504": 80,
    "120562": 80,
    "120620": 80,
    "120678": 80,
    "120736": 80,
    "11426": 80,
    "5090": 80,
    "5229": 80,
    "42193": 80,
    "66197": 80,
    "119850": 113,
    "119902": 113,
    "119954": 113,
    "120006": 113,
    "120058": 113,
    "120110": 113,
    "120162": 113,
    "120214": 113,
    "120266": 113,
    "120318": 113,
    "120370": 113,
    "120422": 113,
    "120474": 113,
    "1307": 113,
    "1379": 113,
    "1382": 113,
    "8474": 81,
    "119824": 81,
    "119876": 81,
    "119928": 81,
    "119980": 81,
    "120032": 81,
    "120084": 81,
    "120188": 81,
    "120240": 81,
    "120292": 81,
    "120344": 81,
    "120396": 81,
    "120448": 81,
    "11605": 81,
    "119851": 114,
    "119903": 114,
    "119955": 114,
    "120007": 114,
    "120059": 114,
    "120111": 114,
    "120163": 114,
    "120215": 114,
    "120267": 114,
    "120319": 114,
    "120371": 114,
    "120423": 114,
    "120475": 114,
    "43847": 114,
    "43848": 114,
    "7462": 114,
    "11397": 114,
    "43905": 114,
    "119318": 82,
    "8475": 82,
    "8476": 82,
    "8477": 82,
    "119825": 82,
    "119877": 82,
    "119929": 82,
    "120033": 82,
    "120189": 82,
    "120241": 82,
    "120293": 82,
    "120345": 82,
    "120397": 82,
    "120449": 82,
    "422": 82,
    "5025": 82,
    "5074": 82,
    "66740": 82,
    "5511": 82,
    "42211": 82,
    "94005": 82,
    "65363": 115,
    "119852": 115,
    "119904": 115,
    "119956": 115,
    "120008": 115,
    "120060": 115,
    "120112": 115,
    "120164": 115,
    "120216": 115,
    "120268": 115,
    "120320": 115,
    "120372": 115,
    "120424": 115,
    "120476": 115,
    "42801": 115,
    "445": 115,
    "1109": 115,
    "43946": 115,
    "71873": 115,
    "66632": 115,
    "65331": 83,
    "119826": 83,
    "119878": 83,
    "119930": 83,
    "119982": 83,
    "120034": 83,
    "120086": 83,
    "120138": 83,
    "120190": 83,
    "120242": 83,
    "120294": 83,
    "120346": 83,
    "120398": 83,
    "120450": 83,
    "1029": 83,
    "1359": 83,
    "5077": 83,
    "5082": 83,
    "42210": 83,
    "94010": 83,
    "66198": 83,
    "66592": 83,
    "119853": 116,
    "119905": 116,
    "119957": 116,
    "120009": 116,
    "120061": 116,
    "120113": 116,
    "120165": 116,
    "120217": 116,
    "120269": 116,
    "120321": 116,
    "120373": 116,
    "120425": 116,
    "120477": 116,
    "8868": 84,
    "10201": 84,
    "128872": 84,
    "65332": 84,
    "119827": 84,
    "119879": 84,
    "119931": 84,
    "119983": 84,
    "120035": 84,
    "120087": 84,
    "120139": 84,
    "120191": 84,
    "120243": 84,
    "120295": 84,
    "120347": 84,
    "120399": 84,
    "120451": 84,
    "932": 84,
    "120507": 84,
    "120565": 84,
    "120623": 84,
    "120681": 84,
    "120739": 84,
    "11430": 84,
    "5026": 84,
    "42196": 84,
    "93962": 84,
    "71868": 84,
    "66199": 84,
    "66225": 84,
    "66325": 84,
    "119854": 117,
    "119906": 117,
    "119958": 117,
    "120010": 117,
    "120062": 117,
    "120114": 117,
    "120166": 117,
    "120218": 117,
    "120270": 117,
    "120322": 117,
    "120374": 117,
    "120426": 117,
    "120478": 117,
    "42911": 117,
    "7452": 117,
    "43854": 117,
    "43858": 117,
    "651": 117,
    "965": 117,
    "120534": 117,
    "120592": 117,
    "120650": 117,
    "120708": 117,
    "120766": 117,
    "1405": 117,
    "66806": 117,
    "71896": 117,
    "8746": 85,
    "8899": 85,
    "119828": 85,
    "119880": 85,
    "119932": 85,
    "119984": 85,
    "120036": 85,
    "120088": 85,
    "120140": 85,
    "120192": 85,
    "120244": 85,
    "120296": 85,
    "120348": 85,
    "120400": 85,
    "120452": 85,
    "1357": 85,
    "4608": 85,
    "66766": 85,
    "5196": 85,
    "42228": 85,
    "94018": 85,
    "71864": 85,
    "8744": 118,
    "8897": 118,
    "65366": 118,
    "8564": 118,
    "119855": 118,
    "119907": 118,
    "119959": 118,
    "120011": 118,
    "120063": 118,
    "120115": 118,
    "120167": 118,
    "120219": 118,
    "120271": 118,
    "120323": 118,
    "120375": 118,
    "120427": 118,
    "120479": 118,
    "7456": 118,
    "957": 118,
    "120526": 118,
    "120584": 118,
    "120642": 118,
    "120700": 118,
    "120758": 118,
    "1141": 118,
    "1496": 118,
    "71430": 118,
    "43945": 118,
    "71872": 118,
    "119309": 86,
    "1639": 86,
    "1783": 86,
    "8548": 86,
    "119829": 86,
    "119881": 86,
    "119933": 86,
    "119985": 86,
    "120037": 86,
    "120089": 86,
    "120141": 86,
    "120193": 86,
    "120245": 86,
    "120297": 86,
    "120349": 86,
    "120401": 86,
    "120453": 86,
    "1140": 86,
    "11576": 86,
    "5081": 86,
    "5167": 86,
    "42719": 86,
    "42214": 86,
    "93960": 86,
    "71840": 86,
    "66845": 86,
    "623": 119,
    "119856": 119,
    "119908": 119,
    "119960": 119,
    "120012": 119,
    "120064": 119,
    "120116": 119,
    "120168": 119,
    "120220": 119,
    "120272": 119,
    "120324": 119,
    "120376": 119,
    "120428": 119,
    "120480": 119,
    "7457": 119,
    "1121": 119,
    "1309": 119,
    "1377": 119,
    "71434": 119,
    "71438": 119,
    "71439": 119,
    "43907": 119,
    "71919": 87,
    "71910": 87,
    "119830": 87,
    "119882": 87,
    "119934": 87,
    "119986": 87,
    "120038": 87,
    "120090": 87,
    "120142": 87,
    "120194": 87,
    "120246": 87,
    "120298": 87,
    "120350": 87,
    "120402": 87,
    "120454": 87,
    "1308": 87,
    "5043": 87,
    "5076": 87,
    "42218": 87,
    "5742": 120,
    "10539": 120,
    "10540": 120,
    "10799": 120,
    "65368": 120,
    "8569": 120,
    "119857": 120,
    "119909": 120,
    "119961": 120,
    "120013": 120,
    "120065": 120,
    "120117": 120,
    "120169": 120,
    "120221": 120,
    "120273": 120,
    "120325": 120,
    "120377": 120,
    "120429": 120,
    "120481": 120,
    "5441": 120,
    "5501": 120,
    "5741": 88,
    "9587": 88,
    "66338": 88,
    "71916": 88,
    "65336": 88,
    "8553": 88,
    "119831": 88,
    "119883": 88,
    "119935": 88,
    "119987": 88,
    "120039": 88,
    "120091": 88,
    "120143": 88,
    "120195": 88,
    "120247": 88,
    "120299": 88,
    "120351": 88,
    "120403": 88,
    "120455": 88,
    "42931": 88,
    "935": 88,
    "120510": 88,
    "120568": 88,
    "120626": 88,
    "120684": 88,
    "120742": 88,
    "11436": 88,
    "11613": 88,
    "5815": 88,
    "42219": 88,
    "66192": 88,
    "66228": 88,
    "66327": 88,
    "66855": 88,
    "611": 121,
    "7564": 121,
    "65369": 121,
    "119858": 121,
    "119910": 121,
    "119962": 121,
    "120014": 121,
    "120066": 121,
    "120118": 121,
    "120170": 121,
    "120222": 121,
    "120274": 121,
    "120326": 121,
    "120378": 121,
    "120430": 121,
    "120482": 121,
    "655": 121,
    "7935": 121,
    "43866": 121,
    "947": 121,
    "8509": 121,
    "120516": 121,
    "120574": 121,
    "120632": 121,
    "120690": 121,
    "120748": 121,
    "1199": 121,
    "4327": 121,
    "71900": 121,
    "65337": 89,
    "119832": 89,
    "119884": 89,
    "119936": 89,
    "119988": 89,
    "120040": 89,
    "120092": 89,
    "120144": 89,
    "120196": 89,
    "120248": 89,
    "120300": 89,
    "120352": 89,
    "120404": 89,
    "120456": 89,
    "933": 89,
    "978": 89,
    "120508": 89,
    "120566": 89,
    "120624": 89,
    "120682": 89,
    "120740": 89,
    "11432": 89,
    "1198": 89,
    "5033": 89,
    "5053": 89,
    "42220": 89,
    "94019": 89,
    "71844": 89,
    "66226": 89,
    "119859": 122,
    "119911": 122,
    "119963": 122,
    "120015": 122,
    "120067": 122,
    "120119": 122,
    "120171": 122,
    "120223": 122,
    "120275": 122,
    "120327": 122,
    "120379": 122,
    "120431": 122,
    "120483": 122,
    "7458": 122,
    "43923": 122,
    "71876": 122,
    "66293": 90,
    "71909": 90,
    "65338": 90,
    "8484": 90,
    "8488": 90,
    "119833": 90,
    "119885": 90,
    "119937": 90,
    "119989": 90,
    "120041": 90,
    "120197": 90,
    "120249": 90,
    "120301": 90,
    "120353": 90,
    "120405": 90,
    "120457": 90,
    "918": 90,
    "120493": 90,
    "120551": 90,
    "120609": 90,
    "120667": 90,
    "120725": 90,
    "5059": 90,
    "42204": 90,
    "71849": 90,
    "65282": 34,
    "65284": 36,
    "65285": 37,
    "65286": 38,
    "65290": 42,
    "65291": 43,
    "65294": 46,
    "65295": 47,
    "65296": 48,
    "65297": 49,
    "65298": 50,
    "65299": 51,
    "65300": 52,
    "65301": 53,
    "65302": 54,
    "65303": 55,
    "65304": 56,
    "65305": 57,
    "65308": 60,
    "65309": 61,
    "65310": 62,
    "65312": 64,
    "65316": 68,
    "65318": 70,
    "65319": 71,
    "65324": 76,
    "65329": 81,
    "65330": 82,
    "65333": 85,
    "65334": 86,
    "65335": 87,
    "65343": 95,
    "65346": 98,
    "65348": 100,
    "65350": 102,
    "65355": 107,
    "65357": 109,
    "65358": 110,
    "65361": 113,
    "65362": 114,
    "65364": 116,
    "65365": 117,
    "65367": 119,
    "65370": 122,
    "65371": 123,
    "65373": 125,
    "160": 32,
    "8211": 45,
    "65374": 126,
    "65306": 58,
    "65281": 33,
    "8216": 96,
    "8217": 96,
    "8245": 96,
    "180": 96,
    "12494": 47,
    "1047": 51,
    "1073": 54,
    "1072": 97,
    "1040": 65,
    "1068": 98,
    "1042": 66,
    "1089": 99,
    "1057": 67,
    "1077": 101,
    "1045": 69,
    "1053": 72,
    "305": 105,
    "1050": 75,
    "921": 73,
    "1052": 77,
    "1086": 111,
    "1054": 79,
    "1009": 112,
    "1088": 112,
    "1056": 80,
    "1075": 114,
    "1058": 84,
    "215": 120,
    "1093": 120,
    "1061": 88,
    "1091": 121,
    "1059": 89,
    "65283": 35,
    "65288": 40,
    "65289": 41,
    "65292": 44,
    "65307": 59,
    "65311": 63,
}



File: spotdl/utils/archive.py

"""
Module for archiving sets of data
"""

from pathlib import Path
from typing import Set

__all__ = ["Archive"]


class Archive(Set):
    """
    Archive class.
    A file-persistable set.
    """

    def load(self, file: str) -> bool:
        """
        Imports the archive from the file.

        ### Arguments
        - file: the file name of the archive

        ### Returns
        - if the file exists
        """

        if not Path(file).exists():
            return False

        with open(file, "r", encoding="utf-8") as archive:
            self.clear()
            self.update([line.strip() for line in archive])

        return True

    def save(self, file: str) -> bool:
        """
        Exports the current archive to the file.

        ### Arguments
        - file: the file name of the archive
        """

        with open(file, "w", encoding="utf-8") as archive:
            for element in sorted(self):
                archive.write(f"{element}\n")

        return True



File: spotdl/utils/search.py

"""
Module for creating Song objects by interacting with Spotify API
or by parsing a query.

To use this module you must first initialize the SpotifyClient.
"""

import concurrent.futures
import json
import logging
import re
from pathlib import Path
from typing import Dict, List, Optional

import requests
from ytmusicapi import YTMusic

from spotdl.types.album import Album
from spotdl.types.artist import Artist
from spotdl.types.playlist import Playlist
from spotdl.types.saved import Saved
from spotdl.types.song import Song, SongList
from spotdl.utils.metadata import get_file_metadata
from spotdl.utils.spotify import SpotifyClient, SpotifyError

__all__ = [
    "QueryError",
    "get_search_results",
    "parse_query",
    "get_simple_songs",
    "reinit_song",
    "get_song_from_file_metadata",
    "gather_known_songs",
    "create_ytm_album",
    "create_ytm_playlist",
    "get_all_user_playlists",
    "get_user_saved_albums",
]

logger = logging.getLogger(__name__)
client = None  # pylint: disable=invalid-name


def get_ytm_client() -> YTMusic:
    """
    Lazily initialize the YTMusic client.

    ### Returns
    - the YTMusic client
    """

    global client  # pylint: disable=global-statement
    if client is None:
        client = YTMusic()

    return client


class QueryError(Exception):
    """
    Base class for all exceptions related to query.
    """


def get_search_results(search_term: str) -> List[Song]:
    """
    Creates a list of Song objects from a search term.

    ### Arguments
    - search_term: the search term to use

    ### Returns
    - a list of Song objects
    """

    return Song.list_from_search_term(search_term)


def parse_query(
    query: List[str],
    threads: int = 1,
    use_ytm_data: bool = False,
    playlist_numbering: bool = False,
    album_type=None,
) -> List[Song]:
    """
    Parse query and return list containing song object

    ### Arguments
    - query: List of strings containing query
    - threads: Number of threads to use

    ### Returns
    - List of song objects
    """

    songs: List[Song] = get_simple_songs(
        query,
        use_ytm_data=use_ytm_data,
        playlist_numbering=playlist_numbering,
        album_type=album_type,
    )

    results = []
    with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor:
        future_to_song = {executor.submit(reinit_song, song): song for song in songs}
        for future in concurrent.futures.as_completed(future_to_song):
            song = future_to_song[future]
            try:
                results.append(future.result())
            except Exception as exc:
                logger.error("%s generated an exception: %s", song.display_name, exc)

    return results


def get_simple_songs(
    query: List[str],
    use_ytm_data: bool = False,
    playlist_numbering: bool = False,
    albums_to_ignore=None,
    album_type=None,
) -> List[Song]:
    """
    Parse query and return list containing simple song objects

    ### Arguments
    - query: List of strings containing query

    ### Returns
    - List of simple song objects
    """

    songs: List[Song] = []
    lists: List[SongList] = []
    for request in query:
        logger.info("Processing query: %s", request)

        # Remove /intl-xxx/ from Spotify URLs with regex
        request = re.sub(r"\/intl-\w+\/", "/", request)

        if (
            (  # pylint: disable=too-many-boolean-expressions
                "watch?v=" in request
                or "youtu.be/" in request
                or "soundcloud.com/" in request
                or "bandcamp.com/" in request
            )
            and "open.spotify.com" in request
            and "track" in request
            and "|" in request
        ):
            split_urls = request.split("|")
            if (
                len(split_urls) <= 1
                or not (
                    "watch?v=" in split_urls[0]
                    or "youtu.be" in split_urls[0]
                    or "soundcloud.com/" in split_urls[0]
                    or "bandcamp.com/" in split_urls[0]
                )
                or "spotify" not in split_urls[1]
            ):
                raise QueryError(
                    'Incorrect format used, please use "YouTubeURL|SpotifyURL"'
                )

            songs.append(
                Song.from_missing_data(url=split_urls[1], download_url=split_urls[0])
            )
        elif "music.youtube.com/watch?v" in request:
            track_data = get_ytm_client().get_song(request.split("?v=", 1)[1])

            yt_song = Song.from_search_term(
                f"{track_data['videoDetails']['author']} - {track_data['videoDetails']['title']}"
            )

            if use_ytm_data:
                yt_song.name = track_data["title"]
                yt_song.artist = track_data["author"]
                yt_song.artists = [track_data["author"]]
                yt_song.duration = track_data["lengthSeconds"]

            yt_song.download_url = request
            songs.append(yt_song)
        elif (
            "youtube.com/playlist?list=" in request
            or "youtube.com/browse/VLPL" in request
        ):
            request = request.replace(
                "https://www.youtube.com/", "https://music.youtube.com/"
            )
            request = request.replace(
                "https://youtube.com/", "https://music.youtube.com/"
            )

            split_urls = request.split("|")
            if len(split_urls) == 1:
                if "?list=OLAK5uy_" in request:
                    lists.append(create_ytm_album(request, fetch_songs=False))
                elif "?list=PL" in request or "browse/VLPL" in request:
                    lists.append(create_ytm_playlist(request, fetch_songs=False))
            else:
                if ("spotify" not in split_urls[1]) or not any(
                    x in split_urls[0]
                    for x in ["?list=PL", "?list=OLAK5uy_", "browse/VLPL"]
                ):
                    raise QueryError(
                        'Incorrect format used, please use "YouTubeMusicURL|SpotifyURL". '
                        "Currently only supports YouTube Music playlists and albums."
                    )

                if ("open.spotify.com" in request and "album" in request) and (
                    "?list=OLAK5uy_" in request
                ):
                    ytm_list: SongList = create_ytm_album(
                        split_urls[0], fetch_songs=False
                    )
                    spot_list = Album.from_url(split_urls[1], fetch_songs=False)
                elif ("open.spotify.com" in request and "playlist" in request) and (
                    "?list=PL" in request or "browse/VLPL" in request
                ):
                    ytm_list = create_ytm_playlist(split_urls[0], fetch_songs=False)
                    spot_list = Playlist.from_url(split_urls[1], fetch_songs=False)
                else:
                    raise QueryError(
                        f"URLs are not of the same type, {split_urls[0]} is not "
                        f"the same type as {split_urls[1]}."
                    )

                if ytm_list.length != spot_list.length:
                    raise QueryError(
                        f"The YouTube Music ({ytm_list.length}) "
                        f"and Spotify ({spot_list.length}) lists have different lengths. "
                    )

                if use_ytm_data:
                    for index, song in enumerate(ytm_list.songs):
                        song.url = spot_list.songs[index].url

                    lists.append(ytm_list)
                else:
                    for index, song in enumerate(spot_list.songs):
                        song.download_url = ytm_list.songs[index].download_url

                    lists.append(spot_list)
        elif "open.spotify.com" in request and "track" in request:
            songs.append(Song.from_url(url=request))
        elif "https://spotify.link/" in request:
            resp = requests.head(request, allow_redirects=True, timeout=10)
            full_url = resp.url
            full_lists = get_simple_songs(
                [full_url],
                use_ytm_data=use_ytm_data,
                playlist_numbering=playlist_numbering,
                album_type=album_type,
            )
            songs.extend(full_lists)
        elif "open.spotify.com" in request and "playlist" in request:
            lists.append(Playlist.from_url(request, fetch_songs=False))
        elif "open.spotify.com" in request and "album" in request:
            lists.append(Album.from_url(request, fetch_songs=False))
        elif "open.spotify.com" in request and "artist" in request:
            lists.append(Artist.from_url(request, fetch_songs=False))
        elif "open.spotify.com" in request and "user" in request:
            lists.extend(get_all_user_playlists(request))
        elif "album:" in request:
            lists.append(Album.from_search_term(request, fetch_songs=False))
        elif "playlist:" in request:
            lists.append(Playlist.from_search_term(request, fetch_songs=False))
        elif "artist:" in request:
            lists.append(Artist.from_search_term(request, fetch_songs=False))
        elif request == "saved":
            lists.append(Saved.from_url(request, fetch_songs=False))
        elif request == "all-user-playlists":
            lists.extend(get_all_user_playlists())
        elif request == "all-user-followed-artists":
            lists.extend(get_user_followed_artists())
        elif request == "all-user-saved-albums":
            lists.extend(get_user_saved_albums())
        elif request == "all-saved-playlists":
            lists.extend(get_all_saved_playlists())
        elif request.endswith(".spotdl"):
            with open(request, "r", encoding="utf-8") as save_file:
                for track in json.load(save_file):
                    # Append to songs
                    songs.append(Song.from_dict(track))
        else:
            songs.append(Song.from_search_term(request))

    for song_list in lists:
        logger.info(
            "Found %s songs in %s (%s)",
            len(song_list.urls),
            song_list.name,
            song_list.__class__.__name__,
        )

        for index, song in enumerate(song_list.songs):
            song_data = song.json
            song_data["list_name"] = song_list.name
            song_data["list_url"] = song_list.url
            song_data["list_position"] = song.list_position
            song_data["list_length"] = song_list.length

            if playlist_numbering:
                song_data["track_number"] = song_data["list_position"]
                song_data["tracks_count"] = song_data["list_length"]
                song_data["album_name"] = song_data["list_name"]
                song_data["disc_number"] = 1
                song_data["disc_count"] = 1
                if isinstance(song_list, Playlist):
                    song_data["album_artist"] = song_list.author_name
                    song_data["cover_url"] = song_list.cover_url

            songs.append(Song.from_dict(song_data))

    # removing songs for --ignore-albums
    original_length = len(songs)
    if albums_to_ignore:
        songs = [
            song
            for song in songs
            if all(
                keyword not in song.album_name.lower() for keyword in albums_to_ignore
            )
        ]
        logger.info("Skipped %s songs (Ignored albums)", (original_length - len(songs)))

    if album_type:
        songs = [song for song in songs if song.album_type == album_type]

        logger.info(
            "Skipped %s songs for Album Type %s",
            (original_length - len(songs)),
            album_type,
        )

    logger.debug("Found %s songs in %s lists", len(songs), len(lists))

    return songs


def songs_from_albums(albums: List[str]):
    """
    Get all songs from albums ids/urls/etc.

    ### Arguments
    - albums: List of albums ids

    ### Returns
    - List of songs
    """

    songs: List[Song] = []
    for album_id in albums:
        album = Album.from_url(album_id, fetch_songs=False)

        songs.extend([Song.from_missing_data(**song.json) for song in album.songs])

    return songs


def get_all_user_playlists(user_url: str = "") -> List[Playlist]:
    """
    Get all user playlists.

    ### Args (optional)
    - user_url: Spotify user profile url.
        If a url is mentioned, get all public playlists of that specific user.

    ### Returns
    - List of all user playlists
    """

    spotify_client = SpotifyClient()
    if spotify_client.user_auth is False:  # type: ignore
        raise SpotifyError("You must be logged in to use this function")

    if user_url and not user_url.startswith("https://open.spotify.com/user/"):
        raise ValueError(f"Invalid user profile url: {user_url}")

    user_id = user_url.split("https://open.spotify.com/user/")[-1].replace("/", "")

    if user_id:
        user_playlists_response = spotify_client.user_playlists(user_id)
    else:
        user_playlists_response = spotify_client.current_user_playlists()
        user_resp = spotify_client.current_user()
        if user_resp is None:
            raise SpotifyError("Couldn't get user info")

        user_id = user_resp["id"]

    if user_playlists_response is None:
        raise SpotifyError("Couldn't get user playlists")

    user_playlists = user_playlists_response["items"]

    # Fetch all saved tracks
    while user_playlists_response and user_playlists_response["next"]:
        response = spotify_client.next(user_playlists_response)
        if response is None:
            break

        user_playlists_response = response
        user_playlists.extend(user_playlists_response["items"])

    return [
        Playlist.from_url(playlist["external_urls"]["spotify"], fetch_songs=False)
        for playlist in user_playlists
        if playlist["owner"]["id"] == user_id
    ]


def get_user_saved_albums() -> List[Album]:
    """
    Get all user saved albums

    ### Returns
    - List of all user saved albums
    """

    spotify_client = SpotifyClient()
    if spotify_client.user_auth is False:  # type: ignore
        raise SpotifyError("You must be logged in to use this function")

    user_saved_albums_response = spotify_client.current_user_saved_albums()
    if user_saved_albums_response is None:
        raise SpotifyError("Couldn't get user saved albums")

    user_saved_albums = user_saved_albums_response["items"]

    # Fetch all saved tracks
    while user_saved_albums_response and user_saved_albums_response["next"]:
        response = spotify_client.next(user_saved_albums_response)
        if response is None:
            break

        user_saved_albums_response = response
        user_saved_albums.extend(user_saved_albums_response["items"])

    return [
        Album.from_url(item["album"]["external_urls"]["spotify"], fetch_songs=False)
        for item in user_saved_albums
    ]


def get_user_followed_artists() -> List[Artist]:
    """
    Get all user playlists

    ### Returns
    - List of all user playlists
    """

    spotify_client = SpotifyClient()
    if spotify_client.user_auth is False:  # type: ignore
        raise SpotifyError("You must be logged in to use this function")

    user_followed_response = spotify_client.current_user_followed_artists()
    if user_followed_response is None:
        raise SpotifyError("Couldn't get user followed artists")

    user_followed_response = user_followed_response["artists"]
    user_followed = user_followed_response["items"]

    # Fetch all artists
    while user_followed_response and user_followed_response["next"]:
        response = spotify_client.next(user_followed_response)
        if response is None:
            break

        user_followed_response = response["artists"]
        user_followed.extend(user_followed_response["items"])

    return [
        Artist.from_url(followed_artist["external_urls"]["spotify"], fetch_songs=False)
        for followed_artist in user_followed
    ]


def get_all_saved_playlists() -> List[Playlist]:
    """
    Get all user playlists.

    ### Args (optional)
    - user_url: Spotify user profile url.
        If a url is mentioned, get all public playlists of that specific user.

    ### Returns
    - List of all user playlists
    """

    spotify_client = SpotifyClient()
    if spotify_client.user_auth is False:  # type: ignore
        raise SpotifyError("You must be logged in to use this function")

    user_playlists_response = spotify_client.current_user_playlists()

    if user_playlists_response is None:
        raise SpotifyError("Couldn't get user playlists")

    user_playlists = user_playlists_response["items"]
    user_id = user_playlists_response["href"].split("users/")[-1].split("/")[0]

    # Fetch all saved tracks
    while user_playlists_response and user_playlists_response["next"]:
        response = spotify_client.next(user_playlists_response)
        if response is None:
            break

        user_playlists_response = response
        user_playlists.extend(user_playlists_response["items"])

    return [
        Playlist.from_url(playlist["external_urls"]["spotify"], fetch_songs=False)
        for playlist in user_playlists
        if playlist["owner"]["id"] != user_id
    ]


def reinit_song(song: Song) -> Song:
    """
    Update song object with new data
    from Spotify

    ### Arguments
    - song: Song object

    ### Returns
    - Updated song object
    """

    data = song.json
    if data.get("url"):
        new_data = Song.from_url(data["url"]).json
    elif data.get("song_id"):
        new_data = Song.from_url(
            "https://open.spotify.com/track/" + data["song_id"]
        ).json
    elif data.get("name") and data.get("artist"):
        new_data = Song.from_search_term(f"{data['artist']} - {data['name']}").json
    else:
        raise QueryError("Song object is missing required data to be reinitialized")

    for key in Song.__dataclass_fields__:  # type: ignore # pylint: disable=E1101
        val = data.get(key)
        new_val = new_data.get(key)
        if new_val is not None and val is None:
            data[key] = new_val
        elif new_val is not None and val is not None:
            data[key] = val

    # return reinitialized song object
    return Song(**data)


def get_song_from_file_metadata(file: Path, id3_separator: str = "/") -> Optional[Song]:
    """
    Get song based on the file metadata or file name

    ### Arguments
    - file: Path to file

    ### Returns
    - Song object
    """

    file_metadata = get_file_metadata(file, id3_separator)

    if file_metadata is None:
        return None

    return Song.from_missing_data(**file_metadata)


def gather_known_songs(output: str, output_format: str) -> Dict[str, List[Path]]:
    """
    Gather all known songs from the output directory

    ### Arguments
    - output: Output path template
    - output_format: Output format

    ### Returns
    - Dictionary containing all known songs and their paths
    """

    # Get the base directory from the path template
    # Path("/Music/test/{artist}/{artists} - {title}.{output-ext}") -> "/Music/test"
    base_dir = output.split("{", 1)[0]
    paths = Path(base_dir).glob(f"**/*.{output_format}")

    known_songs: Dict[str, List[Path]] = {}
    for path in paths:
        # Try to get the song from the metadata
        song = get_song_from_file_metadata(path)

        # If the songs doesn't have metadata, try to get it from the filename
        if song is None or song.url is None:
            search_results = get_search_results(path.stem)
            if len(search_results) == 0:
                continue

            song = search_results[0]

        known_paths = known_songs.get(song.url)
        if known_paths is None:
            known_songs[song.url] = [path]
        else:
            known_songs[song.url].append(path)

    return known_songs


def create_ytm_album(url: str, fetch_songs: bool = True) -> Album:
    """
    Creates a list of Song objects from an album query.

    ### Arguments
    - album_query: the url of the album

    ### Returns
    - a list of Song objects
    """

    if "?list=" not in url or not url.startswith("https://music.youtube.com/"):
        raise ValueError(f"Invalid album url: {url}")

    browse_id = get_ytm_client().get_album_browse_id(
        url.split("?list=")[1].split("&")[0]
    )
    if browse_id is None:
        raise ValueError(f"Invalid album url: {url}")

    album = get_ytm_client().get_album(browse_id)

    if album is None:
        raise ValueError(f"Couldn't fetch album: {url}")

    metadata = {
        "artist": album["artists"][0]["name"],
        "name": album["title"],
        "url": url,
    }

    songs = []
    for track in album["tracks"]:
        artists = [artist["name"] for artist in track["artists"]]

        song = Song.from_missing_data(
            name=track["title"],
            artists=artists,
            artist=artists[0],
            album_name=metadata["name"],
            album_artist=metadata["artist"],
            duration=track["duration_seconds"],
            download_url=f"https://music.youtube.com/watch?v={track['videoId']}",
        )

        if fetch_songs:
            song = Song.from_search_term(f"{song.artist} - {song.name}")

        songs.append(song)

    return Album(**metadata, songs=songs, urls=[song.url for song in songs])


def create_ytm_playlist(url: str, fetch_songs: bool = True) -> Playlist:
    """
    Returns a playlist object from a youtube playlist url

    ### Arguments
    - url: the url of the playlist

    ### Returns
    - a Playlist object
    """

    if not ("?list=" in url or "/browse/VLPL" in url) or not url.startswith(
        "https://music.youtube.com/"
    ):
        raise ValueError(f"Invalid playlist url: {url}")

    if "/browse/VLPL" in url:
        playlist_id = url.split("/browse/")[1]
    else:
        playlist_id = url.split("?list=")[1]
    playlist = get_ytm_client().get_playlist(playlist_id, None)  # type: ignore

    if playlist is None:
        raise ValueError(f"Couldn't fetch playlist: {url}")

    metadata = {
        "description": (
            playlist["description"] if playlist["description"] is not None else ""
        ),
        "author_url": (
            f"https://music.youtube.com/channel/{playlist['author']['id']}"
            if playlist.get("author") is not None
            else "Missing author url"
        ),
        "author_name": (
            playlist["author"]["name"]
            if playlist.get("author") is not None
            else "Missing author"
        ),
        "cover_url": (
            playlist["thumbnails"][0]["url"]
            if playlist.get("thumbnails") is not None
            else "Missing thumbnails"
        ),
        "name": playlist["title"],
        "url": url,
    }

    songs = []
    for track in playlist["tracks"]:
        if track["videoId"] is None or track["isAvailable"] is False:
            continue

        song = Song.from_missing_data(
            name=track["title"],
            artists=(
                [artist["name"] for artist in track["artists"]]
                if track.get("artists") is not None
                else []
            ),
            artist=(
                track["artists"][0]["name"]
                if track.get("artists") is not None
                else None
            ),
            album_name=(
                track.get("album", {}).get("name")
                if track.get("album") is not None
                else None
            ),
            duration=track.get("duration_seconds"),
            explicit=track.get("isExplicit"),
            download_url=f"https://music.youtube.com/watch?v={track['videoId']}",
        )

        if fetch_songs:
            song = reinit_song(song)

        songs.append(song)

    return Playlist(**metadata, songs=songs, urls=[song.url for song in songs])



File: spotdl/utils/github.py

"""
Module for getting information about the current version of spotdl
from GitHub, downloading the latest version, and checking for updates.
"""

import logging
import os
import re
from typing import Tuple

import requests

from spotdl import _version

__all__ = [
    "REPO",
    "WEB_APP_URL",
    "get_status",
    "check_for_updates",
    "get_latest_version",
    "create_github_url",
    "download_github_dir",
]

REPO = "spotdl/spotify-downloader"
WEB_APP_URL = "https://github.com/spotdl/web-ui/tree/master/dist"


class RateLimitError(Exception):
    """
    Raised when the GitHub API rate limit is exceeded.
    """


def get_status(start: str, end: str, repo: str = REPO) -> Tuple[str, int, int]:
    """
    Get the status of a commit range.

    ### Arguments
    - start: the starting commit/branch/tag
    - end: the ending commit/branch/tag
    - repo: the repo to check (defaults to spotdl/spotify-downloader)

    ### Returns
    - tuple of (status, ahead_by, behind_by)
    """

    url = f"https://api.github.com/repos/{repo}/compare/{start}...{end}"

    response = requests.get(url, timeout=10)

    if response.status_code != 200:
        if response.status_code == 403:
            raise RateLimitError("GitHub API rate limit exceeded.")

        raise RuntimeError(
            f"Failed to get commit count. Status code: {response.status_code}"
        )

    data = response.json()

    return (
        data["status"],
        data["ahead_by"],
        data["behind_by"],
    )


def get_latest_version(repo: str = REPO) -> str:
    """
    Get the latest version of spotdl.

    ### Arguments
    - repo: the repo to check (defaults to spotdl/spotify-downloader)

    ### Returns
    - the latest version
    """

    url = f"https://api.github.com/repos/{repo}/releases/latest"

    response = requests.get(url, timeout=10)

    if response.status_code != 200:
        if response.status_code == 403:
            raise RateLimitError("GitHub API rate limit exceeded.")

        raise RuntimeError(
            f"Failed to get commit count. Status code: {response.status_code}"
        )

    data = response.json()

    return data["name"]  # returns "vx.x.x"


def check_for_updates(repo: str = REPO) -> str:
    """
    Check for updates to the current version.

    ### Arguments
    - repo: the repo to check (defaults to spotdl/spotify-downloader)

    ### Returns
    - the latest version
    """

    message = ""

    latest_version = get_latest_version(repo)
    current_version = f"v{_version.__version__}"  # returns "vx.x.x"

    if latest_version != current_version:
        message = f"New version available: {latest_version}.\n\n"
    else:
        message = "No updates available.\n\n"
    try:
        master = get_status(current_version, "master")
        dev = get_status(current_version, "dev")
    except RuntimeError:
        message = "Couldn't check for updates. You might be running a dev version.\n"
        message += "Current version: " + current_version + "\n"
        message += "Latest version: " + latest_version
        return message
    except RateLimitError:
        message = "GitHub API rate limit exceeded. Couldn't check for updates.\n"
        message += "Current version: " + current_version + "\n"
        message += "Latest version: " + latest_version + "\n"
        message += "Please try again later."
        return message

    for branch in ["master", "dev"]:
        name = branch.capitalize()
        if branch == "master":
            status, ahead_by, behind_by = master
        else:
            status, ahead_by, behind_by = dev

        if status == "behind":
            message += f"{name} is {status} by {behind_by} commits.\n"
        elif status == "ahead":
            message += f"{name} is {status} by {ahead_by} commits.\n"
        else:
            message += f"{name} is up to date.\n"

    return message


def create_github_url(url: str = WEB_APP_URL):
    """
    From the given url, produce a URL that is compatible with Github's REST API.

    ### Arguments
    - url: the url to convert

    ### Notes
    - Can handle blob or tree paths.
    """

    repo_only_url = re.compile(
        r"https:\/\/github\.com\/[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,38}\/[a-zA-Z0-9]+$"
    )
    re_branch = re.compile("/(tree|blob)/(.+?)/")

    # Check if the given url is a url to a GitHub repo. If it is, tell the
    # user to use 'git clone' to download it
    if re.match(repo_only_url, url):
        raise ValueError(
            "The given URL is a GitHub repo. Please use 'git clone' to download it."
        )

    # extract the branch name from the given url (e.g master)
    branch = re_branch.search(url)
    if branch:
        download_dirs = url[branch.end() :]
        api_url = (
            url[: branch.start()].replace("github.com", "api.github.com/repos", 1)
            + "/contents/"
            + download_dirs
            + "?ref="
            + branch.group(2)
        )
        return api_url

    raise ValueError("The given url is not a valid GitHub url")


def download_github_dir(
    repo_url: str = WEB_APP_URL, flatten: bool = False, output_dir: str = "./"
):
    """
    Downloads the files and directories in repo_url.

    ### Arguments
    - repo_url: the url to the repo to download
    - flatten: whether to flatten the directory structure
    - output_dir: the directory to download the files to

    ### Notes
    - Modification of https://github.com/sdushantha/gitdir/blob/master/gitdir/gitdir.py
    """

    # generate the url which returns the JSON data
    api_url = create_github_url(repo_url)

    dir_out = output_dir

    response = requests.get(api_url, timeout=10).json()

    if (
        isinstance(response, dict)
        and "message" in response.keys()
        and "rate limit" in response["message"]
    ):
        logging.error(
            "You have been rate limited by Github API attempting to update web client."
            "Proceeding with cached web client. Please try again later."
            "See https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting"
        )
        return None

    if not flatten:
        # make a directory with the name which is taken from
        # the actual repo
        os.makedirs(dir_out, exist_ok=True)

    if isinstance(response, dict) and response["type"] == "file":
        response = [response]

    for file in response:
        file_url = file["download_url"]

        if flatten:
            path = os.path.join(dir_out, os.path.basename(file["path"]))
        else:
            path = os.path.join(dir_out, file["path"])

        dirname = os.path.dirname(path)

        if dirname != "":
            os.makedirs(dirname, exist_ok=True)

        if file_url is not None:
            with open(path, "wb") as new_file:
                new_file.write(requests.get(file_url, timeout=10).content)
        else:
            download_github_dir(file["html_url"], flatten, output_dir)

    return None



File: spotdl/utils/spotify.py

"""
Module for interacting with Spotify API.
To use this module, you must have a Spotify API key and Spotify API secret.

```python
import spotdl.utils.spotify
spotify.Spotify.init(client_id, client_secret)
```
"""

import json
import logging
from typing import Dict, Optional

import requests
from spotipy import Spotify
from spotipy.cache_handler import CacheFileHandler, MemoryCacheHandler
from spotipy.oauth2 import SpotifyClientCredentials, SpotifyOAuth

from spotdl.utils.config import get_cache_path, get_spotify_cache_path

__all__ = [
    "SpotifyError",
    "SpotifyClient",
    "save_spotify_cache",
]

logger = logging.getLogger(__name__)


class SpotifyError(Exception):
    """
    Base class for all exceptions related to SpotifyClient.
    """


class Singleton(type):
    """
    Singleton metaclass for SpotifyClient. Ensures that SpotifyClient is not
    instantiated without prior initialization. Every other instantiation of
    SpotifyClient will return the same instance.
    """

    _instance = None

    def __call__(self):  # pylint: disable=bad-mcs-method-argument
        """
        Call method for Singleton metaclass.

        ### Returns
        - The instance of the SpotifyClient.
        """

        if self._instance is None:
            raise SpotifyError(
                "Spotify client not created. Call SpotifyClient.init"
                "(client_id, client_secret, user_auth, cache_path, no_cache, open_browser) first."
            )
        return self._instance

    def init(  # pylint: disable=bad-mcs-method-argument
        self,
        client_id: str,
        client_secret: str,
        user_auth: bool = False,
        no_cache: bool = False,
        headless: bool = False,
        max_retries: int = 3,
        use_cache_file: bool = False,
        auth_token: Optional[str] = None,
        cache_path: Optional[str] = None,
    ) -> "Singleton":
        """
        Initializes the SpotifyClient.

        ### Arguments
        - client_id: The client ID of the application.
        - client_secret: The client secret of the application.
        - auth_token: The access token to use.
        - user_auth: Whether or not to use user authentication.
        - cache_path: The path to the cache file.
        - no_cache: Whether or not to use the cache.
        - open_browser: Whether or not to open the browser.

        ### Returns
        - The instance of the SpotifyClient.
        """

        # check if initialization has been completed, if yes, raise an Exception
        if isinstance(self._instance, self):
            raise SpotifyError("A spotify client has already been initialized")

        credential_manager = None

        cache_handler = (
            CacheFileHandler(cache_path or get_cache_path())
            if not no_cache
            else MemoryCacheHandler()
        )
        # Use SpotifyOAuth as auth manager
        if user_auth:
            credential_manager = SpotifyOAuth(
                client_id=client_id,
                client_secret=client_secret,
                redirect_uri="http://127.0.0.1:9900/",
                scope="user-library-read,user-follow-read,playlist-read-private",
                cache_handler=cache_handler,
                open_browser=not headless,
            )
        # Use SpotifyClientCredentials as auth manager
        else:
            credential_manager = SpotifyClientCredentials(
                client_id=client_id,
                client_secret=client_secret,
                cache_handler=cache_handler,
            )
        if auth_token is not None:
            credential_manager = None

        self.user_auth = user_auth
        self.no_cache = no_cache
        self.max_retries = max_retries
        self.use_cache_file = use_cache_file

        # Create instance
        self._instance = super().__call__(
            auth=auth_token,
            auth_manager=credential_manager,
            status_forcelist=(429, 500, 502, 503, 504, 404),
        )

        # Return instance
        return self._instance


class SpotifyClient(Spotify, metaclass=Singleton):
    """
    This is the Spotify client meant to be used in the app.
    Has to be initialized first by calling
    `SpotifyClient.init(client_id, client_secret, user_auth, cache_path, no_cache, open_browser)`.
    """

    _initialized = False
    cache: Dict[str, Optional[Dict]] = {}

    def __init__(self, *args, **kwargs):
        """
        Initializes the SpotifyClient.

        ### Arguments
        - auth: The access token to use.
        - auth_manager: The auth manager to use.
        """

        super().__init__(*args, **kwargs)
        self._initialized = True

        use_cache_file: bool = self.use_cache_file  # type: ignore # pylint: disable=E1101
        cache_file_loc = get_spotify_cache_path()

        if use_cache_file and cache_file_loc.exists():
            with open(cache_file_loc, "r", encoding="utf-8") as cache_file:
                self.cache = json.load(cache_file)
        elif use_cache_file:
            with open(cache_file_loc, "w", encoding="utf-8") as cache_file:
                json.dump(self.cache, cache_file)

    def _get(self, url, args=None, payload=None, **kwargs):
        """
        Overrides the get method of the SpotifyClient.
        Allows us to cache requests
        """

        use_cache = not self.no_cache  # type: ignore # pylint: disable=E1101

        if args:
            kwargs.update(args)

        cache_key = None
        if use_cache:
            key_obj = dict(kwargs)
            key_obj["url"] = url
            key_obj["data"] = json.dumps(payload)
            cache_key = json.dumps(key_obj)
            if cache_key is None:
                cache_key = url
            if self.cache.get(cache_key) is not None:
                return self.cache[cache_key]

        # Wrap in a try-except and retry up to `retries` times.
        response = None
        retries = self.max_retries  # type: ignore # pylint: disable=E1101
        while response is None:
            try:
                response = self._internal_call("GET", url, payload, kwargs)
            except (requests.exceptions.Timeout, requests.ConnectionError) as exc:
                retries -= 1
                if retries <= 0:
                    raise exc

        if use_cache and cache_key is not None:
            self.cache[cache_key] = response

        return response


def save_spotify_cache(cache: Dict[str, Optional[Dict]]):
    """
    Saves the Spotify cache to a file.

    ### Arguments
    - cache: The cache to save.
    """

    cache_file_loc = get_spotify_cache_path()

    logger.debug("Saving Spotify cache to %s", cache_file_loc)

    # Only cache tracks
    cache = {
        key: value
        for key, value in cache.items()
        if value is not None and "tracks/" in key
    }

    with open(cache_file_loc, "w", encoding="utf-8") as cache_file:
        json.dump(cache, cache_file)



File: spotdl/utils/lrc.py

"""
LRC related functions
"""

import logging
import re
from pathlib import Path

from syncedlyrics import search as syncedlyrics_search
from syncedlyrics.utils import Lyrics, TargetType, has_translation

from spotdl.types.song import Song

logger = logging.getLogger(__name__)

__all__ = ["generate_lrc", "remomve_lrc"]


def generate_lrc(song: Song, output_file: Path):
    """
    Generates an LRC file for the current song

    ### Arguments
    - song: Song object
    - output_file: Path to the output file
    """

    if song.lyrics and has_translation(song.lyrics):
        lrc_data = song.lyrics
    else:
        try:
            lrc_data = syncedlyrics_search(song.display_name)
        except Exception:
            lrc_data = None

    if lrc_data:
        Lyrics(lrc_data).save_lrc_file(
            str(output_file.with_suffix(".lrc")), TargetType.PREFER_SYNCED
        )
        logger.debug("Saved lrc file for %s", song.display_name)
    else:
        logger.debug("No lrc file found for %s", song.display_name)


def remomve_lrc(lyrics: str) -> str:
    """
    Removes lrc tags from lyrics

    ### Arguments
    - lyrics: Lyrics string

    ### Returns
    - Lyrics string without lrc tags
    """

    return re.sub(r"\[.*?\]", "", lyrics)



File: spotdl/download/downloader.py

"""
Downloader module, this is where all the downloading pre/post processing happens etc.
"""

import asyncio
import datetime
import json
import logging
import re
import shutil
import sys
import traceback
from argparse import Namespace
from pathlib import Path
from typing import Dict, List, Optional, Tuple, Type, Union

from yt_dlp.postprocessor.modify_chapters import ModifyChaptersPP
from yt_dlp.postprocessor.sponsorblock import SponsorBlockPP

from spotdl.download.progress_handler import ProgressHandler
from spotdl.providers.audio import (
    AudioProvider,
    BandCamp,
    Piped,
    SliderKZ,
    SoundCloud,
    YouTube,
    YouTubeMusic,
)
from spotdl.providers.lyrics import AzLyrics, Genius, LyricsProvider, MusixMatch, Synced
from spotdl.types.options import DownloaderOptionalOptions, DownloaderOptions
from spotdl.types.song import Song
from spotdl.utils.archive import Archive
from spotdl.utils.config import (
    DOWNLOADER_OPTIONS,
    GlobalConfig,
    create_settings_type,
    get_errors_path,
    get_temp_path,
    modernize_settings,
)
from spotdl.utils.ffmpeg import FFmpegError, convert, get_ffmpeg_path
from spotdl.utils.formatter import create_file_name
from spotdl.utils.lrc import generate_lrc
from spotdl.utils.m3u import gen_m3u_files
from spotdl.utils.metadata import MetadataError, embed_metadata
from spotdl.utils.search import gather_known_songs, reinit_song, songs_from_albums

__all__ = [
    "AUDIO_PROVIDERS",
    "LYRICS_PROVIDERS",
    "Downloader",
    "DownloaderError",
    "SPONSOR_BLOCK_CATEGORIES",
]

AUDIO_PROVIDERS: Dict[str, Type[AudioProvider]] = {
    "youtube": YouTube,
    "youtube-music": YouTubeMusic,
    "slider-kz": SliderKZ,
    "soundcloud": SoundCloud,
    "bandcamp": BandCamp,
    "piped": Piped,
}

LYRICS_PROVIDERS: Dict[str, Type[LyricsProvider]] = {
    "genius": Genius,
    "musixmatch": MusixMatch,
    "azlyrics": AzLyrics,
    "synced": Synced,
}

SPONSOR_BLOCK_CATEGORIES = {
    "sponsor": "Sponsor",
    "intro": "Intermission/Intro Animation",
    "outro": "Endcards/Credits",
    "selfpromo": "Unpaid/Self Promotion",
    "preview": "Preview/Recap",
    "filler": "Filler Tangent",
    "interaction": "Interaction Reminder",
    "music_offtopic": "Non-Music Section",
}


logger = logging.getLogger(__name__)


class DownloaderError(Exception):
    """
    Base class for all exceptions related to downloaders.
    """


class Downloader:
    """
    Downloader class, this is where all the downloading pre/post processing happens etc.
    It handles the downloading/moving songs, multithreading, metadata embedding etc.
    """

    def __init__(
        self,
        settings: Optional[Union[DownloaderOptionalOptions, DownloaderOptions]] = None,
        loop: Optional[asyncio.AbstractEventLoop] = None,
    ):
        """
        Initialize the Downloader class.

        ### Arguments
        - settings: The settings to use.
        - loop: The event loop to use.

        ### Notes
        - `search-query` uses the same format as `output`.
        - if `audio_provider` or `lyrics_provider` is a list, then if no match is found,
            the next provider in the list will be used.
        """

        if settings is None:
            settings = {}

        # Create settings dictionary, fill in missing values with defaults
        # from spotdl.types.options.DOWNLOADER_OPTIONS
        self.settings: DownloaderOptions = DownloaderOptions(
            **create_settings_type(
                Namespace(config=False), dict(settings), DOWNLOADER_OPTIONS
            )  # type: ignore
        )

        # Handle deprecated values in config file
        modernize_settings(self.settings)
        logger.debug("Downloader settings: %s", self.settings)

        # If no audio providers specified, raise an error
        if len(self.settings["audio_providers"]) == 0:
            raise DownloaderError(
                "No audio providers specified. Please specify at least one."
            )

        # If ffmpeg is the default value and it's not installed
        # try to use the spotdl's ffmpeg
        self.ffmpeg = self.settings["ffmpeg"]
        if self.ffmpeg == "ffmpeg" and shutil.which("ffmpeg") is None:
            ffmpeg_exec = get_ffmpeg_path()
            if ffmpeg_exec is None:
                raise DownloaderError("ffmpeg is not installed")

            self.ffmpeg = str(ffmpeg_exec.absolute())

        logger.debug("FFmpeg path: %s", self.ffmpeg)

        self.loop = loop or (
            asyncio.new_event_loop()
            if sys.platform != "win32"
            else asyncio.ProactorEventLoop()  # type: ignore
        )

        if loop is None:
            asyncio.set_event_loop(self.loop)

        # semaphore is required to limit concurrent asyncio executions
        self.semaphore = asyncio.Semaphore(self.settings["threads"])

        self.progress_handler = ProgressHandler(self.settings["simple_tui"])

        # Gather already present songs
        self.scan_formats = self.settings["detect_formats"] or [self.settings["format"]]
        self.known_songs: Dict[str, List[Path]] = {}
        if self.settings["scan_for_songs"]:
            logger.info("Scanning for known songs, this might take a while...")
            for scan_format in self.scan_formats:
                logger.debug("Scanning for %s files", scan_format)

                found_files = gather_known_songs(self.settings["output"], scan_format)

                logger.debug("Found %s %s files", len(found_files), scan_format)

                for song_url, song_paths in found_files.items():
                    known_paths = self.known_songs.get(song_url)
                    if known_paths is None:
                        self.known_songs[song_url] = song_paths
                    else:
                        self.known_songs[song_url].extend(song_paths)

        logger.debug("Found %s known songs", len(self.known_songs))

        # Initialize lyrics providers
        self.lyrics_providers: List[LyricsProvider] = []
        for lyrics_provider in self.settings["lyrics_providers"]:
            lyrics_class = LYRICS_PROVIDERS.get(lyrics_provider)
            if lyrics_class is None:
                raise DownloaderError(f"Invalid lyrics provider: {lyrics_provider}")
            if lyrics_provider == "genius":
                access_token = self.settings.get("genius_token")
                if not access_token:
                    raise DownloaderError("Genius token not found in settings")
                self.lyrics_providers.append(Genius(access_token))
            else:
                self.lyrics_providers.append(lyrics_class())

        # Initialize audio providers
        self.audio_providers: List[AudioProvider] = []
        for audio_provider in self.settings["audio_providers"]:
            audio_class = AUDIO_PROVIDERS.get(audio_provider)
            if audio_class is None:
                raise DownloaderError(f"Invalid audio provider: {audio_provider}")

            self.audio_providers.append(
                audio_class(
                    output_format=self.settings["format"],
                    cookie_file=self.settings["cookie_file"],
                    search_query=self.settings["search_query"],
                    filter_results=self.settings["filter_results"],
                    yt_dlp_args=self.settings["yt_dlp_args"],
                )
            )

        # Initialize list of errors
        self.errors: List[str] = []

        # Initialize proxy server
        proxy = self.settings["proxy"]
        proxies = None
        if proxy:
            if not re.match(
                pattern=r"^(http|https):\/\/(?:(\w+)(?::(\w+))?@)?((?:\d{1,3})(?:\.\d{1,3}){3})(?::(\d{1,5}))?$",  # pylint: disable=C0301
                string=proxy,
            ):
                raise DownloaderError(f"Invalid proxy server: {proxy}")
            proxies = {"http": proxy, "https": proxy}
            logger.info("Setting proxy server: %s", proxy)

        GlobalConfig.set_parameter("proxies", proxies)

        # Initialize archive
        self.url_archive = Archive()
        if self.settings["archive"]:
            self.url_archive.load(self.settings["archive"])

        logger.debug("Archive: %d urls", len(self.url_archive))

        logger.debug("Downloader initialized")

    def download_song(self, song: Song) -> Tuple[Song, Optional[Path]]:
        """
        Download a single song.

        ### Arguments
        - song: The song to download.

        ### Returns
        - tuple with the song and the path to the downloaded file if successful.
        """

        self.progress_handler.set_song_count(1)

        results = self.download_multiple_songs([song])

        return results[0]

    def download_multiple_songs(
        self, songs: List[Song]
    ) -> List[Tuple[Song, Optional[Path]]]:
        """
        Download multiple songs to the temp directory.

        ### Arguments
        - songs: The songs to download.

        ### Returns
        - list of tuples with the song and the path to the downloaded file if successful.
        """

        if self.settings["fetch_albums"]:
            albums = set(song.album_id for song in songs if song.album_id is not None)
            logger.info(
                "Fetching %d album%s", len(albums), "s" if len(albums) > 1 else ""
            )

            songs.extend(songs_from_albums(list(albums)))

            # Remove duplicates
            return_obj = {}
            for song in songs:
                return_obj[song.url] = song

            songs = list(return_obj.values())

        logger.debug("Downloading %d songs", len(songs))

        if self.settings["archive"]:
            songs = [song for song in songs if song.url not in self.url_archive]
            logger.debug("Filtered %d songs with archive", len(songs))

        self.progress_handler.set_song_count(len(songs))

        # Create tasks list
        tasks = [self.pool_download(song) for song in songs]

        # Call all task asynchronously, and wait until all are finished
        results = list(self.loop.run_until_complete(asyncio.gather(*tasks)))

        # Print errors
        if self.settings["print_errors"]:
            for error in self.errors:
                logger.error(error)

        if self.settings["save_errors"]:
            with open(
                self.settings["save_errors"], "w", encoding="utf-8"
            ) as error_file:
                error_file.write("\n".join(self.errors))

            logger.info("Saved errors to %s", self.settings["save_errors"])

        # Save archive
        if self.settings["archive"]:
            for result in results:
                if result[1] or self.settings["add_unavailable"]:
                    self.url_archive.add(result[0].url)

            self.url_archive.save(self.settings["archive"])
            logger.info(
                "Saved archive with %d urls to %s",
                len(self.url_archive),
                self.settings["archive"],
            )

        # Create m3u playlist
        if self.settings["m3u"]:
            song_list = [
                song
                for song, path in results
                if path or self.settings["add_unavailable"]
            ]

            gen_m3u_files(
                song_list,
                self.settings["m3u"],
                self.settings["output"],
                self.settings["format"],
                self.settings["restrict"],
                False,
                self.settings["detect_formats"],
            )

        # Save results to a file
        if self.settings["save_file"]:
            with open(self.settings["save_file"], "w", encoding="utf-8") as save_file:
                json.dump([song.json for song, _ in results], save_file, indent=4)

            logger.info("Saved results to %s", self.settings["save_file"])

        return results

    async def pool_download(self, song: Song) -> Tuple[Song, Optional[Path]]:
        """
        Run asynchronous task in a pool to make sure that all processes.

        ### Arguments
        - song: The song to download.

        ### Returns
        - tuple with the song and the path to the downloaded file if successful.

        ### Notes
        - This method calls `self.search_and_download` in a new thread.
        """

        # tasks that cannot acquire semaphore will wait here until it's free
        # only certain amount of tasks can acquire the semaphore at the same time
        async with self.semaphore:
            return await self.loop.run_in_executor(None, self.search_and_download, song)

    def search(self, song: Song) -> str:
        """
        Search for a song using all available providers.

        ### Arguments
        - song: The song to search for.

        ### Returns
        - tuple with download url and audio provider if successful.
        """

        for audio_provider in self.audio_providers:
            url = audio_provider.search(song, self.settings["only_verified_results"])
            if url:
                return url

            logger.debug("%s failed to find %s", audio_provider.name, song.display_name)

        raise LookupError(f"No results found for song: {song.display_name}")

    def search_lyrics(self, song: Song) -> Optional[str]:
        """
        Search for lyrics using all available providers.

        ### Arguments
        - song: The song to search for.

        ### Returns
        - lyrics if successful else None.
        """

        for lyrics_provider in self.lyrics_providers:
            lyrics = lyrics_provider.get_lyrics(song.name, song.artists)
            if lyrics:
                logger.debug(
                    "Found lyrics for %s on %s", song.display_name, lyrics_provider.name
                )

                return lyrics

            logger.debug(
                "%s failed to find lyrics for %s",
                lyrics_provider.name,
                song.display_name,
            )

        return None

    def search_and_download(  # pylint: disable=R0911
        self, song: Song
    ) -> Tuple[Song, Optional[Path]]:
        """
        Search for the song and download it.

        ### Arguments
        - song: The song to download.

        ### Returns
        - tuple with the song and the path to the downloaded file if successful.

        ### Notes
        - This function is synchronous.
        """

        # Check if song has name/artist and url/song_id
        if not (song.name and (song.artists or song.artist)) and not (
            song.url or song.song_id
        ):
            logger.error("Song is missing required fields: %s", song.display_name)
            self.errors.append(f"Song is missing required fields: {song.display_name}")

            return song, None

        reinitialized = False
        try:
            # Create the output file path
            output_file = create_file_name(
                song=song,
                template=self.settings["output"],
                file_extension=self.settings["format"],
                restrict=self.settings["restrict"],
                file_name_length=self.settings["max_filename_length"],
            )
        except Exception:
            song = reinit_song(song)

            output_file = create_file_name(
                song=song,
                template=self.settings["output"],
                file_extension=self.settings["format"],
                restrict=self.settings["restrict"],
                file_name_length=self.settings["max_filename_length"],
            )

            reinitialized = True

        if song.explicit is True and self.settings["skip_explicit"] is True:
            logger.info("Skipping explicit song: %s", song.display_name)
            return song, None

        # Initialize the progress tracker
        display_progress_tracker = self.progress_handler.get_new_tracker(song)

        try:
            # Create the temp folder path
            temp_folder = get_temp_path()

            # Check if there is an already existing song file, with the same spotify URL in its
            # metadata, but saved under a different name. If so, save its path.
            dup_song_paths: List[Path] = self.known_songs.get(song.url, [])

            # Remove files from the list that have the same path as the output file
            dup_song_paths = [
                dup_song_path
                for dup_song_path in dup_song_paths
                if (dup_song_path.absolute() != output_file.absolute())
                and dup_song_path.exists()
            ]

            file_exists = output_file.exists() or dup_song_paths
            if not self.settings["scan_for_songs"]:
                for file_extension in self.scan_formats:
                    ext_path = output_file.with_suffix(f".{file_extension}")
                    if ext_path.exists():
                        dup_song_paths.append(ext_path)

            if dup_song_paths:
                logger.debug(
                    "Found duplicate songs for %s at %s",
                    song.display_name,
                    ", ".join(
                        [f"'{str(dup_song_path)}'" for dup_song_path in dup_song_paths]
                    ),
                )

            # If the file already exists and we don't want to overwrite it,
            # we can skip the download
            if (  # pylint: disable=R1705
                Path(str(output_file.absolute()) + ".skip").exists()
                and self.settings["respect_skip_file"]
            ):
                logger.info(
                    "Skipping %s (skip file found) %s",
                    song.display_name,
                    "",
                )

                return song, output_file if output_file.exists() else None

            elif file_exists and self.settings["overwrite"] == "skip":
                logger.info(
                    "Skipping %s (file already exists) %s",
                    song.display_name,
                    "(duplicate)" if dup_song_paths else "",
                )

                display_progress_tracker.notify_download_skip()
                return song, output_file

            # Check if we have all the metadata
            # and that the song object is not a placeholder
            # If it's None extract the current metadata
            # And reinitialize the song object
            # Force song reinitialization if we are fetching albums
            # they have most metadata but not all
            if (
                (song.name is None and song.url)
                or (self.settings["fetch_albums"] and reinitialized is False)
                or None
                in [
                    song.genres,
                    song.disc_count,
                    song.tracks_count,
                    song.track_number,
                    song.album_id,
                    song.album_artist,
                ]
            ):
                song = reinit_song(song)
                reinitialized = True

            # Don't skip if the file exists and overwrite is set to force
            if file_exists and self.settings["overwrite"] == "force":
                logger.info(
                    "Overwriting %s %s",
                    song.display_name,
                    " (duplicate)" if dup_song_paths else "",
                )

                # If the duplicate song path is not None, we can delete the old file
                for dup_song_path in dup_song_paths:
                    try:
                        logger.info("Removing duplicate file: %s", dup_song_path)

                        dup_song_path.unlink()
                    except (PermissionError, OSError) as exc:
                        logger.debug(
                            "Could not remove duplicate file: %s, error: %s",
                            dup_song_path,
                            exc,
                        )

            # Find song lyrics and add them to the song object
            try:
                lyrics = self.search_lyrics(song)
                if lyrics is None:
                    logger.debug(
                        "No lyrics found for %s, lyrics providers: %s",
                        song.display_name,
                        ", ".join(
                            [lprovider.name for lprovider in self.lyrics_providers]
                        ),
                    )
                else:
                    song.lyrics = lyrics
            except Exception as exc:
                logger.debug("Could not search for lyrics: %s", exc)

            # If the file already exists and we want to overwrite the metadata,
            # we can skip the download
            if file_exists and self.settings["overwrite"] == "metadata":
                most_recent_duplicate: Optional[Path] = None
                if dup_song_paths:
                    # Get the most recent duplicate song path and remove the rest
                    most_recent_duplicate = max(
                        dup_song_paths,
                        key=lambda dup_song_path: dup_song_path.stat().st_mtime
                        and dup_song_path.suffix == output_file.suffix,
                    )

                    # Remove the rest of the duplicate song paths
                    for old_song_path in dup_song_paths:
                        if most_recent_duplicate == old_song_path:
                            continue

                        try:
                            logger.info("Removing duplicate file: %s", old_song_path)
                            old_song_path.unlink()
                        except (PermissionError, OSError) as exc:
                            logger.debug(
                                "Could not remove duplicate file: %s, error: %s",
                                old_song_path,
                                exc,
                            )

                    # Move the old file to the new location
                    if (
                        most_recent_duplicate
                        and most_recent_duplicate.suffix == output_file.suffix
                    ):
                        most_recent_duplicate.replace(
                            output_file.with_suffix(f".{self.settings['format']}")
                        )

                if (
                    most_recent_duplicate
                    and most_recent_duplicate.suffix != output_file.suffix
                ):
                    logger.info(
                        "Could not move duplicate file: %s, different file extension",
                        most_recent_duplicate,
                    )

                    display_progress_tracker.notify_complete()

                    return song, None

                # Update the metadata
                embed_metadata(
                    output_file=output_file,
                    song=song,
                    skip_album_art=self.settings["skip_album_art"],
                )

                logger.info(
                    f"Updated metadata for {song.display_name}"
                    f", moved to new location: {output_file}"
                    if most_recent_duplicate
                    else ""
                )

                display_progress_tracker.notify_complete()

                return song, output_file

            # Create the output directory if it doesn't exist
            output_file.parent.mkdir(parents=True, exist_ok=True)
            if song.download_url is None:
                download_url = self.search(song)
            else:
                download_url = song.download_url

            # Initialize audio downloader
            audio_downloader: Union[AudioProvider, Piped]
            if self.settings["audio_providers"][0] == "piped":
                audio_downloader = Piped(
                    output_format=self.settings["format"],
                    cookie_file=self.settings["cookie_file"],
                    search_query=self.settings["search_query"],
                    filter_results=self.settings["filter_results"],
                    yt_dlp_args=self.settings["yt_dlp_args"],
                )
            else:
                audio_downloader = AudioProvider(
                    output_format=self.settings["format"],
                    cookie_file=self.settings["cookie_file"],
                    search_query=self.settings["search_query"],
                    filter_results=self.settings["filter_results"],
                    yt_dlp_args=self.settings["yt_dlp_args"],
                )

            logger.debug("Downloading %s using %s", song.display_name, download_url)

            # Add progress hook to the audio provider
            audio_downloader.audio_handler.add_progress_hook(
                display_progress_tracker.yt_dlp_progress_hook
            )

            download_info = audio_downloader.get_download_metadata(
                download_url, download=True
            )

            temp_file = Path(
                temp_folder / f"{download_info['id']}.{download_info['ext']}"
            )

            if download_info is None:
                logger.debug(
                    "No download info found for %s, url: %s",
                    song.display_name,
                    download_url,
                )

                raise DownloaderError(
                    f"yt-dlp failed to get metadata for: {song.name} - {song.artist}"
                )

            display_progress_tracker.notify_download_complete()

            # Copy the downloaded file to the output file
            # if the temp file and output file have the same extension
            # and the bitrate is set to auto or disable
            # Don't copy if the audio provider is piped
            # unless the bitrate is set to disable
            if (
                self.settings["bitrate"] in ["auto", "disable", None]
                and temp_file.suffix == output_file.suffix
            ) and not (
                self.settings["audio_providers"][0] == "piped"
                and self.settings["bitrate"] != "disable"
            ):
                shutil.move(str(temp_file), output_file)
                success = True
                result = None
            else:
                if self.settings["bitrate"] in ["auto", None]:
                    # Use the bitrate from the download info if it exists
                    # otherwise use `copy`
                    bitrate = (
                        f"{int(download_info['abr'])}k"
                        if download_info.get("abr")
                        else "copy"
                    )
                elif self.settings["bitrate"] == "disable":
                    bitrate = None
                else:
                    bitrate = str(self.settings["bitrate"])

                # Convert the downloaded file to the output format
                success, result = convert(
                    input_file=temp_file,
                    output_file=output_file,
                    ffmpeg=self.ffmpeg,
                    output_format=self.settings["format"],
                    bitrate=bitrate,
                    ffmpeg_args=self.settings["ffmpeg_args"],
                    progress_handler=display_progress_tracker.ffmpeg_progress_hook,
                )

                if self.settings["create_skip_file"]:
                    with open(
                        str(output_file) + ".skip", mode="w", encoding="utf-8"
                    ) as _:
                        pass

            # Remove the temp file
            if temp_file.exists():
                try:
                    temp_file.unlink()
                except (PermissionError, OSError) as exc:
                    logger.debug(
                        "Could not remove temp file: %s, error: %s", temp_file, exc
                    )

                    raise DownloaderError(
                        f"Could not remove temp file: {temp_file}, possible duplicate song"
                    ) from exc

            if not success and result:
                # If the conversion failed and there is an error message
                # create a file with the error message
                # and save it in the errors directory
                # raise an exception with file path
                file_name = (
                    get_errors_path()
                    / f"ffmpeg_error_{datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}.txt"
                )

                error_message = ""
                for key, value in result.items():
                    error_message += f"### {key}:\n{str(value).strip()}\n\n"

                with open(file_name, "w", encoding="utf-8") as error_path:
                    error_path.write(error_message)

                # Remove the file that failed to convert
                if output_file.exists():
                    output_file.unlink()

                raise FFmpegError(
                    f"Failed to convert {song.display_name}, "
                    f"you can find error here: {str(file_name.absolute())}"
                )

            download_info["filepath"] = str(output_file)

            # Set the song's download url
            if song.download_url is None:
                song.download_url = download_url

            display_progress_tracker.notify_conversion_complete()

            # SponsorBlock post processor
            if self.settings["sponsor_block"]:
                # Initialize the sponsorblock post processor
                post_processor = SponsorBlockPP(
                    audio_downloader.audio_handler, SPONSOR_BLOCK_CATEGORIES
                )

                # Run the post processor to get the sponsor segments
                _, download_info = post_processor.run(download_info)
                chapters = download_info["sponsorblock_chapters"]

                # If there are sponsor segments, remove them
                if len(chapters) > 0:
                    logger.info(
                        "Removing %s sponsor segments for %s",
                        len(chapters),
                        song.display_name,
                    )

                    # Initialize the modify chapters post processor
                    modify_chapters = ModifyChaptersPP(
                        downloader=audio_downloader.audio_handler,
                        remove_sponsor_segments=SPONSOR_BLOCK_CATEGORIES,
                    )

                    # Run the post processor to remove the sponsor segments
                    # this returns a list of files to delete
                    files_to_delete, download_info = modify_chapters.run(download_info)

                    # Delete the files that were created by the post processor
                    for file_to_delete in files_to_delete:
                        Path(file_to_delete).unlink()

            try:
                embed_metadata(
                    output_file,
                    song,
                    id3_separator=self.settings["id3_separator"],
                    skip_album_art=self.settings["skip_album_art"],
                )
            except Exception as exception:
                raise MetadataError(
                    "Failed to embed metadata to the song"
                ) from exception

            if self.settings["generate_lrc"]:
                generate_lrc(song, output_file)

            display_progress_tracker.notify_complete()

            # Add the song to the known songs
            self.known_songs.get(song.url, []).append(output_file)

            logger.info('Downloaded "%s": %s', song.display_name, song.download_url)

            return song, output_file
        except (Exception, UnicodeEncodeError) as exception:
            if isinstance(exception, UnicodeEncodeError):
                exception_cause = exception
                exception = DownloaderError(
                    "You may need to add PYTHONIOENCODING=utf-8 to your environment"
                )

                exception.__cause__ = exception_cause

            display_progress_tracker.notify_error(
                traceback.format_exc(), exception, True
            )
            self.errors.append(
                f"{song.url} - {exception.__class__.__name__}: {exception}"
            )
            return song, None



File: spotdl/download/__init__.py

"""
Download module that holds the downloader and progress handler modules.
"""



File: spotdl/download/progress_handler.py

"""
Module that holds the ProgressHandler class and Song Tracker class.
"""

import logging
from typing import Any, Callable, Dict, List, Optional

from rich import get_console
from rich.console import JustifyMethod, OverflowMethod
from rich.highlighter import Highlighter
from rich.markup import escape
from rich.progress import (
    BarColumn,
    Progress,
    ProgressColumn,
    Task,
    TaskID,
    TimeRemainingColumn,
)
from rich.style import StyleType
from rich.text import Text

from spotdl.types.song import Song
from spotdl.utils.static import BAD_CHARS

__all__ = [
    "ProgressHandler",
    "SongTracker",
    "ProgressHandlerError",
    "SizedTextColumn",
]

logger = logging.getLogger(__name__)


class ProgressHandlerError(Exception):
    """
    Base class for all exceptions raised by ProgressHandler subclasses.
    """


class SizedTextColumn(ProgressColumn):
    """
    Custom sized text column based on the Rich library.
    """

    def __init__(
        self,
        text_format: str,
        style: StyleType = "none",
        justify: JustifyMethod = "left",
        markup: bool = True,
        highlighter: Optional[Highlighter] = None,
        overflow: Optional[OverflowMethod] = None,
        width: int = 20,
    ) -> None:
        """
        A column containing text.

        ### Arguments
        - text_format: The format string to use for the text.
        - style: The style to use for the text.
        - justify: The justification to use for the text.
        - markup: Whether or not the text should be rendered as markup.
        - highlighter: A Highlighter to use for highlighting the text.
        - overflow: The overflow method to use for truncating the text.
        - width: The maximum width of the text.
        """

        self.text_format = text_format
        self.justify: JustifyMethod = justify
        self.style = style
        self.markup = markup
        self.highlighter = highlighter
        self.overflow: Optional[OverflowMethod] = overflow
        self.width = width
        super().__init__()

    def render(self, task: Task) -> Text:
        """
        Render the Column.

        ### Arguments
        - task: The Task to render.

        ### Returns
        - A Text object.
        """

        _text = self.text_format.format(task=task)
        if self.markup:
            text = Text.from_markup(_text, style=self.style, justify=self.justify)
        else:
            text = Text(_text, style=self.style, justify=self.justify)
        if self.highlighter:
            self.highlighter.highlight(text)

        text.truncate(max_width=self.width, overflow=self.overflow, pad=True)
        return text


class ProgressHandler:
    """
    Class for handling the progress of a download, including the progress bar.
    """

    def __init__(
        self,
        simple_tui: bool = False,
        update_callback: Optional[Callable[[Any, str], None]] = None,
        web_ui: bool = False,
    ):
        """
        Initialize the progress handler.

        ### Arguments
        - simple_tui: Whether or not to use the simple TUI.
        - update_callback: A callback to call when the progress bar is updated.
        """

        self.songs: List[Song] = []
        self.song_count: int = 0
        self.overall_progress = 0
        self.overall_total = 100
        self.overall_completed_tasks = 0
        self.update_callback = update_callback
        self.previous_overall = self.overall_completed_tasks

        self.simple_tui = simple_tui
        self.web_ui = web_ui
        self.quiet = logger.getEffectiveLevel() < 10
        self.overall_task_id: Optional[TaskID] = None

        if not self.simple_tui:
            console = get_console()

            self.rich_progress_bar = Progress(
                SizedTextColumn(
                    "[white]{task.description}",
                    overflow="ellipsis",
                    width=int(console.width / 3),
                ),
                SizedTextColumn(
                    "{task.fields[message]}", width=18, style="nonimportant"
                ),
                BarColumn(bar_width=None, finished_style="green"),
                "[progress.percentage]{task.percentage:>3.0f}%",
                TimeRemainingColumn(),
                # Normally when you exit the progress context manager (or call stop())
                # the last refreshed display remains in the terminal with the cursor on
                # the following line. You can also make the progress display disappear on
                # exit by setting transient=True on the Progress constructor
                transient=True,
            )

            # Basically a wrapper for rich's: with ... as ...
            self.rich_progress_bar.__enter__()

    def add_song(self, song: Song) -> None:
        """
        Adds a song to the list of songs.

        ### Arguments
        - song: The song to add.
        """

        self.songs.append(song)
        self.set_song_count(len(self.songs))

    def set_songs(self, songs: List[Song]) -> None:
        """
        Sets the list of songs to be downloaded.

        ### Arguments
        - songs: The list of songs to download.
        """

        self.songs = songs
        self.set_song_count(len(songs))

    def set_song_count(self, count: int) -> None:
        """
        Set the number of songs to download.

        ### Arguments
        - count: The number of songs to download.
        """

        self.song_count = count
        self.overall_total = 100 * count

        if not self.simple_tui:
            if self.song_count > 4:
                self.overall_task_id = self.rich_progress_bar.add_task(
                    description="Total",
                    message=(
                        f"{self.overall_completed_tasks}/{int(self.overall_total / 100)} "
                        "complete"
                    ),
                    total=self.overall_total,
                    visible=(not self.quiet),
                )

    def update_overall(self) -> None:
        """
        Update the overall progress bar.
        """

        if not self.simple_tui:
            # If the overall progress bar exists
            if self.overall_task_id is not None:
                self.rich_progress_bar.update(
                    self.overall_task_id,
                    message=f"{self.overall_completed_tasks}/"
                    f"{int(self.overall_total / 100)} "
                    "complete",
                    completed=self.overall_progress,
                )
        else:
            if self.previous_overall != self.overall_completed_tasks:
                logger.info(
                    "%s/%s complete", self.overall_completed_tasks, self.song_count
                )
                self.previous_overall = self.overall_completed_tasks

    def get_new_tracker(self, song: Song) -> "SongTracker":
        """
        Get a new progress tracker.

        ### Arguments
        - song: The song to track.

        ### Returns
        - A new progress tracker.
        """

        return SongTracker(self, song)

    def close(self) -> None:
        """
        Close the Tui Progress Handler.
        """

        if not self.simple_tui:
            self.rich_progress_bar.stop()

        logging.shutdown()


class SongTracker:
    """
    Class to track the progress of a song.
    """

    def __init__(self, parent, song: Song) -> None:
        """
        Initialize the Tui Song Tracker.

        ### Arguments
        - parent: The parent Tui Progress Handler.
        """

        self.parent: "ProgressHandler" = parent
        self.song = song

        # Clean up the song name
        # from weird unicode characters
        self.song_name = "".join(
            char
            for char in self.song.display_name
            if char not in [chr(i) for i in BAD_CHARS]
        )

        self.progress: int = 0
        self.old_progress: int = 0
        self.status = ""

        if not self.parent.simple_tui:
            self.task_id = self.parent.rich_progress_bar.add_task(
                description=escape(self.song_name),
                message="Processing",
                total=100,
                completed=self.progress,
                start=False,
                visible=(not self.parent.quiet),
            )

    def update(self, message=""):
        """
        Called at every event.

        ### Arguments
        - message: The message to display.
        """

        old_message = self.status
        self.status = message

        # The change in progress since last update
        delta = self.progress - self.old_progress

        if not self.parent.simple_tui:
            # Update the progress bar
            # `start_task` called everytime to ensure progress is remove from indeterminate state
            self.parent.rich_progress_bar.start_task(self.task_id)
            self.parent.rich_progress_bar.update(
                self.task_id,
                description=escape(self.song_name),
                message=message,
                completed=self.progress,
            )

            # If task is complete
            if self.progress == 100 or message == "Error":
                self.parent.overall_completed_tasks += 1
                self.parent.rich_progress_bar.remove_task(self.task_id)
        else:
            # If task is complete
            if self.progress == 100 or message == "Error":
                self.parent.overall_completed_tasks += 1

            # When running web ui print progress
            # only one time when downloading/converting/embedding
            if self.parent.web_ui and old_message != self.status:
                logger.info("%s: %s", self.song_name, message)
            elif not self.parent.web_ui and delta:
                logger.info("%s: %s", self.song_name, message)

        # Update the overall progress bar
        if self.parent.song_count == self.parent.overall_completed_tasks:
            self.parent.overall_progress = self.parent.song_count * 100
        else:
            self.parent.overall_progress += delta

        self.parent.update_overall()
        self.old_progress = self.progress

        if self.parent.update_callback:
            self.parent.update_callback(self, message)

    def notify_error(
        self, message: str, traceback: Exception, finish: bool = False
    ) -> None:
        """
        Logs an error message.

        ### Arguments
        - message: The message to log.
        - traceback: The traceback of the error.
        - finish: Whether to finish the task.
        """

        self.update("Error")
        if finish:
            self.progress = 100

        if logger.getEffectiveLevel() == logging.DEBUG:
            logger.exception(message)
        else:
            logger.error("%s: %s", traceback.__class__.__name__, traceback)

    def notify_download_complete(self, status="Converting") -> None:
        """
        Notifies the progress handler that the song has been downloaded.

        ### Arguments
        - status: The status to display.
        """

        self.progress = 50
        self.update(status)

    def notify_conversion_complete(self, status="Embedding metadata") -> None:
        """
        Notifies the progress handler that the song has been converted.

        ### Arguments
        - status: The status to display.
        """

        self.progress = 95
        self.update(status)

    def notify_complete(self, status="Done") -> None:
        """
        Notifies the progress handler that the song has been downloaded and converted.

        ### Arguments
        - status: The status to display.
        """

        self.progress = 100
        self.update(status)

    def notify_download_skip(self, status="Skipped") -> None:
        """
        Notifies the progress handler that the song has been skipped.

        ### Arguments
        - status: The status to display.
        """

        self.progress = 100
        self.update(status)

    def ffmpeg_progress_hook(self, progress: int) -> None:
        """
        Updates the progress.

        ### Arguments
        - progress: The progress to update to.
        """

        if self.parent.simple_tui and not self.parent.web_ui:
            self.progress = 50
        else:
            self.progress = 50 + int(progress * 0.45)

        self.update("Converting")

    def yt_dlp_progress_hook(self, data: Dict[str, Any]) -> None:
        """
        Updates the progress.

        ### Arguments
        - progress: The progress to update to.
        """

        if data["status"] == "downloading":
            file_bytes = data.get("total_bytes")
            if file_bytes is None:
                file_bytes = data.get("total_bytes_estimate")

            downloaded_bytes = data.get("downloaded_bytes")
            if self.parent.simple_tui and not self.parent.web_ui:
                self.progress = 50
            elif file_bytes and downloaded_bytes:
                self.progress = downloaded_bytes / file_bytes * 50

            self.update("Downloading")



File: spotdl/console/save.py

"""
Save module for the console.
"""

import asyncio
import json
import logging
from typing import List

from spotdl.download.downloader import Downloader, DownloaderError
from spotdl.types.song import Song
from spotdl.utils.m3u import gen_m3u_files
from spotdl.utils.search import parse_query

__all__ = ["save"]

logger = logging.getLogger(__name__)


def save(
    query: List[str],
    downloader: Downloader,
) -> None:
    """
    Save metadata from spotify to the disk.

    ### Arguments
    - query: list of strings to search for.
    - downloader: Already initialized downloader instance.

    ### Notes
    - This function is multi-threaded.
    """

    save_path = downloader.settings["save_file"]
    m3u_file = downloader.settings["m3u"]

    to_stdout = save_path == "-"

    if save_path is None and not to_stdout:
        raise DownloaderError("Save file is not specified")

    # Parse the query
    songs = parse_query(
        query=query,
        threads=downloader.settings["threads"],
        use_ytm_data=downloader.settings["ytm_data"],
        playlist_numbering=downloader.settings["playlist_numbering"],
        album_type=downloader.settings["album_type"],
    )
    save_data = [song.json for song in songs]

    def process_song(song: Song):
        try:
            data = downloader.search(song)
            if data is None:
                logger.error("Could not find a match for %s", song.display_name)

                return None

            logger.info("Found url for %s: %s", song.display_name, data)

            return {**song.json, "download_url": data}
        except Exception as exception:
            logger.error("%s generated an exception: %s", song.display_name, exception)

        return None

    async def pool_worker(song: Song):
        async with downloader.semaphore:
            # The following function calls blocking code, which would block whole event loop.
            # Therefore it has to be called in a separate thread via ThreadPoolExecutor. This
            # is not a problem, since GIL is released for the I/O operations, so it shouldn't
            # hurt performance.
            return await downloader.loop.run_in_executor(None, process_song, song)

    if downloader.settings["preload"]:
        tasks = [pool_worker(song) for song in songs]

        # call all task asynchronously, and wait until all are finished
        save_data = list(downloader.loop.run_until_complete(asyncio.gather(*tasks)))

    if to_stdout:
        # Print the songs to stdout
        print(json.dumps(save_data, indent=4, ensure_ascii=False))
    elif save_path:
        # Save the songs to a file
        with open(save_path, "w", encoding="utf-8") as save_file:
            json.dump(save_data, save_file, indent=4, ensure_ascii=False)

    if m3u_file:
        gen_m3u_files(
            songs,
            m3u_file,
            downloader.settings["output"],
            downloader.settings["format"],
            downloader.settings["restrict"],
            False,
        )

    if not to_stdout:
        logger.info(
            "Saved %s song%s to %s",
            len(save_data),
            "s" if len(save_data) > 1 else "",
            save_path,
        )



File: spotdl/console/sync.py

"""
Sync module for the console.
"""

import json
import logging
from typing import List, Tuple
from pathlib import Path

from spotdl.download.downloader import Downloader
from spotdl.types.song import Song
from spotdl.utils.formatter import create_file_name
from spotdl.utils.m3u import gen_m3u_files
from spotdl.utils.search import parse_query

__all__ = ["sync"]

logger = logging.getLogger(__name__)


def sync(
    query: List[str],
    downloader: Downloader,
) -> None:
    """
    Sync function for the console.
    It will download the songs and remove the ones that are no longer
    present in the playlists/albums/etc


    ### Arguments
    - query: list of strings to search for.
    - downloader: Already initialized downloader instance.
    """

    save_path = downloader.settings["save_file"]
    downloader.settings["save_file"] = None

    m3u_file = downloader.settings["m3u"]
    downloader.settings["m3u"] = None

    # Query and save file
    # Create initial sync file
    if query and save_path:
        if any(req for req in query if req.endswith(".spotdl")):
            # If the query contains a .spotdl file, and we are about to create
            # .spotdl file, raise an error.
            raise ValueError(
                "Cannot create a sync file with a .spotdl file in the query."
            )

        # Parse the query
        songs_list = parse_query(
            query=query,
            threads=downloader.settings["threads"],
            use_ytm_data=downloader.settings["ytm_data"],
            playlist_numbering=downloader.settings["playlist_numbering"],
            album_type=downloader.settings["album_type"],
        )

        # Create sync file
        with open(save_path, "w", encoding="utf-8") as save_file:
            json.dump(
                {
                    "type": "sync",
                    "query": query,
                    "songs": [song.json for song in songs_list],
                },
                save_file,
                indent=4,
                ensure_ascii=False,
            )

        # Perform initial download
        downloader.download_multiple_songs(songs_list)

        # Create m3u file
        if m3u_file:
            gen_m3u_files(
                songs_list,
                m3u_file,
                downloader.settings["output"],
                downloader.settings["format"],
                downloader.settings["restrict"],
                False,
            )

        return None

    # If the query is a single file, download it
    if (  # pylint: disable=R1702
        len(query) == 1  # pylint: disable=R1702
        and query[0].endswith(".spotdl")  # pylint: disable=R1702
        and not save_path  # pylint: disable=R1702
    ):
        # Load the sync file
        with open(query[0], "r", encoding="utf-8") as sync_file:
            sync_data = json.load(sync_file)

        # Verify the sync file
        if (
            not isinstance(sync_data, dict)
            or sync_data.get("type") != "sync"
            or sync_data.get("songs") is None
        ):
            raise ValueError("Sync file is not a valid sync file.")

        # Parse the query
        songs_playlist = parse_query(
            query=sync_data["query"],
            threads=downloader.settings["threads"],
            use_ytm_data=downloader.settings["ytm_data"],
            playlist_numbering=downloader.settings["playlist_numbering"],
            album_type=downloader.settings["album_type"],
        )

        # Get the names and URLs of previously downloaded songs from the sync file
        old_files = []
        for entry in sync_data["songs"]:
            file_name = create_file_name(
                Song.from_dict(entry),
                downloader.settings["output"],
                downloader.settings["format"],
                downloader.settings["restrict"],
            )

            old_files.append((file_name, entry["url"]))

        new_urls = [song.url for song in songs_playlist]

        # Delete all song files whose URL is no longer part of the latest playlist
        if not downloader.settings["sync_without_deleting"]:
            # Rename songs that have "{list-length}", "{list-position}", "{list-name}",
            # in the output path so that we don't have to download them again,
            # and to avoid mangling the directory structure.
            to_rename: List[Tuple[Path, Path]] = []
            to_delete = []
            for path, url in old_files:
                if url not in new_urls:
                    to_delete.append(path)
                else:
                    new_song = songs_playlist[new_urls.index(url)]

                    new_path = create_file_name(
                        Song.from_dict(new_song.json),
                        downloader.settings["output"],
                        downloader.settings["format"],
                        downloader.settings["restrict"],
                    )

                    if path != new_path:
                        to_rename.append((path, new_path))

            # TODO: Downloading duplicate songs in the same playlist
            # will trigger a re-download of the song. To fix this we have to copy the song
            # to the new location without removing the old one.
            for old_path, new_path in to_rename:
                if old_path.exists():
                    logger.info("Renaming %s to %s", f"'{old_path}'", f"'{new_path}'")
                    if new_path.exists():
                        old_path.unlink()
                        continue

                    try:
                        old_path.rename(new_path)
                    except (PermissionError, OSError) as exc:
                        logger.debug(
                            "Could not rename temp file: %s, error: %s", old_path, exc
                        )
                else:
                    logger.debug("%s does not exist.", old_path)

                if downloader.settings["sync_remove_lrc"]:
                    lrc_file = old_path.with_suffix(".lrc")
                    new_lrc_file = new_path.with_suffix(".lrc")
                    if lrc_file.exists():
                        logger.debug(
                            "Renaming lrc %s to %s",
                            f"'{lrc_file}'",
                            f"'{new_lrc_file}'",
                        )
                        try:
                            lrc_file.rename(new_lrc_file)
                        except (PermissionError, OSError) as exc:
                            logger.debug(
                                "Could not rename lrc file: %s, error: %s",
                                lrc_file,
                                exc,
                            )
                    else:
                        logger.debug("%s does not exist.", lrc_file)

            for file in to_delete:
                if file.exists():
                    logger.info("Deleting %s", file)
                    try:
                        file.unlink()
                    except (PermissionError, OSError) as exc:
                        logger.debug(
                            "Could not remove temp file: %s, error: %s", file, exc
                        )
                else:
                    logger.debug("%s does not exist.", file)

                if downloader.settings["sync_remove_lrc"]:
                    lrc_file = file.with_suffix(".lrc")
                    if lrc_file.exists():
                        logger.debug("Deleting lrc %s", lrc_file)
                        try:
                            lrc_file.unlink()
                        except (PermissionError, OSError) as exc:
                            logger.debug(
                                "Could not remove lrc file: %s, error: %s",
                                lrc_file,
                                exc,
                            )
                    else:
                        logger.debug("%s does not exist.", lrc_file)

            if len(to_delete) == 0:
                logger.info("Nothing to delete...")
            else:
                logger.info("%s old songs were deleted.", len(to_delete))

        if m3u_file:
            gen_m3u_files(
                songs_playlist,
                m3u_file,
                downloader.settings["output"],
                downloader.settings["format"],
                downloader.settings["restrict"],
                False,
            )

        # Write the new sync file
        with open(query[0], "w", encoding="utf-8") as save_file:
            json.dump(
                {
                    "type": "sync",
                    "query": sync_data["query"],
                    "songs": [song.json for song in songs_playlist],
                },
                save_file,
                indent=4,
                ensure_ascii=False,
            )

        downloader.download_multiple_songs(songs_playlist)

        return None

    raise ValueError(
        "Wrong combination of arguments. "
        "Either provide a query and a save path. Or a single sync file in the query"
    )



File: spotdl/console/web.py

"""
Web module for the console.
"""

import asyncio
import logging
import os
import sys
import webbrowser
from pathlib import Path

from fastapi import Depends, FastAPI
from fastapi.middleware.cors import CORSMiddleware
from uvicorn import Config, Server

from spotdl._version import __version__
from spotdl.types.options import DownloaderOptions, WebOptions
from spotdl.utils.config import get_web_ui_path
from spotdl.utils.github import download_github_dir
from spotdl.utils.logging import NAME_TO_LEVEL
from spotdl.utils.web import (
    ALLOWED_ORIGINS,
    SPAStaticFiles,
    app_state,
    fix_mime_types,
    get_current_state,
    router,
)

__all__ = ["web"]

logger = logging.getLogger(__name__)


def web(web_settings: WebOptions, downloader_settings: DownloaderOptions):
    """
    Run the web server.

    ### Arguments
    - web_settings: Web server settings.
    - downloader_settings: Downloader settings.
    """

    # Apply the fix for mime types
    fix_mime_types()

    # Set up the app loggers
    uvicorn_logger = logging.getLogger("uvicorn")
    uvicorn_logger.propagate = False

    spotipy_logger = logging.getLogger("spotipy")
    spotipy_logger.setLevel(logging.NOTSET)

    # Initialize the web server settings
    app_state.web_settings = web_settings
    app_state.logger = uvicorn_logger

    # Create the event loop
    app_state.loop = (
        asyncio.new_event_loop()
        if sys.platform != "win32"
        else asyncio.ProactorEventLoop()  # type: ignore
    )

    downloader_settings["simple_tui"] = True

    # Download web app from GitHub if not already downloaded or force flag set
    web_app_dir = get_web_ui_path()
    dist_dir = web_app_dir / "dist"
    if (not dist_dir.exists() or web_settings["force_update_gui"]) and web_settings[
        "web_gui_location"
    ] is None:
        if web_settings["web_gui_repo"] is None:
            gui_repo = "https://github.com/spotdl/web-ui/tree/master/dist"
        else:
            gui_repo = web_settings["web_gui_repo"]

        logger.info("Updating web app from %s", gui_repo)

        download_github_dir(
            gui_repo,
            output_dir=str(web_app_dir),
        )
        web_app_dir = Path(os.path.join(web_app_dir, "dist")).resolve()
    elif web_settings["web_gui_location"]:
        web_app_dir = Path(web_settings["web_gui_location"]).resolve()
        logger.info("Using custom web app location: %s", web_app_dir)
    else:
        logger.info(
            "Using cached web app. To update use the `--force-update-gui` flag."
        )
        web_app_dir = Path(os.path.join(web_app_dir, "dist")).resolve()

    app_state.api = FastAPI(
        title="spotDL",
        description="Download music from Spotify",
        version=__version__,
        dependencies=[Depends(get_current_state)],
    )

    app_state.api.include_router(router)

    # Add the CORS middleware
    app_state.api.add_middleware(
        CORSMiddleware,
        allow_origins=(
            ALLOWED_ORIGINS + web_settings["allowed_origins"]
            if web_settings["allowed_origins"]
            else ALLOWED_ORIGINS
        ),
        allow_credentials=True,
        allow_methods=["*"],
        allow_headers=["*"],
    )

    # Add the static files
    app_state.api.mount(
        "/",
        SPAStaticFiles(directory=web_app_dir, html=True),
        name="static",
    )
    protocol = "http"
    config = Config(
        app=app_state.api,
        host=web_settings["host"],
        port=web_settings["port"],
        workers=1,
        log_level=NAME_TO_LEVEL[downloader_settings["log_level"]],
        loop=app_state.loop,  # type: ignore
    )
    if web_settings["enable_tls"]:
        logger.info("Enabeling TLS")
        protocol = "https"
        config.ssl_certfile = web_settings["cert_file"]
        config.ssl_keyfile = web_settings["key_file"]
        config.ssl_ca_certs = web_settings["ca_file"]

    app_state.server = Server(config)

    app_state.downloader_settings = downloader_settings

    # Open the web browser
    webbrowser.open(f"{protocol}://{web_settings['host']}:{web_settings['port']}/")

    if not web_settings["web_use_output_dir"]:
        logger.info(
            "Files are stored in temporary directory "
            "and will be deleted after the program exits "
            "to save them to current directory permanently "
            "enable the `web_use_output_dir` option "
        )
    else:
        logger.info(
            "Files are stored in current directory "
            "to save them to temporary directory "
            "disable the `web_use_output_dir` option "
        )

    logger.info("Starting web server \n")

    # Start the web server
    app_state.loop.run_until_complete(app_state.server.serve())



File: spotdl/console/download.py

"""
Download module for the console.
"""

from typing import List

from spotdl.download.downloader import Downloader
from spotdl.utils.search import get_simple_songs

__all__ = ["download"]


def download(
    query: List[str],
    downloader: Downloader,
) -> None:
    """
    Find songs with the provided audio provider and save them to the disk.

    ### Arguments
    - query: list of strings to search for.
    """

    # Parse the query
    songs = get_simple_songs(
        query,
        use_ytm_data=downloader.settings["ytm_data"],
        playlist_numbering=downloader.settings["playlist_numbering"],
        albums_to_ignore=downloader.settings["ignore_albums"],
        album_type=downloader.settings["album_type"],
    )

    # Download the songs
    downloader.download_multiple_songs(songs)



File: spotdl/console/entry_point.py

"""
Module that holds the entry point for the console.
"""

import cProfile
import logging
import pstats
import signal
import sys
import time

from spotdl.console.download import download
from spotdl.console.meta import meta
from spotdl.console.save import save
from spotdl.console.sync import sync
from spotdl.console.url import url
from spotdl.console.web import web
from spotdl.download.downloader import Downloader, DownloaderError
from spotdl.utils.arguments import parse_arguments
from spotdl.utils.config import create_settings
from spotdl.utils.console import ACTIONS, generate_initial_config, is_executable
from spotdl.utils.downloader import check_ytmusic_connection
from spotdl.utils.ffmpeg import FFmpegError, download_ffmpeg, is_ffmpeg_installed
from spotdl.utils.logging import init_logging
from spotdl.utils.spotify import SpotifyClient, SpotifyError, save_spotify_cache

__all__ = ["console_entry_point", "OPERATIONS"]

OPERATIONS = {
    "download": download,
    "sync": sync,
    "save": save,
    "meta": meta,
    "url": url,
}

logger = logging.getLogger(__name__)


def console_entry_point():
    """
    Entry point for the console. With profile flag, it runs the code with cProfile.
    """

    if "--profile" in sys.argv:
        with cProfile.Profile() as profile:
            entry_point()

        stats = pstats.Stats(profile)
        stats.sort_stats(pstats.SortKey.TIME)
        stats.dump_stats("spotdl.profile")
    else:
        entry_point()


def entry_point():
    """
    Console entry point for spotdl. This is where the magic happens.
    """

    # Create config file if it doesn't exist
    generate_initial_config()

    # Check if sys.argv contains an action
    # If it does, we run the action and exit
    try:
        action_to_run = next(
            action for action_name, action in ACTIONS.items() if action_name in sys.argv
        )
    except StopIteration:
        action_to_run = None

    if action_to_run:
        action_to_run()
        return None

    # Parse the arguments
    arguments = parse_arguments()

    # Create settings dicts
    spotify_settings, downloader_settings, web_settings = create_settings(arguments)

    init_logging(downloader_settings["log_level"], downloader_settings["log_format"])

    # If the application is frozen, we check for ffmpeg
    # if it's not present download it create config file
    if is_executable():
        if is_ffmpeg_installed() is False:
            download_ffmpeg()

    # Check if ffmpeg is installed
    if is_ffmpeg_installed(downloader_settings["ffmpeg"]) is False:
        raise FFmpegError(
            "FFmpeg is not installed. Please run `spotdl --download-ffmpeg` to install it, "
            "or `spotdl --ffmpeg /path/to/ffmpeg` to specify the path to ffmpeg."
        )

    # Check if we are not blocked by ytm
    if "youtube-music" in downloader_settings["audio_providers"]:
        if not check_ytmusic_connection():
            raise DownloaderError(
                "You are blocked by YouTube Music. "
                "Please use a VPN, change youtube-music to piped, or use other audio providers"
            )

    # Initialize spotify client
    SpotifyClient.init(**spotify_settings)
    spotify_client = SpotifyClient()

    # If the application is frozen start web ui
    # or if the operation is `web`
    if is_executable() or arguments.operation == "web":

        # Default to the current directory when running a frozen application
        if is_executable():
            web_settings["web_use_output_dir"] = True

        # Start web ui
        web(web_settings, downloader_settings)

        return None

    # Check if save file is present and if it's valid
    if isinstance(downloader_settings["save_file"], str) and (
        not downloader_settings["save_file"].endswith(".spotdl")
        and not downloader_settings["save_file"] == "-"
    ):
        raise DownloaderError("Save file has to end with .spotdl")

    # Check if the user is logged in
    if (
        arguments.query
        and "saved" in arguments.query
        and not spotify_settings["user_auth"]
    ):
        raise SpotifyError(
            "You must be logged in to use the saved query. "
            "Log in by adding the --user-auth flag"
        )

    # Initialize the downloader
    # for download, load and preload operations
    downloader = Downloader(downloader_settings)

    def graceful_exit(_signal, _frame):
        if spotify_settings["use_cache_file"]:
            save_spotify_cache(spotify_client.cache)

        downloader.progress_handler.close()
        sys.exit(0)

    signal.signal(signal.SIGINT, graceful_exit)
    signal.signal(signal.SIGTERM, graceful_exit)

    start_time = time.perf_counter()

    try:
        # Pick the operation to perform
        # based on the name and run it!
        OPERATIONS[arguments.operation](
            query=arguments.query,
            downloader=downloader,
        )
    except Exception as exc:
        if downloader_settings["save_errors"]:
            with open(
                downloader_settings["save_errors"], "a", encoding="utf-8"
            ) as error_file:
                error_file.write("\n".join([exc + "\n" for exc in exc.args]))

            logger.debug("Saved errors to %s", downloader_settings["save_errors"])

        end_time = time.perf_counter()
        logger.debug("Took %d seconds", end_time - start_time)

        downloader.progress_handler.close()
        logger.exception("An error occurred")

        sys.exit(1)

    end_time = time.perf_counter()
    logger.debug("Took %d seconds", end_time - start_time)

    if spotify_settings["use_cache_file"]:
        save_spotify_cache(spotify_client.cache)

    downloader.progress_handler.close()

    return None



File: spotdl/console/__init__.py

"""
Console module, contains the console entry point and different subcommands.
"""

from spotdl.console.entry_point import console_entry_point

__all__ = [
    "console_entry_point",
]



File: spotdl/console/url.py

"""
Url module for the console.
"""

import asyncio
import logging
from typing import List

from spotdl.download.downloader import Downloader
from spotdl.types.song import Song
from spotdl.utils.search import parse_query

__all__ = ["url"]

logger = logging.getLogger(__name__)


def url(
    query: List[str],
    downloader: Downloader,
) -> None:
    """
    Print download url for the provided songs.

    ### Arguments
    - query: list of strings to search for.
    """

    # Parse the query
    songs = parse_query(
        query=query,
        threads=downloader.settings["threads"],
        use_ytm_data=downloader.settings["ytm_data"],
        playlist_numbering=downloader.settings["playlist_numbering"],
        album_type=downloader.settings["album_type"],
    )

    def process_song(song: Song):
        try:
            data = downloader.search(song)
            if data is None:
                logger.error("Could not find a match for %s", song.display_name)

                return None

            audio_provider = downloader.audio_providers[0]
            download_url = audio_provider.get_download_metadata(data)["url"]

            print(download_url)
        except Exception as exception:
            logger.error("%s generated an exception: %s", song.display_name, exception)

        return None

    async def pool_worker(song: Song):
        async with downloader.semaphore:
            # The following function calls blocking code, which would block whole event loop.
            # Therefore it has to be called in a separate thread via ThreadPoolExecutor. This
            # is not a problem, since GIL is released for the I/O operations, so it shouldn't
            # hurt performance.
            return await downloader.loop.run_in_executor(None, process_song, song)

    tasks = [pool_worker(song) for song in songs]
    downloader.loop.run_until_complete(asyncio.gather(*tasks))



File: spotdl/console/meta.py

"""
Sync Lyrics module for the console
"""

import asyncio
import logging
from pathlib import Path
from typing import List

from spotdl.download.downloader import Downloader
from spotdl.types.song import Song
from spotdl.utils.ffmpeg import FFMPEG_FORMATS
from spotdl.utils.lrc import generate_lrc
from spotdl.utils.metadata import embed_metadata, get_file_metadata
from spotdl.utils.search import QueryError, get_search_results, parse_query, reinit_song

__all__ = ["meta"]

logger = logging.getLogger(__name__)


def meta(query: List[str], downloader: Downloader) -> None:
    """
    This function applies metadata to the selected songs
    based on the file name.
    If song already has metadata, missing metadata is added

    ### Arguments
    - query: list of strings to search for.
    - downloader: Already initialized downloader instance.

    ### Notes
    - This function is multi-threaded.
    """

    # Create a list of all songs from all paths in query
    paths: List[Path] = []
    for path in query:
        test_path = Path(path)
        if not test_path.exists():
            logger.error("Path does not exist: %s", path)
            continue

        if test_path.is_dir():
            for out_format in FFMPEG_FORMATS:
                paths.extend(test_path.glob(f"*.{out_format}"))
        elif test_path.is_file():
            if test_path.suffix.split(".")[-1] not in FFMPEG_FORMATS:
                logger.error("File is not a supported audio format: %s", path)
                continue

            paths.append(test_path)

    def process_file(file: Path):
        # metadata of the file, url is present in the file.
        song_meta = get_file_metadata(file, downloader.settings["id3_separator"])

        # Check if song has metadata
        # and if it has all the required fields
        # if it has all of these fields, we can assume that the metadata is correct
        if song_meta and not downloader.settings["force_update_metadata"]:
            if (
                song_meta.get("artist")
                and song_meta.get("artists")
                and song_meta.get("name")
                and song_meta.get("lyrics")
                and song_meta.get("album_art")
            ):
                logger.info("Song already has metadata: %s", file.name)
                if downloader.settings["generate_lrc"]:
                    lrc_file = file.with_suffix(".lrc")
                    if lrc_file.exists():
                        logger.info("Lrc file already exists for %s", file.name)
                        return None

                    song = Song.from_missing_data(
                        name=song_meta["name"],
                        artists=song_meta["artists"],
                        artist=song_meta["artist"],
                    )

                    generate_lrc(song, file)
                    if lrc_file.exists():
                        logger.info("Saved lrc file for %s", song.display_name)
                    else:
                        logger.info("Could not find lrc file for %s", song.display_name)

                return None

        # Same as above
        if (
            not song_meta
            or None
            in [
                song_meta.get("name"),
                song_meta.get("album_art"),
                song_meta.get("artist"),
                song_meta.get("artists"),
                song_meta.get("track_number"),
            ]
            or downloader.settings["force_update_metadata"]
        ):
            # Song does not have metadata, or it is missing some fields
            # or we are forcing update of metadata
            # so we search for it
            logger.debug("Searching metadata for %s", file.name)
            search_results = get_search_results(file.stem)
            if not search_results:
                logger.error("Could not find metadata for %s", file.name)
                return None

            song = search_results[0]
        else:
            # Song has metadata, so we use it to reinitialize the song object
            # and fill in the missing metadata
            try:
                song = reinit_song(Song.from_missing_data(**song_meta))
            except QueryError:
                logger.error("Could not find metadata for %s", file.name)
                return None

        # Check if the song has lyric
        # if not use downloader to find lyrics
        if song_meta is None or song_meta.get("lyrics") is None:
            logger.debug("Fetching lyrics for %s", song.display_name)
            song.lyrics = downloader.search_lyrics(song)
            if song.lyrics:
                logger.info("Found lyrics for song: %s", song.display_name)
        else:
            song.lyrics = song_meta.get("lyrics")

        # Apply metadata to the song
        embed_metadata(file, song, skip_album_art=downloader.settings["skip_album_art"])

        logger.info("Applied metadata to %s", file.name)

        if downloader.settings["generate_lrc"]:
            lrc_file = file.with_suffix(".lrc")
            if lrc_file.exists():
                logger.info("Lrc file already exists for %s", file.name)
                return None

            generate_lrc(song, file)
            if lrc_file.exists():
                logger.info("Saved lrc file for %s", song.display_name)
            else:
                logger.info("Could not find lrc file for %s", song.display_name)
        return None

    async def pool_worker(file_path: Path) -> None:
        async with downloader.semaphore:
            # The following function calls blocking code, which would block whole event loop.
            # Therefore it has to be called in a separate thread via ThreadPoolExecutor. This
            # is not a problem, since GIL is released for the I/O operations, so it shouldn't
            # hurt performance.
            await downloader.loop.run_in_executor(None, process_file, file_path)

    tasks = [pool_worker(path) for path in paths]

    # call all task asynchronously, and wait until all are finished
    downloader.loop.run_until_complete(asyncio.gather(*tasks))

    # to re-download the local songs
    if downloader.settings["redownload"]:
        songs_url: List[str] = []
        for file in paths:
            meta_data = get_file_metadata(
                Path(file), downloader.settings["id3_separator"]
            )
            if meta_data and meta_data["url"]:
                songs_url.append(meta_data["url"])

        songs_list = parse_query(
            query=songs_url,
            threads=downloader.settings["threads"],
            use_ytm_data=downloader.settings["ytm_data"],
            playlist_numbering=downloader.settings["playlist_numbering"],
            album_type=downloader.settings["album_type"],
        )

        downloader.download_multiple_songs(songs_list)




<!--- mdformat-toc start --slug=github --->

<!---
!!! IF EDITING THE README, ENSURE TO COPY THE WHOLE FILE TO index.md in `/docs/` AND REMOVE THE REFERENCES TO ReadTheDocs THERE.
--->

<div align="center">

# spotDL v4

**spotDL** finds songs from Spotify playlists on YouTube and downloads them - along with album art, lyrics and metadata.


[![MIT License](https://img.shields.io/github/license/spotdl/spotify-downloader?color=44CC11&style=flat-square)](https://github.com/spotDL/spotify-downloader/blob/master/LICENSE)
[![PyPI version](https://img.shields.io/pypi/pyversions/spotDL?color=%2344CC11&style=flat-square)](https://pypi.org/project/spotdl/)
[![PyPi downloads](https://img.shields.io/pypi/dw/spotDL?label=downloads@pypi&color=344CC11&style=flat-square)](https://pypi.org/project/spotdl/)
![Contributors](https://img.shields.io/github/contributors/spotDL/spotify-downloader?style=flat-square)
[![Discord](https://img.shields.io/discord/771628785447337985?label=discord&logo=discord&style=flat-square)](https://discord.gg/xCa23pwJWY)

> spotDL: The fastest, easiest and most accurate command-line music downloader.
</div>

______________________________________________________________________
**[Read the documentation on ReadTheDocs!](https://spotdl.readthedocs.io)**
______________________________________________________________________


## Installation

Refer to our [Installation Guide](https://spotdl.rtfd.io/en/latest/installation/) for more details.

### Python (Recommended Method)
  - _spotDL_ can be installed by running `pip install spotdl`.
  - To update spotDL run `pip install --upgrade spotdl`

  > On some systems you might have to change `pip` to `pip3`.

<details>
    <summary style="font-size:1.25em"><strong>Other options</strong></summary>

- Prebuilt executable
  - You can download the latest version from the
    [Releases Tab](https://github.com/spotDL/spotify-downloader/releases)
- On Termux
  - `curl -L https://raw.githubusercontent.com/spotDL/spotify-downloader/master/scripts/termux.sh | sh`
- Arch
  - There is an [Arch User Repository (AUR) package](https://aur.archlinux.org/packages/python-spotdl/) for
    spotDL.
- Docker
  - Build image:

    ```bash
    docker build -t spotdl .
    ```

  - Launch container with spotDL parameters (see section below). You need to create mapped
    volume to access song files

    ```bash
    docker run --rm -v $(pwd):/music spotdl download [trackUrl]
    ```

 - Build from source
	```bash
	git clone https://github.com/spotDL/spotify-downloader && cd spotify-downloader
	pip install poetry
	poetry install
	poetry run python3 scripts/build.py
	```
	An executable is created in `spotify-downloader/dist/`.

</details>


### Installing FFmpeg

FFmpeg is required for spotDL. If using FFmpeg only for spotDL, you can simply install FFmpeg to your spotDL installation directory:
`spotdl --download-ffmpeg`

We recommend the above option, but if you want to install FFmpeg system-wide,
follow these instructions

- [Windows Tutorial](https://windowsloop.com/install-ffmpeg-windows-10/)
- OSX - `brew install ffmpeg`
- Linux - `sudo apt install ffmpeg` or use your distro's package manager

## Usage

Using SpotDL without options::
```sh
spotdl [urls]
```
You can run _spotDL_ as a package if running it as a script doesn't work:
```sh
python -m spotdl [urls]
```

General usage:
```sh
spotdl [operation] [options] QUERY
```

There are different **operations** spotDL can perform. The *default* is `download`, which simply downloads the songs from YouTube and embeds metadata.

The **query** for spotDL is usually a list of Spotify URLs, but for some operations like **sync**, only a single link or file is required.
For a list of all **options** use ```spotdl -h```

<details>
<summary style="font-size:1em"><strong>Supported operations</strong></summary>

- `save`: Saves only the metadata from Spotify without downloading anything.
    - Usage:
        `spotdl save [query] --save-file {filename}.spotdl`

- `web`: Starts a web interface instead of using the command line. However, it has limited features and only supports downloading single songs.

- `url`: Get direct download link for each song from the query.
    - Usage:
        `spotdl url [query]`

- `sync`: Updates directories. Compares the directory with the current state of the playlist. Newly added songs will be downloaded and removed songs will be deleted. No other songs will be downloaded and no other files will be deleted.

    - Usage:
        `spotdl sync [query] --save-file {filename}.spotdl`

        This create a new **sync** file, to update the directory in the future, use:

        `spotdl sync {filename}.spotdl`

- `meta`: Updates metadata for the provided song files.
</details>

## Music Sourcing and Audio Quality

spotDL uses YouTube as a source for music downloads. This method is used to avoid any issues related to downloading music from Spotify.

> **Note**
> Users are responsible for their actions and potential legal consequences. We do not support unauthorized downloading of copyrighted material and take no responsibility for user actions.

### Audio Quality

spotDL downloads music from YouTube and is designed to always download the highest possible bitrate; which is 128 kbps for regular users and 256 kbps for YouTube Music premium users.

Check the [Audio Formats](docs/usage.md#audio-formats-and-quality) page for more info.

## Contributing

Interested in contributing? Check out our [CONTRIBUTING.md](docs/CONTRIBUTING.md) to find
resources around contributing along with a guide on how to set up a development environment.

#### Join our amazing community as a code contributor, and help accelerate
<br><br>
<a href="https://github.com/spotDL/spotify-downloader/graphs/contributors">
  <img class="dark-light" src="https://contrib.rocks/image?repo=spotDL/spotify-downloader&anon=0&columns=25&max=100&r=true" />
</a>
## Donate

help support the development and maintenance of the software ❤️

[![paypal](https://img.shields.io/badge/paypal-%2300457C.svg?&style=for-the-badge&logo=paypal&logoColor=white)](https://paypal.me/kko7)
[![kofi](https://img.shields.io/badge/kofi-%23F16061.svg?&style=for-the-badge&logo=ko-fi&logoColor=white)](https://ko-fi.com/xnetcat)

## License

This project is Licensed under the [MIT](/LICENSE) License.

