From 75f8522802bdbbafc4fb555ca8761247bd84bbe1 Mon Sep 17 00:00:00 2001 From: tekamiocho Date: Fri, 2 May 2025 10:58:14 +0200 Subject: [PATCH 1/3] feat: reworked how temp files are managed --- orpheus/core.py | 3 +-- orpheus/music_downloader.py | 4 ++-- utils/utils.py | 17 +++++++++++++---- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/orpheus/core.py b/orpheus/core.py index bed2ee8..bbdacb4 100644 --- a/orpheus/core.py +++ b/orpheus/core.py @@ -357,7 +357,6 @@ def update_module_storage(self): # Should be refactored eventually def orpheus_core_download(orpheus_session: Orpheus, media_to_download, third_party_modules, separate_download_module, output_path): downloader = Downloader(orpheus_session.settings['global'], orpheus_session.module_controls, oprinter, output_path) - os.makedirs('temp', exist_ok=True) for mainmodule, items in media_to_download.items(): for media in items: @@ -404,4 +403,4 @@ def orpheus_core_download(orpheus_session: Orpheus, media_to_download, third_par else: raise Exception(f'\tUnknown media type "{mediatype}"') - if os.path.exists('temp'): shutil.rmtree('temp') \ No newline at end of file + if os.path.exists('temp') and len(os.listdir('temp')) == 0: shutil.rmtree('temp') diff --git a/orpheus/music_downloader.py b/orpheus/music_downloader.py index 19e3abf..5e970a3 100644 --- a/orpheus/music_downloader.py +++ b/orpheus/music_downloader.py @@ -412,7 +412,7 @@ def download_track(self, track_id, album_location='', main_artist='', track_inde delete_cover = False if not cover_temp_location: - cover_temp_location = create_temp_filename() + cover_temp_location = touch_file(create_temp_filename()) delete_cover = True covers_module_name = self.third_party_modules[ModuleModes.covers] covers_module_name = covers_module_name if covers_module_name != self.service_name else None @@ -566,7 +566,7 @@ def download_track(self, track_id, album_location='', main_artist='', track_inde self.print('Warning: conversion_flags setting is invalid, using defaults') conv_flags = conversion_flags[new_codec] if new_codec in conversion_flags else {} - temp_track_location = f'{create_temp_filename()}.{new_codec_data.container.name}' + temp_track_location = touch_file(f'{create_temp_filename()}.{new_codec_data.container.name}') new_track_location = f'{track_location_name}.{new_codec_data.container.name}' stream: ffmpeg = ffmpeg.input(track_location, hide_banner=None, y=None) diff --git a/utils/utils.py b/utils/utils.py index b830dfe..a16182d 100644 --- a/utils/utils.py +++ b/utils/utils.py @@ -41,7 +41,7 @@ def fix_byte_limit(path: str, byte_limit=250): r_session = create_requests_session() def download_file(url, file_location, headers={}, enable_progress_bar=False, indent_level=0, artwork_settings=None): - if os.path.isfile(file_location): + if os.path.isfile(file_location) and os.path.getsize(file_location) > 0: return None r = r_session.get(url, stream=True, headers=headers, verify=False) @@ -146,14 +146,23 @@ def set_temporary_setting(settings_location, module, root_setting, setting=None, session[root_setting] = value pickle.dump(temporary_settings, open(settings_location, 'wb')) -create_temp_filename = lambda : f'temp/{os.urandom(16).hex()}' +create_temp_filename = lambda: f'temp/{os.urandom(16).hex()}' + +def touch_file(temp_file): + """ + Creates an empty file placeholder. + """ + os.makedirs('temp', exist_ok=True) + with open(temp_file, 'a'): + os.utime(temp_file) + return temp_file def save_to_temp(input: bytes): - location = create_temp_filename() + location = touch_file(create_temp_filename()) open(location, 'wb').write(input) return location def download_to_temp(url, headers={}, extension='', enable_progress_bar=False, indent_level=0): - location = create_temp_filename() + (('.' + extension) if extension else '') + location = touch_file(create_temp_filename() + (('.' + extension) if extension else '')) download_file(url, location, headers=headers, enable_progress_bar=enable_progress_bar, indent_level=indent_level) return location From 20263c4416f23b3e768c2ff4033916a76a1fc48d Mon Sep 17 00:00:00 2001 From: tekamiocho Date: Fri, 2 May 2025 19:06:23 +0200 Subject: [PATCH 2/3] feat: partition temp files for downloader instance --- orpheus/core.py | 6 ++++++ orpheus/music_downloader.py | 23 ++++++++++++++--------- utils/utils.py | 28 ++++++++++++---------------- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/orpheus/core.py b/orpheus/core.py index bbdacb4..1f1809a 100644 --- a/orpheus/core.py +++ b/orpheus/core.py @@ -361,6 +361,7 @@ def orpheus_core_download(orpheus_session: Orpheus, media_to_download, third_par for mainmodule, items in media_to_download.items(): for media in items: if ModuleModes.download not in orpheus_session.module_settings[mainmodule].module_supported_modes: + downloader.cleanup() raise Exception(f'{mainmodule} does not support track downloading') # TODO: replace with ModuleDoesNotSupportAbility # Load and prepare module @@ -372,8 +373,10 @@ def orpheus_core_download(orpheus_session: Orpheus, media_to_download, third_par moduleselected = third_party_modules[i] if moduleselected: if moduleselected not in orpheus_session.module_list: + downloader.cleanup() raise Exception(f'{moduleselected} does not exist in modules.') # TODO: replace with InvalidModuleError elif i not in orpheus_session.module_settings[moduleselected].module_supported_modes: + downloader.cleanup() raise Exception(f'Module {moduleselected} does not support {i}') # TODO: replace with ModuleDoesNotSupportAbility else: # If all checks pass, load up the selected module @@ -389,6 +392,7 @@ def orpheus_core_download(orpheus_session: Orpheus, media_to_download, third_par # Mode to download playlist using other service if separate_download_module != 'default' and separate_download_module != mainmodule: if mediatype is not DownloadTypeEnum.playlist: + downloader.cleanup() raise Exception('The separate download module option is only for playlists.') # TODO: replace with ModuleDoesNotSupportAbility downloader.download_playlist(media_id, custom_module=separate_download_module, extra_kwargs=media.extra_kwargs) else: # Standard download modes @@ -401,6 +405,8 @@ def orpheus_core_download(orpheus_session: Orpheus, media_to_download, third_par elif mediatype is DownloadTypeEnum.artist: downloader.download_artist(media_id, extra_kwargs=media.extra_kwargs) else: + downloader.cleanup() raise Exception(f'\tUnknown media type "{mediatype}"') + downloader.cleanup() if os.path.exists('temp') and len(os.listdir('temp')) == 0: shutil.rmtree('temp') diff --git a/orpheus/music_downloader.py b/orpheus/music_downloader.py index 5e970a3..05215a2 100644 --- a/orpheus/music_downloader.py +++ b/orpheus/music_downloader.py @@ -41,6 +41,7 @@ def __init__(self, settings, module_controls, oprinter, path): self.oprinter = oprinter self.print = self.oprinter.oprint self.set_indent_number = self.oprinter.set_indent_number + self.temp_subfolder = os.urandom(16).hex() def search_by_tags(self, module_name, track_info: TrackInfo): return self.loaded_modules[module_name].search(DownloadTypeEnum.track, f'{track_info.name} {" ".join(track_info.artists)}', track_info=track_info) @@ -229,8 +230,8 @@ def download_album(self, album_id, artist_name='', path=None, indent_level=1, ex if album_info.booklet_url and not os.path.exists(album_path + 'Booklet.pdf'): self.print('Downloading booklet') download_file(album_info.booklet_url, album_path + 'Booklet.pdf') - - cover_temp_location = download_to_temp(album_info.all_track_cover_jpg_url) if album_info.all_track_cover_jpg_url else '' + + cover_temp_location = download_to_temp(album_info.all_track_cover_jpg_url, subfolder=self.temp_subfolder) if album_info.all_track_cover_jpg_url else '' # Download booklet, animated album cover and album cover if present self._download_album_files(album_path, album_info) @@ -412,7 +413,6 @@ def download_track(self, track_id, album_location='', main_artist='', track_inde delete_cover = False if not cover_temp_location: - cover_temp_location = touch_file(create_temp_filename()) delete_cover = True covers_module_name = self.third_party_modules[ModuleModes.covers] covers_module_name = covers_module_name if covers_module_name != self.service_name else None @@ -426,7 +426,7 @@ def download_track(self, track_id, album_location='', main_artist='', track_inde compression=CoverCompressionEnum[self.global_settings['covers']['external_compression'].lower()]) if covers_module_name: - default_temp = download_to_temp(track_info.cover_url) + default_temp = download_to_temp(track_info.cover_url, subfolder=self.temp_subfolder) test_cover_options = CoverOptions(file_type=ImageFileTypeEnum.jpg, resolution=get_image_resolution(default_temp), compression=CoverCompressionEnum.high) cover_module = self.loaded_modules[covers_module_name] rms_threshold = self.global_settings['advanced']['cover_variance_threshold'] @@ -438,14 +438,14 @@ def download_track(self, track_id, album_location='', main_artist='', track_inde test_cover_info: CoverInfo = cover_module.get_track_cover(r.result_id, test_cover_options, **r.extra_kwargs) if test_cover_info.url not in attempted_urls: attempted_urls.append(test_cover_info.url) - test_temp = download_to_temp(test_cover_info.url) + test_temp = download_to_temp(test_cover_info.url, subfolder=self.temp_subfolder) rms = compare_images(default_temp, test_temp) silentremove(test_temp) self.print(f'Attempt {i} RMS: {rms!s}') # The smaller the root mean square, the closer the image is to the desired one if rms < rms_threshold: self.print('Match found below threshold ' + str(rms_threshold)) jpg_cover_info: CoverInfo = cover_module.get_track_cover(r.result_id, jpg_cover_options, **r.extra_kwargs) - download_file(jpg_cover_info.url, cover_temp_location, artwork_settings=self._get_artwork_settings(covers_module_name)) + cover_temp_location = download_to_temp(jpg_cover_info.url, subfolder=self.temp_subfolder, artwork_settings=self._get_artwork_settings(covers_module_name)) silentremove(default_temp) if self.global_settings['covers']['save_external']: ext_cover_info: CoverInfo = cover_module.get_track_cover(r.result_id, ext_cover_options, **r.extra_kwargs) @@ -453,9 +453,9 @@ def download_track(self, track_id, album_location='', main_artist='', track_inde break else: self.print('Third-party module could not find cover, using fallback') - shutil.move(default_temp, cover_temp_location) + cover_temp_location = default_temp else: - download_file(track_info.cover_url, cover_temp_location, artwork_settings=self._get_artwork_settings()) + cover_temp_location = download_to_temp(track_info.cover_url, subfolder=self.temp_subfolder, artwork_settings=self._get_artwork_settings()) if self.global_settings['covers']['save_external'] and ModuleModes.covers in self.module_settings[self.service_name].module_supported_modes: ext_cover_info: CoverInfo = self.service.get_track_cover(track_id, ext_cover_options, **track_info.cover_extra_kwargs) download_file(ext_cover_info.url, f'{track_location_name}.{ext_cover_info.file_type.name}', artwork_settings=self._get_artwork_settings(is_external=True)) @@ -566,7 +566,7 @@ def download_track(self, track_id, album_location='', main_artist='', track_inde self.print('Warning: conversion_flags setting is invalid, using defaults') conv_flags = conversion_flags[new_codec] if new_codec in conversion_flags else {} - temp_track_location = touch_file(f'{create_temp_filename()}.{new_codec_data.container.name}') + temp_track_location = f'{create_temp_filename(subfolder = self.temp_subfolder)}.{new_codec_data.container.name}' new_track_location = f'{track_location_name}.{new_codec_data.container.name}' stream: ffmpeg = ffmpeg.input(track_location, hide_banner=None, y=None) @@ -643,3 +643,8 @@ def _get_artwork_settings(self, module_name = None, is_external = False): 'compression': self.global_settings['covers']['external_compression'] if is_external else self.global_settings['covers']['main_compression'], 'format': self.global_settings['covers']['external_format'] if is_external else 'jpg' } + + def cleanup(self): + temp_folder = f"temp/{self.temp_subfolder}" + if os.path.exists(temp_folder): + shutil.rmtree(temp_folder) diff --git a/utils/utils.py b/utils/utils.py index a16182d..ac436b8 100644 --- a/utils/utils.py +++ b/utils/utils.py @@ -146,23 +146,19 @@ def set_temporary_setting(settings_location, module, root_setting, setting=None, session[root_setting] = value pickle.dump(temporary_settings, open(settings_location, 'wb')) -create_temp_filename = lambda: f'temp/{os.urandom(16).hex()}' - -def touch_file(temp_file): - """ - Creates an empty file placeholder. - """ - os.makedirs('temp', exist_ok=True) - with open(temp_file, 'a'): - os.utime(temp_file) - return temp_file - -def save_to_temp(input: bytes): - location = touch_file(create_temp_filename()) +def create_temp_filename(subfolder=''): + filename = 'temp/' + if subfolder: + filename += subfolder + '/' + os.makedirs(filename, exist_ok=True) + return filename + os.urandom(16).hex() + +def save_to_temp(input: bytes, subfolder=''): + location = create_temp_filename(subfolder) open(location, 'wb').write(input) return location -def download_to_temp(url, headers={}, extension='', enable_progress_bar=False, indent_level=0): - location = touch_file(create_temp_filename() + (('.' + extension) if extension else '')) - download_file(url, location, headers=headers, enable_progress_bar=enable_progress_bar, indent_level=indent_level) +def download_to_temp(url, extension='', subfolder='', **kwargs): + location = create_temp_filename(subfolder) + (('.' + extension) if extension else '') + download_file(url, location, **kwargs) return location From c3c5c1f26d05260da41abb413a4df3cd785347c4 Mon Sep 17 00:00:00 2001 From: tekamiocho Date: Fri, 2 May 2025 19:08:16 +0200 Subject: [PATCH 3/3] revert: file size check on `download_file` --- utils/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/utils.py b/utils/utils.py index ac436b8..8fe4642 100644 --- a/utils/utils.py +++ b/utils/utils.py @@ -41,7 +41,7 @@ def fix_byte_limit(path: str, byte_limit=250): r_session = create_requests_session() def download_file(url, file_location, headers={}, enable_progress_bar=False, indent_level=0, artwork_settings=None): - if os.path.isfile(file_location) and os.path.getsize(file_location) > 0: + if os.path.isfile(file_location): return None r = r_session.get(url, stream=True, headers=headers, verify=False)