Skip to content

Fix temp folder deletion when downloading with multiple processes. #92

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tekamiocho wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Fix temp folder deletion when downloading with multiple processes. #92

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
75f8522
feat: reworked how temp files are managed
tekamiocho May 2, 2025
20263c4
feat: partition temp files for downloader instance
tekamiocho May 2, 2025
c3c5c1f
revert: file size check on `download_file`
tekamiocho May 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions orpheus/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -357,11 +357,11 @@ def update_module_storage(self): # Should be refactored eventually

def orpheus_core_download(orpheus_session: Orpheus, media_to_download, third_party_modules, separate_download_module, output_path):
downloader = Downloader(orpheus_session.settings['global'], orpheus_session.module_controls, oprinter, output_path)
os.makedirs('temp', exist_ok=True)

for mainmodule, items in media_to_download.items():
for media in items:
if ModuleModes.download not in orpheus_session.module_settings[mainmodule].module_supported_modes:
downloader.cleanup()
raise Exception(f'{mainmodule} does not support track downloading') # TODO: replace with ModuleDoesNotSupportAbility

# Load and prepare module
Expand All @@ -373,8 +373,10 @@ def orpheus_core_download(orpheus_session: Orpheus, media_to_download, third_par
moduleselected = third_party_modules[i]
if moduleselected:
if moduleselected not in orpheus_session.module_list:
downloader.cleanup()
raise Exception(f'{moduleselected} does not exist in modules.') # TODO: replace with InvalidModuleError
elif i not in orpheus_session.module_settings[moduleselected].module_supported_modes:
downloader.cleanup()
raise Exception(f'Module {moduleselected} does not support {i}') # TODO: replace with ModuleDoesNotSupportAbility
else:
# If all checks pass, load up the selected module
Expand All @@ -390,6 +392,7 @@ def orpheus_core_download(orpheus_session: Orpheus, media_to_download, third_par
# Mode to download playlist using other service
if separate_download_module != 'default' and separate_download_module != mainmodule:
if mediatype is not DownloadTypeEnum.playlist:
downloader.cleanup()
raise Exception('The separate download module option is only for playlists.') # TODO: replace with ModuleDoesNotSupportAbility
downloader.download_playlist(media_id, custom_module=separate_download_module, extra_kwargs=media.extra_kwargs)
else: # Standard download modes
Expand All @@ -402,6 +405,8 @@ def orpheus_core_download(orpheus_session: Orpheus, media_to_download, third_par
elif mediatype is DownloadTypeEnum.artist:
downloader.download_artist(media_id, extra_kwargs=media.extra_kwargs)
else:
downloader.cleanup()
raise Exception(f'\tUnknown media type "{mediatype}"')

if os.path.exists('temp'): shutil.rmtree('temp')
downloader.cleanup()
if os.path.exists('temp') and len(os.listdir('temp')) == 0: shutil.rmtree('temp')
23 changes: 14 additions & 9 deletions orpheus/music_downloader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ def __init__(self, settings, module_controls, oprinter, path):
self.oprinter = oprinter
self.print = self.oprinter.oprint
self.set_indent_number = self.oprinter.set_indent_number
self.temp_subfolder = os.urandom(16).hex()

def search_by_tags(self, module_name, track_info: TrackInfo):
return self.loaded_modules[module_name].search(DownloadTypeEnum.track, f'{track_info.name} {" ".join(track_info.artists)}', track_info=track_info)
Expand Down Expand Up @@ -229,8 +230,8 @@ def download_album(self, album_id, artist_name='', path=None, indent_level=1, ex
if album_info.booklet_url and not os.path.exists(album_path + 'Booklet.pdf'):
self.print('Downloading booklet')
download_file(album_info.booklet_url, album_path + 'Booklet.pdf')
cover_temp_location = download_to_temp(album_info.all_track_cover_jpg_url) if album_info.all_track_cover_jpg_url else ''

cover_temp_location = download_to_temp(album_info.all_track_cover_jpg_url, subfolder=self.temp_subfolder) if album_info.all_track_cover_jpg_url else ''

# Download booklet, animated album cover and album cover if present
self._download_album_files(album_path, album_info)
Expand Down Expand Up @@ -412,7 +413,6 @@ def download_track(self, track_id, album_location='', main_artist='', track_inde

delete_cover = False
if not cover_temp_location:
cover_temp_location = create_temp_filename()
delete_cover = True
covers_module_name = self.third_party_modules[ModuleModes.covers]
covers_module_name = covers_module_name if covers_module_name != self.service_name else None
Expand All @@ -426,7 +426,7 @@ def download_track(self, track_id, album_location='', main_artist='', track_inde
compression=CoverCompressionEnum[self.global_settings['covers']['external_compression'].lower()])

if covers_module_name:
default_temp = download_to_temp(track_info.cover_url)
default_temp = download_to_temp(track_info.cover_url, subfolder=self.temp_subfolder)
test_cover_options = CoverOptions(file_type=ImageFileTypeEnum.jpg, resolution=get_image_resolution(default_temp), compression=CoverCompressionEnum.high)
cover_module = self.loaded_modules[covers_module_name]
rms_threshold = self.global_settings['advanced']['cover_variance_threshold']
Expand All @@ -438,24 +438,24 @@ def download_track(self, track_id, album_location='', main_artist='', track_inde
test_cover_info: CoverInfo = cover_module.get_track_cover(r.result_id, test_cover_options, **r.extra_kwargs)
if test_cover_info.url not in attempted_urls:
attempted_urls.append(test_cover_info.url)
test_temp = download_to_temp(test_cover_info.url)
test_temp = download_to_temp(test_cover_info.url, subfolder=self.temp_subfolder)
rms = compare_images(default_temp, test_temp)
silentremove(test_temp)
self.print(f'Attempt {i} RMS: {rms!s}') # The smaller the root mean square, the closer the image is to the desired one
if rms < rms_threshold:
self.print('Match found below threshold ' + str(rms_threshold))
jpg_cover_info: CoverInfo = cover_module.get_track_cover(r.result_id, jpg_cover_options, **r.extra_kwargs)
download_file(jpg_cover_info.url, cover_temp_location, artwork_settings=self._get_artwork_settings(covers_module_name))
cover_temp_location = download_to_temp(jpg_cover_info.url, subfolder=self.temp_subfolder, artwork_settings=self._get_artwork_settings(covers_module_name))
silentremove(default_temp)
if self.global_settings['covers']['save_external']:
ext_cover_info: CoverInfo = cover_module.get_track_cover(r.result_id, ext_cover_options, **r.extra_kwargs)
download_file(ext_cover_info.url, f'{track_location_name}.{ext_cover_info.file_type.name}', artwork_settings=self._get_artwork_settings(covers_module_name, is_external=True))
break
else:
self.print('Third-party module could not find cover, using fallback')
shutil.move(default_temp, cover_temp_location)
cover_temp_location = default_temp
else:
download_file(track_info.cover_url, cover_temp_location, artwork_settings=self._get_artwork_settings())
cover_temp_location = download_to_temp(track_info.cover_url, subfolder=self.temp_subfolder, artwork_settings=self._get_artwork_settings())
if self.global_settings['covers']['save_external'] and ModuleModes.covers in self.module_settings[self.service_name].module_supported_modes:
ext_cover_info: CoverInfo = self.service.get_track_cover(track_id, ext_cover_options, **track_info.cover_extra_kwargs)
download_file(ext_cover_info.url, f'{track_location_name}.{ext_cover_info.file_type.name}', artwork_settings=self._get_artwork_settings(is_external=True))
Expand Down Expand Up @@ -566,7 +566,7 @@ def download_track(self, track_id, album_location='', main_artist='', track_inde
self.print('Warning: conversion_flags setting is invalid, using defaults')

conv_flags = conversion_flags[new_codec] if new_codec in conversion_flags else {}
temp_track_location = f'{create_temp_filename()}.{new_codec_data.container.name}'
temp_track_location = f'{create_temp_filename(subfolder = self.temp_subfolder)}.{new_codec_data.container.name}'
new_track_location = f'{track_location_name}.{new_codec_data.container.name}'

stream: ffmpeg = ffmpeg.input(track_location, hide_banner=None, y=None)
Expand Down Expand Up @@ -643,3 +643,8 @@ def _get_artwork_settings(self, module_name = None, is_external = False):
'compression': self.global_settings['covers']['external_compression'] if is_external else self.global_settings['covers']['main_compression'],
'format': self.global_settings['covers']['external_format'] if is_external else 'jpg'
}

def cleanup(self):
temp_folder = f"temp/{self.temp_subfolder}"
if os.path.exists(temp_folder):
shutil.rmtree(temp_folder)
19 changes: 12 additions & 7 deletions utils/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,14 +146,19 @@ def set_temporary_setting(settings_location, module, root_setting, setting=None,
session[root_setting] = value
pickle.dump(temporary_settings, open(settings_location, 'wb'))

create_temp_filename = lambda : f'temp/{os.urandom(16).hex()}'

def save_to_temp(input: bytes):
location = create_temp_filename()
def create_temp_filename(subfolder=''):
filename = 'temp/'
if subfolder:
filename += subfolder + '/'
os.makedirs(filename, exist_ok=True)
return filename + os.urandom(16).hex()

def save_to_temp(input: bytes, subfolder=''):
location = create_temp_filename(subfolder)
open(location, 'wb').write(input)
return location

def download_to_temp(url, headers={}, extension='', enable_progress_bar=False, indent_level=0):
location = create_temp_filename() + (('.' + extension) if extension else '')
download_file(url, location, headers=headers, enable_progress_bar=enable_progress_bar, indent_level=indent_level)
def download_to_temp(url, extension='', subfolder='', **kwargs):
location = create_temp_filename(subfolder) + (('.' + extension) if extension else '')
download_file(url, location, **kwargs)
return location