From 2eb93df897202cb5bd5a55594e34935b22a56987 Mon Sep 17 00:00:00 2001 From: Alastair Porter Date: Tue, 17 Feb 2026 12:24:28 +0100 Subject: [PATCH 1/5] Migrate types from docstrings to code --- general/templatetags/util.py | 8 +- sounds/models.py | 57 +++-- .../freesound_audio_processing.py | 14 +- utils/audioprocessing/processing.py | 1 - utils/downloads.py | 17 +- utils/mail.py | 43 ++-- utils/ratelimit.py | 16 +- utils/search/__init__.py | 205 +++++++++--------- utils/search/backends/solr9pysolr.py | 14 +- utils/search/search_forum.py | 21 +- utils/search/search_query_processor.py | 47 ++-- .../search/search_query_processor_options.py | 65 +++--- utils/search/search_sounds.py | 57 ++--- utils/sound_upload.py | 30 ++- utils/tags.py | 16 +- utils/test_helpers.py | 52 ++--- 16 files changed, 373 insertions(+), 290 deletions(-) diff --git a/general/templatetags/util.py b/general/templatetags/util.py index 296b18c14..0b1b9ca86 100644 --- a/general/templatetags/util.py +++ b/general/templatetags/util.py @@ -18,6 +18,8 @@ # See AUTHORS file. # +from __future__ import annotations + import datetime import json import time @@ -93,11 +95,11 @@ def in_list(value, arg): @register.filter -def chunks(l, n): +def chunks(l: list, n: int) -> list: """ Returns the elements of l grouped in chunks of size n. - :param list l: list of elements to regroup - :param int n: number of elements per group + :param l: list of elements to regroup + :param n: number of elements per group :return: list of n-sized lists """ if not isinstance(l, list): diff --git a/sounds/models.py b/sounds/models.py index 2d336c5d8..ee42d648e 100644 --- a/sounds/models.py +++ b/sounds/models.py @@ -18,6 +18,8 @@ # See AUTHORS file. # +from __future__ import annotations + import datetime import glob import json @@ -988,11 +990,11 @@ def get_sound_tags_string(self, limit=None): """ return " ".join(self.get_sound_tags(limit=limit)) - def set_tags(self, tags): + def set_tags(self, tags: list): """ Updates the tags of the Sound object. To do that it first removes all SoundTag objects which relate the sound with tags which are not in the provided list of tags, and then adds the new tags. - :param list tags: list of strings representing the new tags that the Sound object should be assigned + :param tags: list of strings representing the new tags that the Sound object should be assigned """ # remove tags that are not in the list for tagged_item in self.soundtag_set.select_related("tag").all(): @@ -1006,11 +1008,11 @@ def set_tags(self, tags): tag_object, _ = Tag.objects.get_or_create(name=tag) SoundTag.objects.get_or_create(tag=tag_object, sound=self, defaults={"user": self.user}) - def set_license(self, new_license): + def set_license(self, new_license: License): """ Set `new_license` as the current license of the sound. Create the corresponding SoundLicenseHistory object. Note that this method *does not save* the sound object, it needs to be manually done afterwards. - :param License new_license: License object representing the new license + :param new_license: License object representing the new license """ self.license = new_license SoundLicenseHistory.objects.create(sound=self, license=new_license) @@ -1021,7 +1023,7 @@ def get_sound_sources_as_set(self): """ return set(self.sources.values_list("id", flat=True)) - def change_sources_and_propagate(self, new_sources): + def change_sources_and_propagate(self, new_sources: set): """ Replace this sound's remix sources with ``new_sources`` and propagate the change to dependent systems: @@ -1075,36 +1077,43 @@ def change_sources_and_propagate(self, new_sources): # N.B. The set_xxx functions below are used in the distributed processing and other parts of the app where we only # want to save an individual field of the model to prevent overwriting other model fields. - def set_processing_ongoing_state(self, state): + def set_processing_ongoing_state(self, state: str): """ Updates self.processing_ongoing_state field of the Sound object and saves to DB without updating other fields. This function is used in cases when two instances of the same Sound object could be edited by two processes in parallel and we want to avoid possible field overwrites. - :param str state: new state to which self.processing_ongoing_state should be set + :param state: new state to which self.processing_ongoing_state should be set """ self.processing_ongoing_state = state self.save(update_fields=["processing_ongoing_state"]) - def set_similarity_state(self, state): + def set_similarity_state(self, state: str): """ Updates self.similarity_state field of the Sound object and saves to DB without updating other fields. This function is used in cases when two instances of the same Sound object could be edited by two processes in parallel and we want to avoid possible field overwrites. - :param str state: new state to which self.similarity_state should be set + :param state: new state to which self.similarity_state should be set """ self.similarity_state = state self.save(update_fields=["similarity_state"]) - def set_audio_info_fields(self, samplerate=None, bitrate=None, bitdepth=None, channels=None, duration=None): + def set_audio_info_fields( + self, + samplerate: int | None = None, + bitrate: int | None = None, + bitdepth: int | None = None, + channels: int | None = None, + duration: float | None = None, + ): """ Updates several fields of the Sound object which store some audio properties and saves to DB without updating other fields. This function is used in cases when two instances of the same Sound object could be edited by two processes in parallel and we want to avoid possible field overwrites. - :param int samplerate: samplerate to store - :param int bitrate: bitrate to store - :param int bitdepth: bitdepth to store - :param int channels: number of channels to store - :param float duration: duration to store + :param samplerate: samplerate to store + :param bitrate: bitrate to store + :param bitdepth: bitdepth to store + :param channels: number of channels to store + :param duration: duration to store """ update_fields = [] if samplerate is not None: @@ -1124,11 +1133,11 @@ def set_audio_info_fields(self, samplerate=None, bitrate=None, bitdepth=None, ch update_fields.append("duration") self.save(update_fields=update_fields) - def change_moderation_state(self, new_state): + def change_moderation_state(self, new_state: str): """ Change the moderation state of a sound and perform related tasks such as marking the sound as index dirty or sending a pack to process if required. - :param str new_state: new moderation state to which the sound should be set + :param new_state: new moderation state to which the sound should be set """ current_state = self.moderation_state if current_state != new_state: @@ -1154,13 +1163,13 @@ def change_moderation_state(self, new_state): self.invalidate_template_caches() - def change_processing_state(self, new_state, processing_log=None): + def change_processing_state(self, new_state: str, processing_log: str | None = None): """ Change the processing state of a sound and perform related tasks such as set the sound as index dirty if required. Only the fields that are changed are saved to the object. This is needed when the processing tasks change the processing state of the sound to avoid potential collisions when saving the whole object. - :param str new_state: new processing state to which the sound should be set - :param str processing_log: processing log to be saved in the Sound object + :param new_state: new processing state to which the sound should be set + :param processing_log: processing log to be saved in the Sound object """ current_state = self.processing_state if current_state != new_state: @@ -1192,14 +1201,14 @@ def change_processing_state(self, new_state, processing_log=None): self.invalidate_template_caches() - def change_owner(self, new_owner): + def change_owner(self, new_owner: User): """ Change the owner (i.e. author) of a Sound object by assigning a new User object to the user field. If sound is part of a Pack, when changing the owner a new Pack object is created for the new owner. Changing the owner of the sound also includes renaming and moving all associated files (i.e. sound, previews, displays and analysis) to include the ID of the new owner and be located accordingly. NOTE: see comments in https://github.com/MTG/freesound/issues/750 for more information - :param User new_owner: User object of the new sound owner + :param new_owner: User object of the new sound owner """ def replace_user_id_in_path(path, old_owner_id, new_owner_id): @@ -1407,7 +1416,7 @@ def analyze( def consolidate_analysis( self, - no_db_operations=False, + no_db_operations: bool = False, fail_if_missing=False, verbose=True, existing_analyzer_object_names=None, @@ -1419,7 +1428,7 @@ def consolidate_analysis( audio descriptors data from various analyzers that will be stored in the DB and also used for indexing in the search engine. - :param bool no_db_operations: If True, the method only computes the data but does not save it in the DB. Also it returns + :param no_db_operations: If True, the method only computes the data but does not save it in the DB. Also it returns the consolidated analysis data dictionary instead of the SoundAnalysis object. :param bool fail_if_missing: If True, the method raises an exception if any analyzer data or descriptor data is missing. :param bool verbose: If True, the method logs information about the consolidation process. diff --git a/utils/audioprocessing/freesound_audio_processing.py b/utils/audioprocessing/freesound_audio_processing.py index c8f1aad11..208969a8a 100644 --- a/utils/audioprocessing/freesound_audio_processing.py +++ b/utils/audioprocessing/freesound_audio_processing.py @@ -18,6 +18,7 @@ # See AUTHORS file. # +from __future__ import annotations import json import logging @@ -47,11 +48,11 @@ class WorkerException(Exception): """ -def set_timeout_alarm(time, msg): +def set_timeout_alarm(time: int, msg: str): """ Sets a timeout alarm which raises a WorkerException after a number of seconds. - :param float time: seconds until WorkerException is raised - :param str msg: message to add to WorkerException when raised + :param time: seconds until WorkerException is raised + :param msg: message to add to WorkerException when raised :raises WorkerException: when timeout is reached """ @@ -70,13 +71,14 @@ def cancel_timeout_alarm(): def check_if_free_space( - directory=settings.PROCESSING_TEMP_DIR, min_disk_space_percentage=settings.WORKER_MIN_FREE_DISK_SPACE_PERCENTAGE + directory: str = settings.PROCESSING_TEMP_DIR, + min_disk_space_percentage: float = settings.WORKER_MIN_FREE_DISK_SPACE_PERCENTAGE, ): """ Checks if there is free disk space in the volume of the given 'directory'. If percentage of free disk space in this volume is lower than 'min_disk_space_percentage', this function raises WorkerException. - :param str directory: path of the directory whose volume will be checked for free space - :param float min_disk_space_percentage: free disk space percentage to check against + :param directory: path of the directory whose volume will be checked for free space + :param min_disk_space_percentage: free disk space percentage to check against :raises WorkerException: if available percentage of free space is below the threshold """ stats = os.statvfs(directory) diff --git a/utils/audioprocessing/processing.py b/utils/audioprocessing/processing.py index 2f927ede8..26014afa6 100644 --- a/utils/audioprocessing/processing.py +++ b/utils/audioprocessing/processing.py @@ -20,7 +20,6 @@ # See AUTHORS file. # - import math import os import re diff --git a/utils/downloads.py b/utils/downloads.py index 268f71bdd..ea6f72a36 100644 --- a/utils/downloads.py +++ b/utils/downloads.py @@ -18,10 +18,13 @@ # See AUTHORS file. # +from __future__ import annotations + import datetime import random import zlib +from django.db.models import QuerySet from django.http import HttpResponse from django.utils import timezone @@ -30,19 +33,21 @@ from utils.nginxsendfile import prepare_sendfile_arguments_for_sound_download -def download_sounds(licenses_file_url, licenses_file_content, sounds_list, download_filename): +def download_sounds( + licenses_file_url: str, licenses_file_content: str, sounds_list: QuerySet, download_filename: str +) -> HttpResponse: """From a list of sounds generates the HttpResponse with the information of the wav files of the sounds and a text file with the license. This response is handled by mod_zipfile of nginx to generate a zip file with the content. Args: - licenses_file_url (str): url to the sound Pack or BookmarkCategory licenses - licenses_file_content (str): attributions for the different sounds in the Pack or BookmarkCategory - sounds_list (django.db.models.query.QuerySet): list of sounds forming the Pack or BookmarkCategory - download_filename (str): name of the zip file to be downloaded + licenses_file_url: url to the sound Pack or BookmarkCategory licenses + licenses_file_content: attributions for the different sounds in the Pack or BookmarkCategory + sounds_list: list of sounds forming the Pack or BookmarkCategory + download_filename: name of the zip file to be downloaded Returns: - HttpResponse: information of the wav files of the sounds and a text file with the license + information of the wav files of the sounds and a text file with the license """ license_crc = zlib.crc32(licenses_file_content.encode("UTF-8")) & 0xFFFFFFFF filelist = "%02x %i %s %s\r\n" % ( diff --git a/utils/mail.py b/utils/mail.py index c887f914c..21b0444f9 100644 --- a/utils/mail.py +++ b/utils/mail.py @@ -18,10 +18,17 @@ # See AUTHORS file. # +from __future__ import annotations + import json import logging +from typing import TYPE_CHECKING from django.conf import settings + +if TYPE_CHECKING: + from django.contrib.auth.models import User + from django.core.mail import get_connection from django.core.mail.message import EmailMessage from django.template.loader import render_to_string @@ -45,15 +52,15 @@ def _ensure_list(item): def send_mail( - subject, - email_body, - user_to=None, - email_to=None, - email_from=None, - reply_to=None, - email_type_preference_check=None, - extra_subject="", -): + subject: str, + email_body: str, + user_to: User | list[User] | None = None, + email_to: str | list[str] | None = None, + email_from: str | None = None, + reply_to: str | None = None, + email_type_preference_check: str | None = None, + extra_subject: str = "", +) -> bool: """Sends email with a lot of defaults. The function will check if user's email is valid based on bounce info. The function will also check email @@ -61,22 +68,22 @@ def send_mail( other should be None. Args: - subject (str): subject of the email. - email_body (str): body of the email. - user_to (Union[User, List[User]]): a User object or a list of User objects to send the email to. If user_to + subject: subject of the email. + email_body: body of the email. + user_to: a User object or a list of User objects to send the email to. If user_to is set, email_to should be None. - email_to (Union[str, List[str]]): a string representing an email address or a list of strings representing + email_to: a string representing an email address or a list of strings representing email addresses who to send the email to. If email_to is set, user_to should be None. - email_from (str): email string that shows up as sender. The default value is DEFAULT_FROM_EMAIL in config. - reply_to (str): email string that will be added as a "Reply-To" header to all mails being sent. - email_type_preference_check (str): name of EmailPreferenceType that users should have enabled for the email to + email_from: email string that shows up as sender. The default value is DEFAULT_FROM_EMAIL in config. + reply_to: email string that will be added as a "Reply-To" header to all mails being sent. + email_type_preference_check: name of EmailPreferenceType that users should have enabled for the email to be sent. If set to None, no checks will be carried out. - extra_subject (str): extra contents for the email subject which will be appended to the 'subject' param above + extra_subject: extra contents for the email subject which will be appended to the 'subject' param above and separated by a dash (this should be used to separate parts of the subject which are not common for a given type of email, e.g. to pass the "topic name" in a "topic reply notification" email). Returns: - (bool): True if all emails were sent successfully, False otherwise. + True if all emails were sent successfully, False otherwise. """ assert bool(user_to) != bool(email_to), "One of parameters user_to and email_to should be set, but not both" diff --git a/utils/ratelimit.py b/utils/ratelimit.py index cdca70208..a601dc77e 100644 --- a/utils/ratelimit.py +++ b/utils/ratelimit.py @@ -18,6 +18,8 @@ # See AUTHORS file. # +from __future__ import annotations + import enum import ipaddress import logging @@ -36,7 +38,7 @@ last_cached_blocked_ips_timestamp = 0 -def get_ips_to_block(): +def get_ips_to_block() -> list[str]: """ Get a list of IPs to block. Returned IPs include a combination of the IPs listed in settings.BLOCKED_IPS and a list of IPs cached with the key settings.CACHED_BLOCKED_IPS_KEY. To avoid many queries to the cache, this @@ -45,7 +47,7 @@ def get_ips_to_block(): adds a 2nd layer of cache. Returns: - List[str]: List of IPs to block + List of IPs to block """ global last_cached_blocked_ips, last_cached_blocked_ips_timestamp now = time.time() @@ -58,7 +60,7 @@ def get_ips_to_block(): return settings.BLOCKED_IPS -def add_new_ip_to_block(ip): +def add_new_ip_to_block(ip: str) -> None: """ Add a new IP to the cached list of IPs to block so that further requests to that IP will get blocked. Note that it might take up to settings.CACHED_BLOCKED_IPS_TIME before that IP actually starts to @@ -72,7 +74,7 @@ def add_new_ip_to_block(ip): add_new_ip_to_block("1.2.3.4") Args: - ip (str): IP to block. Can also specify a range as described in ipaddress.IPv4Network docs. + ip: IP to block. Can also specify a range as described in ipaddress.IPv4Network docs. """ try: ipaddress.ip_network(str(ip)) @@ -88,16 +90,16 @@ def add_new_ip_to_block(ip): cache.set(settings.CACHED_BLOCKED_IPS_KEY, cached_ips_to_block) -def ip_is_blocked(ip): +def ip_is_blocked(ip: str) -> bool: """ Determines whether an IP should be blocked. To do that, it uses ipaddress.ip_network objects which support IP comparison using ranges. Args: - ip (str): IP to check. Can also specify a range as described in ipaddress.IPv4Network docs. + ip: IP to check. Can also specify a range as described in ipaddress.IPv4Network docs. Returns: - bool: True if the IP should be blocked, False otherwise. + True if the IP should be blocked, False otherwise. """ for ip_to_block in get_ips_to_block(): diff --git a/utils/search/__init__.py b/utils/search/__init__.py index 077e7767b..caed8b7f8 100644 --- a/utils/search/__init__.py +++ b/utils/search/__init__.py @@ -17,14 +17,21 @@ # Authors: # See AUTHORS file. # +from __future__ import annotations + import importlib +from typing import TYPE_CHECKING from django.conf import settings +if TYPE_CHECKING: + import forum.models + import sounds.models + def get_search_engine( backend_class=settings.SEARCH_ENGINE_BACKEND_CLASS, sounds_index_url=None, forum_index_url=None -) -> "SearchEngineBase": +) -> SearchEngineBase: """Return SearchEngine class instance to carry out search engine actions Args: @@ -33,7 +40,7 @@ def get_search_engine( forum_index_url: url of the forum index in solr. If not set, use the default URL for the backend Returns: - utils.search.SearchEngineBase: search engine backend class instance + search engine backend class instance """ module_name, class_name = backend_class.rsplit(".", 1) module = importlib.import_module(module_name) @@ -43,20 +50,20 @@ def get_search_engine( class SearchResults: def __init__( self, - docs=None, - num_found=-1, - start=-1, - num_rows=-1, - non_grouped_number_of_results=-1, - facets=None, - highlighting=None, - q_time=-1, + docs: list[dict] | None = None, + num_found: int = -1, + start: int = -1, + num_rows: int = -1, + non_grouped_number_of_results: int = -1, + facets: dict[str, list[tuple[str, int]]] | None = None, + highlighting: dict[str, dict] | None = None, + q_time: int = -1, ): """ Class that holds the results of a search query. It must contain the fields defined below. Args: - docs (List[Dict]): list of dictionaries with the results of the query. Each dict object includes information + docs: list of dictionaries with the results of the query. Each dict object includes information about that individual result. That information is typically the ID of the matched object (Eg: the ID of a sound), but it can also include other properties. Here is a list of possible properties: - id (int): ID of the matched object (a sound or forum post). If results are grouped, this will be the ID of @@ -113,12 +120,12 @@ def __init__( 'forum_name_slug': 'test_forum' }, ... ] }, ...] - start (int): offset of the search results query (Eg: return matched documents starting at 15) - num_rows (int): number of results per "page" - num_found (int): total number of matches found - non_grouped_number_of_results (int, optional): total number of non-grouped matches found (it will be + start: offset of the search results query (Eg: return matched documents starting at 15) + num_rows: number of results per "page" + num_found: total number of matches found + non_grouped_number_of_results: total number of non-grouped matches found (it will be the same as num_found for queries which did not group results) - facets (Dict{str:List[Tuple(str,int)]}, optional): data structure including information about the facets + facets: data structure including information about the facets calculated by the search engine. The keys of the main dictionary correspond to the field names of each returned facet. For each facet, a list of tuples is returned with the most common facet elements and their count (sorted in descending count order). Example: @@ -127,7 +134,7 @@ def __init__( 'bitdepth': [('16', 938), ('24', 594), ('0', 298), ('32', 68), ('4', 4)], 'license': [('Attribution', 890), ('Creative Commons 0', 727), ...] } - highlighting (Dict{str:Dict}): datab structure containing "highlighted" contents of the search results (if + highlighting: data structure containing "highlighted" contents of the search results (if any). This could be the content of a forum post with the words that matched the search criteria surrounded by "" tags. @@ -137,7 +144,7 @@ def __init__( '31943': {'post_body': [" the level by 4db. In broadcasting 0db is the pivotal..."] } - q_time (int, optional): time that it took to execute the query in the backend, in ms + q_time: time that it took to execute the query in the backend, in ms """ self.docs = docs if docs is not None else list() self.facets = facets if facets is not None else dict() @@ -167,15 +174,17 @@ class SearchEngineBase: # Sound search related methods - def add_sounds_to_index(self, sound_objects, update=False, include_similarity_vectors=False): + def add_sounds_to_index( + self, sound_objects: list[sounds.models.Sound], update: bool = False, include_similarity_vectors: bool = False + ): """Indexes the provided sound objects in the search index Args: - sound_objects (list[sounds.models.Sound]): Sound objects of the sounds to index - update (bool): Whether to perform an update of the existing documents in the index or to + sound_objects: Sound objects of the sounds to index + update: Whether to perform an update of the existing documents in the index or to completely replace them. An update is useful so that fields not included in the document are not removed from the index. - include_similarity_vectors (bool): Whether to include similarity vectors in the index. + include_similarity_vectors: Whether to include similarity vectors in the index. """ raise NotImplementedError @@ -183,11 +192,11 @@ def update_similarity_vectors_in_index(self, sound_objects): """Create an update document to add only similarity vectors to sounds that already exist in the index""" raise NotImplementedError - def remove_sounds_from_index(self, sound_objects_or_ids): + def remove_sounds_from_index(self, sound_objects_or_ids: list[sounds.models.Sound] | list[int]): """Removes sounds from the search index Args: - sound_objects_or_ids (list[sounds.models.Sound] or list[int]): Sound objects or sound IDs to remove + sound_objects_or_ids: Sound objects or sound IDs to remove """ raise NotImplementedError @@ -195,68 +204,68 @@ def remove_all_sounds(self): """Removes all sounds from the search index""" raise NotImplementedError - def sound_exists_in_index(self, sound_object_or_id): + def sound_exists_in_index(self, sound_object_or_id: sounds.models.Sound | int) -> bool: """Check if a sound is indexed in the search engine Args: - sound_object_or_id (sounds.models.Sound or int): Sound object or sound ID to check if indexed + sound_object_or_id: Sound object or sound ID to check if indexed Returns: - bool: whether the sound is indexed in the search engine + whether the sound is indexed in the search engine """ raise NotImplementedError - def get_all_sound_ids_from_index(self): + def get_all_sound_ids_from_index(self) -> list[int]: """Return a list of all sound IDs indexed in the search engine Returns: - List[int]: list of all sound IDs indexed in the search engine + list of all sound IDs indexed in the search engine """ raise NotImplementedError def search_sounds( self, - textual_query="", - query_fields=None, - query_filter="", - field_list=None, - offset=0, - current_page=None, - num_sounds=settings.SOUNDS_PER_PAGE, - sort=settings.SEARCH_SOUNDS_SORT_OPTION_AUTOMATIC, - group_by_pack=False, - num_sounds_per_pack_group=1, - facets=None, - only_sounds_with_pack=False, - only_sounds_within_ids=False, - group_counts_as_one_in_facets=False, - similar_to=None, - similar_to_min_similarity=settings.SIMILARITY_MIN_THRESHOLD, - similar_to_similarity_space=settings.SIMILARITY_SPACE_DEFAULT, - ): + textual_query: str = "", + query_fields: list[str] | dict[str, int] | None = None, + query_filter: str = "", + field_list: list[str] | None = None, + offset: int = 0, + current_page: int | None = None, + num_sounds: int = settings.SOUNDS_PER_PAGE, + sort: str = settings.SEARCH_SOUNDS_SORT_OPTION_AUTOMATIC, + group_by_pack: bool = False, + num_sounds_per_pack_group: int = 1, + facets: dict[str, dict] | None = None, + only_sounds_with_pack: bool = False, + only_sounds_within_ids: list[int] | bool = False, + group_counts_as_one_in_facets: bool = False, + similar_to: int | list[float] | None = None, + similar_to_min_similarity: float = settings.SIMILARITY_MIN_THRESHOLD, + similar_to_similarity_space: str = settings.SIMILARITY_SPACE_DEFAULT, + ) -> SearchResults: """Search for sounds that match specific criteria and return them in a SearchResults object Args: - textual_query (str, optional): the textual query - query_fields (List[str] or Dict{str: int}, optional): a list of the fields that should be matched when + textual_query: the textual query + query_fields: a list of the fields that should be matched when querying. Field weights can also be specified if a dict is passed with keys as field names and values as weights. Field names should use the names defined in settings.SEARCH_SOUNDS_FIELD_*. Eg: query_fields = [settings.SEARCH_SOUNDS_FIELD_ID, settings.SEARCH_SOUNDS_FIELD_USER_NAME] query_fields = {settings.SEARCH_SOUNDS_FIELD_ID:1 , settings.SEARCH_SOUNDS_FIELD_USER_NAME: 4} - query_filter (str, optional): filter expression following lucene filter syntax - field_list (List[str], optional): list of fields to return by the search engine. Typically we're only interested + query_filter: filter expression following lucene filter syntax + field_list: list of fields to return by the search engine. Typically we're only interested in sound IDs because we don't use data form the search engine to display sounds, but in some cases it can be necessary to return further data. - offset (int, optional): offset for the returned results - current_page (int, optional): alternative way to set offset using page numbers. Using current_page will + offset: offset for the returned results + current_page: alternative way to set offset using page numbers. Using current_page will set offset like offset=current_page*num_sounds - num_sounds (int, optional): number of sounds to return - sort (str, optional): sorting criteria. should be one of settings.SEARCH_SOUNDS_SORT_OPTIONS_WEB - group_by_pack (bool, optional): whether the search results should be grouped by sound pack. When grouped + num_sounds: number of sounds to return + sort: sorting criteria. should be one of settings.SEARCH_SOUNDS_SORT_OPTIONS_WEB + group_by_pack: whether the search results should be grouped by sound pack. When grouped by pack, only "num_sounds_per_pack_group" sounds per pack will be returned, together with additional information about the number of other sounds in the pack that would be i the same group. - num_sounds_per_pack_group (int, optional): number of sounds to return per pack group (minimum one) - facets (Dict{str: Dict}, optional): information about facets to be returned. Can be None if no faceting + num_sounds_per_pack_group: number of sounds to return per pack group (minimum one) + facets: information about facets to be returned. Can be None if no faceting information is required. Facets should be specified as a dictionary with the "db" field names to be included in the faceting as keys, and a dictionary as values with optional specific parameters for every field facet. Field names should use the names defined in settings.SEARCH_SOUNDS_FIELD_*. Eg: @@ -267,56 +276,56 @@ def search_sounds( } Supported individual facet options include: - limit: the number of items returned per facet - only_sounds_with_pack (bool, optional): whether to only include sounds that belong to a pack - only_sounds_within_ids (List[int], optional): restrict search results to sounds with these IDs - group_counts_as_one_in_facets (bool, optional): whether only one result from a group should be counted + only_sounds_with_pack: whether to only include sounds that belong to a pack + only_sounds_within_ids: restrict search results to sounds with these IDs + group_counts_as_one_in_facets: whether only one result from a group should be counted when computing facets or all should be taken into account. This is used to reduce the influence of large groups in facets. We use it for computing the main tag cloud and avoiding a large packs of sounds with the same tags to largely influence the general tag cloud (only one sound of the pack will be counted) - similar_to (int or List[float], optional): sound ID or similarity vector to be used as target for similarity + similar_to: sound ID or similarity vector to be used as target for similarity search. Note that when this parameter is passed, some of the other parameters will be ignored ('textual_query', 'facets', 'group_by_pack', 'num_sounds_per_pack_group', 'group_counts_as_one_in_facets'). 'query_filter' should still be usable, although this remains to be thoroughly tested. - similar_to_min_similarity (float, optional): min similarity score to consider a sound as similar. - similar_to_similarity_space (str, optional): similarity space from which to select similarity vectors for + similar_to_min_similarity: min similarity score to consider a sound as similar. + similar_to_similarity_space: similarity space from which to select similarity vectors for similarity search. It defaults to settings.SIMILARITY_SPACE_DEFAULT, but it can be changed to something else if we want to use a different type of similarity vectors for a similarity search query. Returns: - SearchResults: SearchResults object containing the results of the query + SearchResults object containing the results of the query """ raise NotImplementedError - def get_random_sound_id(self): + def get_random_sound_id(self) -> int: """Return the id of a random sound from the search engine. This is used for random sound browsing. We filter explicit sounds, but otherwise don't have any other restrictions on sound attributes. Returns: - int: the ID of the selected random sound (or 0 if there were errors) + the ID of the selected random sound (or 0 if there were errors) """ raise NotImplementedError - def get_num_sim_vectors_indexed_per_similarity_space(self): + def get_num_sim_vectors_indexed_per_similarity_space(self) -> dict: """Returns the number of similarity vectors indexed in the search engine for each similarity space. Because there might be several similarity vectors per sound, we distinguish between the total number of similarity vectors and the total number of sounds per similarity space. Returns: - dict: dictionary with the number of similarity vectors and number of sounds indexed per similarity space. + dictionary with the number of similarity vectors and number of sounds indexed per similarity space. E.g.: {'freesound_classic': {'num_sounds': 0, 'num_vectors': 0}, 'laion_clap': {'num_sounds': 15876, 'num_vectors': 25448}} """ raise NotImplementedError - def get_all_sim_vector_document_ids_per_similarity_space(self): + def get_all_sim_vector_document_ids_per_similarity_space(self) -> dict: """Returns indexed Solr document IDs for all similarity vector documents for each similarity space. Solr document IDs for similarity vector documents have the format: "simvec__" Returns: - dict: dictionary with a list of Solr document IDs per similarity space. + dictionary with a list of Solr document IDs per similarity space. E.g.: {'freesound_classic': [693610/similarity_vectors#0, 693610/similarity_vectors#1, ...], 'laion_clap': [1234/similarity_vectors#0, 1235/similarity_vectors#0, ...]} """ @@ -324,19 +333,19 @@ def get_all_sim_vector_document_ids_per_similarity_space(self): # Forum search related methods - def add_forum_posts_to_index(self, forum_post_objects): + def add_forum_posts_to_index(self, forum_post_objects: list[forum.models.Post]): """Indexes the provided forum post objects in the search index Args: - forum_post_objects (list[forum.models.Post]): Post objects of the forum posts to index + forum_post_objects: Post objects of the forum posts to index """ raise NotImplementedError - def remove_forum_posts_from_index(self, forum_post_objects_or_ids): + def remove_forum_posts_from_index(self, forum_post_objects_or_ids: list[forum.models.Post] | list[int]): """Removes forum posts from the search index Args: - forum_post_objects_or_ids (list[forum.models.Post] or list[int]): Post objects or post IDs to remove + forum_post_objects_or_ids: Post objects or post IDs to remove """ raise NotImplementedError @@ -344,63 +353,63 @@ def remove_all_forum_posts(self): """Removes all forum posts from the search index""" raise NotImplementedError - def forum_post_exists_in_index(self, forum_post_object_or_id): + def forum_post_exists_in_index(self, forum_post_object_or_id: forum.models.Post | int) -> bool: """Check if a sound is indexed in the search engine Args: - forum_post_object_or_id (forum.models.Post or int): Post object or post ID to check if indexed + forum_post_object_or_id: Post object or post ID to check if indexed Returns: - bool: whether the post is indexed in the search engine + whether the post is indexed in the search engine """ raise NotImplementedError def search_forum_posts( self, - textual_query="", - query_filter="", - offset=0, - sort=None, - current_page=None, - num_posts=settings.FORUM_POSTS_PER_PAGE, - group_by_thread=True, - ): + textual_query: str = "", + query_filter: str = "", + offset: int = 0, + sort: str | None = None, + current_page: int | None = None, + num_posts: int = settings.FORUM_POSTS_PER_PAGE, + group_by_thread: bool = True, + ) -> SearchResults: """Search for forum posts that match specific criteria and return them in a SearchResults object Args: - textual_query (str, optional): the textual query - query_filter (str, optional): filter expression following lucene filter syntax - offset (int, optional): offset for the returned results - sort (str, optional): sorting criteria. should be one of settings.SEARCH_FORUM_SORT_OPTIONS_WEB - current_page (int, optional): alternative way to set offset using page numbers. Using current_page will + textual_query: the textual query + query_filter: filter expression following lucene filter syntax + offset: offset for the returned results + sort: sorting criteria. should be one of settings.SEARCH_FORUM_SORT_OPTIONS_WEB + current_page: alternative way to set offset using page numbers. Using current_page will set offset like offset=current_page*num_sounds - num_posts (int, optional): number of forum posts to return - group_by_thread (bool, optional): whether the search results should be grouped by forum post thread. When + num_posts: number of forum posts to return + group_by_thread: whether the search results should be grouped by forum post thread. When grouped by thread, all matching results per every thread will be returned following the structure defined in SearchResults. Note that this is different than the group_by_pack option of search_sounds, with which only 1 result is returned per group. Returns: - SearchResults: SearchResults object containing the results of the query + SearchResults object containing the results of the query """ raise NotImplementedError # Tag clouds methods - def get_user_tags(self, username): + def get_user_tags(self, username) -> list[tuple[str, int]]: """Retrieves the tags used by a user and their counts Args: username: name of the user for which we want to know tags and counts Returns: - List[Tuple(str, int)]: List of tuples with the tags and counts of the tags used by the user. + List of tuples with the tags and counts of the tags used by the user. Eg: [('cat', 1), ('echo', 1), ('forest', 1)] """ raise NotImplementedError - def get_pack_tags(self, username, pack_name): + def get_pack_tags(self, username, pack_name) -> list[tuple[str, int]]: """Retrieves the tags in the sounds of a pack and their counts Args: @@ -408,7 +417,7 @@ def get_pack_tags(self, username, pack_name): pack_name: name of the pack for which tags and counts should be retrieved Returns: - List[Tuple(str, int)]: List of tuples with the tags and counts of the tags in the pack. + List of tuples with the tags and counts of the tags in the pack. Eg: [('cat', 1), ('echo', 1), ('forest', 1)] """ raise NotImplementedError diff --git a/utils/search/backends/solr9pysolr.py b/utils/search/backends/solr9pysolr.py index 21efc6e29..36f9baaf4 100644 --- a/utils/search/backends/solr9pysolr.py +++ b/utils/search/backends/solr9pysolr.py @@ -20,6 +20,8 @@ # See AUTHORS file. # +from __future__ import annotations + import re import pysolr @@ -62,7 +64,9 @@ def get_forum_index(self, timeout=settings.SEARCH_SOLR_TIMEOUT_SECONDS): timeout=timeout, ) - def search_process_filter(self, query_filter, only_sounds_within_ids=False, only_sounds_with_pack=False): + def search_process_filter( + self, query_filter: str, only_sounds_within_ids: list[int] | bool = False, only_sounds_with_pack: bool = False + ) -> str: """Process the filter to make a number of adjustments 1) Add type suffix to human-readable audio analyzer descriptor names (needed for dynamic solr field names). @@ -77,12 +81,12 @@ def search_process_filter(self, query_filter, only_sounds_within_ids=False, only suffices to the filter names so users do not need to deal with that and Solr understands recognizes the field name. Args: - query_filter (str): query filter string. - only_sounds_with_pack (bool, optional): whether to only include sounds that belong to a pack - only_sounds_within_ids (List[int], optional): restrict search results to sounds with these IDs + query_filter: query filter string. + only_sounds_with_pack: whether to only include sounds that belong to a pack + only_sounds_within_ids: restrict search results to sounds with these IDs Returns: - str: processed filter query string. + processed filter query string. """ # Add type suffix to human-readable audio analyzer descriptor names which is needed for solr dynamic fields query_filter = self.add_solr_suffix_to_dynamic_fieldnames_in_filter(query_filter) diff --git a/utils/search/search_forum.py b/utils/search/search_forum.py index 0f6bc9c1e..605b6cdeb 100644 --- a/utils/search/search_forum.py +++ b/utils/search/search_forum.py @@ -17,22 +17,29 @@ # Authors: # See AUTHORS file. # + +from __future__ import annotations + import logging +from typing import TYPE_CHECKING from utils.search import SearchEngineException, get_search_engine +if TYPE_CHECKING: + from forum.models import Post + search_logger = logging.getLogger("search") console_logger = logging.getLogger("console") -def add_posts_to_search_engine(post_objects, solr_collection_url=None): +def add_posts_to_search_engine(post_objects: list[Post], solr_collection_url=None) -> int: """Add forum posts to search engine Args: - post_objects (List[forum.models.Post]): list (or queryset) of forum Post objects to index + post_objects: list (or queryset) of forum Post objects to index Returns: - int: number of sounds added to the index + number of sounds added to the index """ num_posts = len(post_objects) try: @@ -44,11 +51,11 @@ def add_posts_to_search_engine(post_objects, solr_collection_url=None): return 0 -def delete_posts_from_search_engine(post_ids, solr_collection_url=None): +def delete_posts_from_search_engine(post_ids: list[int], solr_collection_url=None): """Delete forum posts from the search engine Args: - post_ids (list[int]): IDs of the forum posts to delete + post_ids: IDs of the forum posts to delete """ console_logger.info(f"Deleting {len(post_ids)} forum posts from search engine") search_logger.info(f"Deleting {len(post_ids)} forum posts from search engine") @@ -67,14 +74,14 @@ def delete_all_posts_from_search_engine(solr_collection_url=None): console_logger.info(f"Could not delete forum posts: {str(e)}") -def get_all_post_ids_from_search_engine(solr_collection_url=None, page_size=2000): +def get_all_post_ids_from_search_engine(solr_collection_url=None, page_size=2000) -> list[int]: """Retrieves the list of all forum post IDs currently indexed in the search engine Args: page_size: number of post IDs to retrieve per search engine query Returns: - list[int]: list of forum IDs indexed in the search engine + list of forum IDs indexed in the search engine """ console_logger.info("Getting all forum post ids from search engine") search_engine = get_search_engine(forum_index_url=solr_collection_url) diff --git a/utils/search/search_query_processor.py b/utils/search/search_query_processor.py index 270a70cd9..2bad369f8 100644 --- a/utils/search/search_query_processor.py +++ b/utils/search/search_query_processor.py @@ -18,9 +18,11 @@ # See AUTHORS file. # +from __future__ import annotations import json import urllib.parse +from typing import TYPE_CHECKING import luqum.exceptions import luqum.tree @@ -39,6 +41,9 @@ from utils.search.filter_validation import parse_filter, validate_filter_types from utils.search.search_sounds import allow_beta_search_features +if TYPE_CHECKING: + from django.http import HttpRequest + from .search_query_processor_options import ( SearchOptionBool, SearchOptionBoolElementInPath, @@ -311,12 +316,12 @@ class SearchQueryProcessor: ), ] - def __init__(self, request, facets=None): + def __init__(self, request: HttpRequest, facets: dict | None = None): """Initializes the SearchQueryProcessor object by parsing data from the request and setting up search options. Args: - request (django.http.HttpRequest): request object from which to parse search options - facets (dict, optional): dictionary with facet options to be used in the search. If not provided, default + request: request object from which to parse search options + facets: dictionary with facet options to be used in the search. If not provided, default facets will be used. Default is None. """ @@ -464,24 +469,24 @@ def __init__(self, request, facets=None): # Filter-related methods def get_active_filters( self, - include_filters_from_options=True, - include_non_option_filters=True, - include_filters_from_facets=True, - extra_filters=None, - ignore_filters=None, + include_filters_from_options: bool = True, + include_non_option_filters: bool = True, + include_filters_from_facets: bool = True, + extra_filters: list | None = None, + ignore_filters: list | None = None, ): """Returns a list of all filters which are active in the query in a ["field:value", "field:value", ...] format. This method also allows to add extra filters to the list or ignore some of the existing filters. Args: - include_filters_from_options (bool, optional): If True, filters from search options will be included. Default is True. - include_non_option_filters (bool, optional): If True, filters from non-option filters will be included. Default is True. - include_filters_from_facets (bool, optional): If True, filters from search facets will be included. Note that if + include_filters_from_options: If True, filters from search options will be included. Default is True. + include_non_option_filters: If True, filters from non-option filters will be included. Default is True. + include_filters_from_facets: If True, filters from search facets will be included. Note that if include_non_option_filters is set to False, include_filters_from_facets will have no effect as facet filters are part of non-option filters. Default is True. - extra_filters (list, optional): List of extra filters to be added. Each filter should be a string in the format "field:value", + extra_filters: List of extra filters to be added. Each filter should be a string in the format "field:value", e.g.: extra_filters=["tag:tagname"]. Default is None. - ignore_filters (list, optional): List of filters to be ignored. Each filter should be a string in the format "field:value", + ignore_filters: List of filters to be ignored. Each filter should be a string in the format "field:value", e.g.: ignore_filters=["tag:tagname"]. Default is None. """ # Create initial list of the active filters according to the types of filters that are requested to be included @@ -628,7 +633,7 @@ def contains_active_advanced_search_options(self): return True return False - def get_clustering_data_cache_key(self, include_filters_from_facets=False): + def get_clustering_data_cache_key(self, include_filters_from_facets: bool = False) -> str: """Generates a cache key used to store clustering results in the cache. Note that the key excludes facet filters by default because clusters are computed on the subset of results BEFORE applying the facet filters (this is by design to avoid recomputing clusters when changing facets). However, the key can be generated including facets as @@ -636,12 +641,12 @@ def get_clustering_data_cache_key(self, include_filters_from_facets=False): are applied after the main clustering computation. Args: - include_filters_from_facets (bool): If True, the key will include filters from facets as well. Default is False. + include_filters_from_facets: If True, the key will include filters from facets as well. Default is False. Filters that are included in facets correspond to the facet fields defined in self.facets, which defaults to settings.SEARCH_SOUNDS_DEFAULT_FACETS. Returns: - str: Cache key for the clustering data + Cache key for the clustering data """ query_filter = self.get_filter_string_for_search_engine(include_filters_from_facets=include_filters_from_facets) key = ( @@ -682,14 +687,14 @@ def print(self): for filter in self.non_option_filters: print("-", f"{filter[0]}={filter[1]}") - def as_query_params(self, exclude_facet_filters=False): + def as_query_params(self, exclude_facet_filters: bool = False) -> dict: """Returns a dictionary with the search options and filters to be used as parameters for the SearchEngine.search_sounds method. This method post-processes the data loaded into the SearchQueryProcessor to generate an appropriate query_params dict. Note that this method includes some complex logic that takes into account the interaction with some option values to calculate the query_params values to be used by the search engine. Args: - exclude_facet_filters (bool, optional): If True, facet filters will not be used to create the query_params dict. Default is False. + exclude_facet_filters: If True, facet filters will not be used to create the query_params dict. Default is False. This is useful as part of the clustering features for which we want to make a query which ignores the facet filters provided in the URL. Returns: @@ -791,15 +796,15 @@ def as_query_params(self, exclude_facet_filters=False): else settings.SIMILARITY_SPACE_LAION_CLAP, ) - def get_url(self, add_filters=None, remove_filters=None): + def get_url(self, add_filters: list | None = None, remove_filters: list | None = None): """Returns the URL of the search page (or tags page, see below) corresponding to the current parameters loaded in the SearchQueryProcessor. This method also has parameters to "add_filters" and "remove_filters", which will return the URL to the search page corresponding to the current parameters loaded in the SearchQueryProcessor BUT with some filters added or removed. Args: - add_filters (list, optional): List of filters to be added. Each filter should be a string in the format "field:value", + add_filters: List of filters to be added. Each filter should be a string in the format "field:value", e.g.: add_filters=["tag:tagname"]. Default is None. - remove_filters (list, optional): List of filters to be ignored. Each filter should be a string in the format "field:value", + remove_filters: List of filters to be ignored. Each filter should be a string in the format "field:value", e.g.: remove_filters=["tag:tagname"]. Default is None. """ # Decide the base url (if in the tags page, we'll use the base URL for tags, otherwise we use the one for the normal search page) diff --git a/utils/search/search_query_processor_options.py b/utils/search/search_query_processor_options.py index 7bfc141aa..03730518b 100644 --- a/utils/search/search_query_processor_options.py +++ b/utils/search/search_query_processor_options.py @@ -18,6 +18,11 @@ # See AUTHORS file. # +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + from django.conf import settings @@ -36,33 +41,33 @@ class SearchOption(object): def __init__( self, - advanced=True, - label="", - help_text="", - placeholder="", - search_engine_field_name=None, - query_param_name=None, - value_default=None, - get_default_value=None, - should_be_disabled=None, - get_value_to_apply=None, + advanced: bool = True, + label: str = "", + help_text: str = "", + placeholder: str = "", + search_engine_field_name: str | None = None, + query_param_name: str | None = None, + value_default: Any = None, + get_default_value: Callable | None = None, + should_be_disabled: Callable | None = None, + get_value_to_apply: Callable | None = None, ): """Initialize the SearchOption object. Args: - advanced (bool, optional): Whether this option is part of the advanced search options (defaults to True). - label (str, optional): Label to be used in the frontend template when displaying the option. - help_text (str, optional): Help text to be used in the frontend template when displaying the option. - placeholder (str, optional): Placeholder text to be used in input elements of the frontend template when displaying the option. - search_engine_field_name (str, optional): Field name of the search engine index corresponding to this option (can be None). - query_param_name (str, optional): Name to represent this option in the URL query parameters (can be None). - value_default (any, optional): Value of the option to be used when not set in the request (as valid Python type). Note that + advanced: Whether this option is part of the advanced search options (defaults to True). + label: Label to be used in the frontend template when displaying the option. + help_text: Help text to be used in the frontend template when displaying the option. + placeholder: Placeholder text to be used in input elements of the frontend template when displaying the option. + search_engine_field_name: Field name of the search engine index corresponding to this option (can be None). + query_param_name: Name to represent this option in the URL query parameters (can be None). + value_default: Value of the option to be used when not set in the request (as valid Python type). Note that this value can be overriden if passing the get_default_value optional parameter. - get_default_value (function, optional): A function returning the default value (as a valid Python type) that the option should take + get_default_value: A function returning the default value (as a valid Python type) that the option should take if not set in the request. The function will be passed the SearchOption itself as an argument. - should_be_disabled (function, optional): Function to determine if the option should be disabled based on the data of the search query. + should_be_disabled: Function to determine if the option should be disabled based on the data of the search query. The function will be passed the SearchOption itself as an argument. - get_value_to_apply (function, optional): Function to determine the value to be used when applying the option in the search engine. + get_value_to_apply: Function to determine the value to be used when applying the option in the search engine. The function will be passed the SearchOption itself as an argument. """ self.advanced = advanced @@ -258,9 +263,9 @@ class SearchOptionChoice(SearchOptionStr): Django forms. """ - def __init__(self, choices=[], **kwargs): + def __init__(self, choices: list = [], **kwargs): """Args: - choices (list): List of available choices in the format [(value, label), ...]. + choices: List of available choices in the format [(value, label), ...]. """ self.choices = choices super().__init__(**kwargs) @@ -284,10 +289,10 @@ class SearchOptionMultipleChoice(SearchOption): value_default = [] - def __init__(self, choices=[], query_param_name_prefix="", **kwargs): + def __init__(self, choices: list = [], query_param_name_prefix: str = "", **kwargs): """Args: - choices (list): List of available choices in the format [(value, label), ...]. - query_param_name_prefix (str): Prefix to be used in the URL parameters to represent the multiple choices. + choices: List of available choices in the format [(value, label), ...]. + query_param_name_prefix: Prefix to be used in the URL parameters to represent the multiple choices. """ self.choices = choices self.query_param_name_prefix = query_param_name_prefix @@ -327,10 +332,10 @@ class SearchOptionRange(SearchOption): query_param_min = None query_param_max = None - def __init__(self, query_param_min=None, query_param_max=None, **kwargs): + def __init__(self, query_param_min: str | None = None, query_param_max: str | None = None, **kwargs): """Args: - query_param_min (str, optional): Name of the URL parameter to represent the minimum value of the range. - query_param_max (str, optional): Name of the URL parameter to represent the maximum value of the range. + query_param_min: Name of the URL parameter to represent the minimum value of the range. + query_param_max: Name of the URL parameter to represent the maximum value of the range. """ self.query_param_min = query_param_min self.query_param_max = query_param_max @@ -377,9 +382,9 @@ class SearchOptionBoolElementInPath(SearchOptionBool): The "element_in_path" is compared with the request path and the value of the option is set to True if the element is present. """ - def __init__(self, element_in_path="", **kwargs): + def __init__(self, element_in_path: str = "", **kwargs): """Args: - element_in_path (str): Element to be checked in the request path. + element_in_path: Element to be checked in the request path. """ self.element_in_path = element_in_path super().__init__(**kwargs) diff --git a/utils/search/search_sounds.py b/utils/search/search_sounds.py index 7e1bb5183..fd344874b 100644 --- a/utils/search/search_sounds.py +++ b/utils/search/search_sounds.py @@ -18,6 +18,8 @@ # See AUTHORS file. # +from __future__ import annotations + import logging from django.conf import settings @@ -77,14 +79,14 @@ def perform_search_engine_query(query_params: dict) -> tuple[SearchResults, PreS def add_sounds_to_search_engine( - sound_objects: list["sounds.models.Sound"], update=False, include_similarity_vectors=False, solr_collection_url=None -): + sound_objects: list[sounds.models.Sound], update=False, include_similarity_vectors=False, solr_collection_url=None +) -> int: """Add the Sounds from the queryset to the search engine Args: sound_objects: list (or queryset) of Sound objects to index Returns: - int: number of sounds added to the index + number of sounds added to the index """ if isinstance(sound_objects, RawQuerySet): num_sounds = len(list(sound_objects)) @@ -104,14 +106,14 @@ def add_sounds_to_search_engine( def send_update_similarity_vectors_in_search_engine( - sound_objects: list["sounds.models.Sound"], solr_collection_url=None -): + sound_objects: list[sounds.models.Sound], solr_collection_url=None +) -> int: """Update the similarity vectors for the Sounds from the queryset in the search engine Args: sound_objects: list (or queryset) of Sound objects to index Returns: - int: number of sounds added to the index + number of sounds added to the index """ if isinstance(sound_objects, RawQuerySet): num_sounds = len(list(sound_objects)) @@ -128,11 +130,11 @@ def send_update_similarity_vectors_in_search_engine( return 0 -def delete_sounds_from_search_engine(sound_ids, solr_collection_url=None): +def delete_sounds_from_search_engine(sound_ids: list[int], solr_collection_url=None): """Delete sounds from the search engine Args: - sound_ids (list[int]): IDs of the sounds to delete + sound_ids: IDs of the sounds to delete """ console_logger.info(f"Deleting {len(sound_ids)} sounds from search engine") search_logger.info(f"Deleting {len(sound_ids)} sounds from search engine") @@ -154,14 +156,14 @@ def delete_all_sounds_from_search_engine(solr_collection_url=None): search_logger.info(f"Could not delete sounds: {str(e)}") -def get_all_sound_ids_from_search_engine(solr_collection_url=None): +def get_all_sound_ids_from_search_engine(solr_collection_url=None) -> list[int]: """Retrieves the list of all sound IDs currently indexed in the search engine Args: page_size: number of sound IDs to retrieve per search engine query Returns: - list[int]: list of sound IDs indexed in the search engine + list of sound IDs indexed in the search engine """ console_logger.info("Getting all sound ids from search engine") search_engine = get_search_engine(sounds_index_url=solr_collection_url) @@ -172,7 +174,7 @@ def get_all_sound_ids_from_search_engine(solr_collection_url=None): return [] -def get_all_sim_vector_sound_ids_from_search_engine(solr_collection_url=None): +def get_all_sim_vector_sound_ids_from_search_engine(solr_collection_url=None) -> dict[str, list[int]]: """Retrieves the list of all sound IDs with similarity vectors for all similarity spaces currently indexed in the search engine @@ -180,7 +182,7 @@ def get_all_sim_vector_sound_ids_from_search_engine(solr_collection_url=None): page_size: number of sound IDs to retrieve per search engine query Returns: - dict[list]: list of sound IDs with per similarity space indexed in the search engine + list of sound IDs with per similarity space indexed in the search engine """ console_logger.info("Getting all sound ids with similarity vectors from search engine") search_engine = get_search_engine(sounds_index_url=solr_collection_url) @@ -207,21 +209,24 @@ def get_random_sound_id_from_search_engine(solr_collection_url=None): def get_sound_similarity_vectors_from_search_engine_query( - query_params, similarity_space=settings.SIMILARITY_SPACE_DEFAULT, current_page=None, num_sounds=None -): + query_params: dict, + similarity_space: str = settings.SIMILARITY_SPACE_DEFAULT, + current_page: int | None = None, + num_sounds: int | None = None, +) -> dict: """Gets the similarity vectors for the first "num_results" sounds for the given query. Args: - query_params (dict): query parameters dictionary with parameters following the specification of search_sounds + query_params: query parameters dictionary with parameters following the specification of search_sounds function from utils.search.SearchEngine. - similarity_space (str): name of the similarity space from which to get the vector - current_page (int): page number of the results to retrieve similarity vectors for. If None, the current page + similarity_space: name of the similarity space from which to get the vector + current_page: page number of the results to retrieve similarity vectors for. If None, the current page from query_params will be used. - num_sounds (int): number of sounds to retrieve similarity vectors for. If None, the number of sounds + num_sounds: number of sounds to retrieve similarity vectors for. If None, the number of sounds in the query_params will be used. Returns: - dict: dictionary with sound IDs as keys and similarity vectors as values + dictionary with sound IDs as keys and similarity vectors as values """ # Update query params to get similarity vectors of the first config_options = settings.SIMILARITY_SPACES[similarity_space] @@ -260,18 +265,20 @@ def get_sound_similarity_vectors_from_search_engine_query( return similarity_vectors_map -def get_sound_ids_from_search_engine_query(query_params, current_page=None, num_sounds=None): +def get_sound_ids_from_search_engine_query( + query_params: dict, current_page: int | None = None, num_sounds: int | None = None +) -> list[int]: """Performs Solr query and returns results as a list of sound ids. Args: - query_params (dict): contains the query parameters to replicate the user query. - current_page (int): page number of the results to retrieve IDs for. If None, the current page + query_params: contains the query parameters to replicate the user query. + current_page: page number of the results to retrieve IDs for. If None, the current page from query_params will be used. - num_sounds (int): number of sounds to retrieve IDs for. If None, the number of sounds + num_sounds: number of sounds to retrieve IDs for. If None, the number of sounds in the query_params will be used. - Returns - List[int]: list containing the ids of the retrieved sounds (for the current_page or num_sounds). + Returns: + list containing the ids of the retrieved sounds (for the current_page or num_sounds). """ # We set include_facets to False in order to reduce the amount of data that search engine will return. query_params.update( diff --git a/utils/sound_upload.py b/utils/sound_upload.py index 1acdf1c3a..94d0e8a8c 100644 --- a/utils/sound_upload.py +++ b/utils/sound_upload.py @@ -17,6 +17,8 @@ # Authors: # See AUTHORS file. # +from __future__ import annotations + import csv import hashlib import json @@ -24,6 +26,7 @@ import os import shutil from collections import defaultdict +from typing import TYPE_CHECKING import openpyxl import xlrd @@ -45,6 +48,10 @@ ) from utils.text import remove_control_chars +if TYPE_CHECKING: + from apiv2.models import ApiV2Client + from sounds.models import BulkUploadProgress, Sound + console_logger = logging.getLogger("console") sounds_logger = logging.getLogger("sounds") @@ -119,26 +126,33 @@ def get_processing_before_describe_sound_base_url(audio_file_path): return settings.PROCESSING_BEFORE_DESCRIPTION_URL + relative + "/" -def create_sound(user, sound_fields, apiv2_client=None, bulk_upload_progress=None, process=True, remove_exists=False): +def create_sound( + user: User, + sound_fields: dict, + apiv2_client: ApiV2Client | None = None, + bulk_upload_progress: BulkUploadProgress | None = None, + process: bool = True, + remove_exists: bool = False, +) -> Sound: """ This function is used to create sound objects uploaded via the sound describe form, the API or the bulk describe feature. Args: - user (User): user that will appear as the uploader of the sound (author) - sound_fields (dict): dictionary with data to populate the different fields of the sound object. Check example + user: user that will appear as the uploader of the sound (author) + sound_fields: dictionary with data to populate the different fields of the sound object. Check example usages of create_sound for more information about what are these fields and their expected format - apiv2_client (ApiV2Client): ApiV2Client object corresponding to the API account that triggered the creation + apiv2_client: ApiV2Client object corresponding to the API account that triggered the creation of that sound object (if not provided, will be set to None) - bulk_upload_progress (BulkUploadProgress): BulkUploadProgress object corresponding to the bulk upload progress + bulk_upload_progress: BulkUploadProgress object corresponding to the bulk upload progress that triggered the creation of this sound object (if not provided, will be set to None) - process (bool): whether to trigger processing and analysis of the sound object after being created + process: whether to trigger processing and analysis of the sound object after being created (defaults to True) - remove_exists (bool): if the sound we're trying to create an object for already exists (according to + remove_exists: if the sound we're trying to create an object for already exists (according to md5 check), delete it (defaults to False) Returns: - Sound: returns the created Sound object + The created Sound object. """ # Import models using apps.get_model (to avoid circular dependencies) diff --git a/utils/tags.py b/utils/tags.py index c170b0700..bed64ebbb 100644 --- a/utils/tags.py +++ b/utils/tags.py @@ -18,6 +18,8 @@ # See AUTHORS file. # +from __future__ import annotations + import re @@ -35,18 +37,20 @@ def annotate(dictionary, **kwargs): return x -def annotate_tags(tags, sort=None, small_size=0.7, large_size=1.8): +def annotate_tags( + tags: list[dict], sort: str | None = None, small_size: float = 0.7, large_size: float = 1.8 +) -> list[dict]: """ Process a list of tags with counts and annotate it with computed size. Size will be proportional to count. Args: - tags (List[dict]): list of dictionaries with the tag "name" and "count" (see example below) - sort (str or None): whether to sort the annotated list by "name", "count" or None - small_size (float): smallest annotated size - large_size (float): highest annotated range + tags: list of dictionaries with the tag "name" and "count" (see example below) + sort: whether to sort the annotated list by "name", "count" or None + small_size: smallest annotated size + large_size: highest annotated range Returns: - List[dict]: list of dictionaries with the tag "name", "count" and "size" + list of dictionaries with the tag "name", "count" and "size" For example, if tags are given as: [ {"name": "tag1", "count": 1}, {"name": "tag2", "count": 200}, {"name": "tag3", "count": 200}] diff --git a/utils/test_helpers.py b/utils/test_helpers.py index f83fe37dc..d92b7360c 100644 --- a/utils/test_helpers.py +++ b/utils/test_helpers.py @@ -18,6 +18,8 @@ # See AUTHORS file. # +from __future__ import annotations + import os import pickle from functools import partial, wraps @@ -65,18 +67,18 @@ def create_test_files( def create_user_and_sounds( - num_sounds=1, - num_packs=0, - user=None, - count_offset=0, - bst_category="ss-n", - tags=None, - description=None, - processing_state="PE", - moderation_state="PE", - type="wav", - username="testuser", -): + num_sounds: int = 1, + num_packs: int = 0, + user: User | None = None, + count_offset: int = 0, + bst_category: str | None = "ss-n", + tags: str | None = None, + description: str | None = None, + processing_state: str = "PE", + moderation_state: str = "PE", + type: str = "wav", + username: str = "testuser", +) -> tuple[User, list[Pack], list[Sound]]: """Creates User, Sound and Pack objects useful for testing. A counter is used to make sound names unique as well as other fields like md5 (see `sound_counter` variable). @@ -84,19 +86,19 @@ def create_user_and_sounds( 'licenses' fixture, i.e. "fixtures = ['licenses']". Args: - num_sounds (int): N sounds to generate. - num_packs (int): N packs in which the sounds above will be grouped. - user (User): user owner of the created sounds (if not provided, a new user will be created). - count_offset (int): start counting sounds at X. - bst_category (str or None): category code to be added to the sounds (all sounds will have the same category) - tags (str or None): string of tags to be added to the sounds (all sounds will have the same tags). - description (str or None): description to be added to the sounds (all sounds will have the same description). - processing_state (str): processing state of the created sounds. - moderation_state (str): moderation state of the created sounds. - type (str): type of the sounds to be created (e.g. 'wav'). + num_sounds: N sounds to generate. + num_packs: N packs in which the sounds above will be grouped. + user: user owner of the created sounds (if not provided, a new user will be created). + count_offset: start counting sounds at X. + bst_category: category code to be added to the sounds (all sounds will have the same category) + tags: string of tags to be added to the sounds (all sounds will have the same tags). + description: description to be added to the sounds (all sounds will have the same description). + processing_state: processing state of the created sounds. + moderation_state: moderation state of the created sounds. + type: type of the sounds to be created (e.g. 'wav'). Returns: - (Tuple(User, List[Pack], List[Sound]): 3-element tuple containing the user owning the sounds, + 3-element tuple containing the user owning the sounds, a list of the packs created and a list of the sounds created. """ count_offset = count_offset + next(sound_counter) @@ -130,11 +132,11 @@ def create_user_and_sounds( return user, packs, sounds -def create_consolidated_audio_descriptors_and_similarity_vectors_for_sound(sound): +def create_consolidated_audio_descriptors_and_similarity_vectors_for_sound(sound: Sound) -> None: """Creates fake consolidated audio descriptors and similarity vectors for a given sound. Args: - sound (Sound): Sound object for which to create the descriptors and vectors. + sound: Sound object for which to create the descriptors and vectors. """ # Create fake consolidated audio descriptors # The value that the feature takes is the idx of the feature in the provided list of feature names From 0ff25bdc12fb8371cadfa446028ff57279449607 Mon Sep 17 00:00:00 2001 From: Alastair Porter Date: Wed, 18 Feb 2026 10:47:58 +0100 Subject: [PATCH 2/5] Annotate module-level variables --- donations/forms.py | 2 +- freesound/audio_descriptor_settings.py | 4 ++-- freesound/settings.py | 22 +++++++++++----------- fscollections/models.py | 6 +++++- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/donations/forms.py b/donations/forms.py index ab28c4c4d..c00a3604d 100644 --- a/donations/forms.py +++ b/donations/forms.py @@ -8,7 +8,7 @@ class DonateForm(forms.Form): - RADIO_CHOICES = [] + RADIO_CHOICES: list[tuple[str, str]] = [] donation_type = forms.ChoiceField( widget=forms.RadioSelect(), diff --git a/freesound/audio_descriptor_settings.py b/freesound/audio_descriptor_settings.py index 7814b2efb..693b0e4bc 100644 --- a/freesound/audio_descriptor_settings.py +++ b/freesound/audio_descriptor_settings.py @@ -5,14 +5,14 @@ AUDIO_DESCRIPTOR_TYPE_LIST_STRINGS = "list_of_strings" AUDIO_DESCRIPTOR_TYPE_FLOAT_ARRAY = "float_array" AUDIO_DESCRIPTOR_TYPE_JSON = "json" # For complex structures -DEFAULT_AUDIO_DESCRIPTOR_TYPE = AUDIO_DESCRIPTOR_TYPE_FLOAT +DEFAULT_AUDIO_DESCRIPTOR_TYPE: str = AUDIO_DESCRIPTOR_TYPE_FLOAT DEFAULT_AUDIO_DESCRIPTOR_FLOAT_PRECISION = 3 # Number of decimal digits for float audio descriptors condition_music_or_instrument_samples = lambda s: s.category_names[0] in ["Music", "Instrument samples"] condition_instrument_samples = lambda s: s.category_names[0] == "Instrument samples" condition_sfx_or_soundscapes = lambda s: s.category_names[0] in ["Sound effects", "Soundscapes"] CONSOLIDATED_ANALYZER_NAME = "consolidated" -CONSOLIDATED_AUDIO_DESCRIPTORS = [ +CONSOLIDATED_AUDIO_DESCRIPTORS: list[dict] = [ { "name": "category", "analyzer": "bst-extractor_v2", diff --git a/freesound/settings.py b/freesound/settings.py index 06db0ff30..a59b6af77 100644 --- a/freesound/settings.py +++ b/freesound/settings.py @@ -246,7 +246,7 @@ AWS_SES_SHORT_BOUNCE_RATE_DATAPOINTS = 4 # cron period (1hr) / AWS stats period (15min) # If ALLOWED emails is not empty, only emails going to these destinations will be actually sent -ALLOWED_EMAILS = [] +ALLOWED_EMAILS: list[str] = [] # Email subjects EMAIL_SUBJECT_PREFIX = "[freesound]" @@ -292,7 +292,7 @@ # ------------------------------------------------------------------------------- # Freesound miscellaneous settings -SUPPORT = () +SUPPORT: tuple[tuple[str, str], ...] = () IFRAME_PLAYER_SIZE = {"large": [920, 245], "medium": [481, 86], "small": [375, 30], "twitter_card": [440, 132]} @@ -352,7 +352,7 @@ ALLOWED_AUDIOFILE_EXTENSIONS = ["wav", "aiff", "aif", "ogg", "flac", "mp3", "m4a", "wv"] LOSSY_FILE_EXTENSIONS = ["ogg", "mp3", "m4a"] # Note that some SOUND_TYPE_CHOICES below might correspond to multiple extensions (aiff/aif > aiff) -SOUND_TYPE_CHOICES = ( +SOUND_TYPE_CHOICES: tuple[tuple[str, str], ...] = ( ("wav", "Wave"), ("ogg", "Ogg Vorbis"), ("aiff", "AIFF"), @@ -378,7 +378,7 @@ # Time since last post (in seconds) and maximum number of posts per day LAST_FORUM_POST_MINIMUM_TIME = 60 * 5 BASE_MAX_POSTS_PER_DAY = 5 -SPAM_BLACKLIST = [] +SPAM_BLACKLIST: list[str] = [] MIN_DAYS_FOR_COMMENTING_WITH_LINKS = 10 # Random Sound of the day settings @@ -470,7 +470,7 @@ def load_broad_sound_taxonomy_from_csv(path): BST_CATEGORY_CHOICES = [ (key, value["name"]) for key, value in BROAD_SOUND_TAXONOMY.items() if "-" not in key ] # Top-level categories -BST_SUBCATEGORY_CHOICES = [ +BST_SUBCATEGORY_CHOICES: list = [ (key, value["name"]) for key, value in BROAD_SOUND_TAXONOMY.items() if "-" in key ] # Second-level categories @@ -532,7 +532,7 @@ def load_broad_sound_taxonomy_from_csv(path): BST_ANALYZER_NAME = "bst-extractor_v1" BSTV2_ANALYZER_NAME = "bst-extractor_v2" -ANALYZERS_CONFIGURATION = { +ANALYZERS_CONFIGURATION: dict[str, dict] = { AUDIOCOMMONS_ANALYZER_NAME: {}, FREESOUND_ESSENTIA_EXTRACTOR_NAME: {}, BIRDNET_ANALYZER_NAME: {}, @@ -644,7 +644,7 @@ def load_broad_sound_taxonomy_from_csv(path): SIMILARITY_SPACE_LAION_CLAP = "laion_clap" SIMILARITY_FREESOUND_CLASSIC = "freesound_classic" -SIMILARITY_SPACES = { +SIMILARITY_SPACES: dict[str, dict] = { SIMILARITY_SPACE_LAION_CLAP: { "vector_property_name": "clap_embedding", "vector_size": 512, @@ -662,7 +662,7 @@ def load_broad_sound_taxonomy_from_csv(path): } SIMILARITY_SPACES_NAMES = list(SIMILARITY_SPACES.keys()) SIMILARITY_SPACES_ANALYZER_NAMES = list(set([ss["analyzer"] for ss in SIMILARITY_SPACES.values()])) -SIMILARITY_SPACE_DEFAULT = SIMILARITY_SPACE_LAION_CLAP +SIMILARITY_SPACE_DEFAULT: str = SIMILARITY_SPACE_LAION_CLAP SIMILARITY_MIN_THRESHOLD = 0.7 MAX_SEARCH_RESULTS_IN_MAP_DISPLAY = ( @@ -704,7 +704,7 @@ def load_broad_sound_taxonomy_from_csv(path): # ------------------------------------------------------------------------------- # Sentry settings -SENTRY_DSN = None +SENTRY_DSN: str | None = None TRACES_SAMPLE_RATE = 0.1 # ------------------------------------------------------------------------------- @@ -814,7 +814,7 @@ def load_broad_sound_taxonomy_from_csv(path): RATELIMIT_SIMILARITY_GROUP: "2/s", RATELIMIT_REGISTRATION_GROUP: "5/m", } -BLOCKED_IPS = [] +BLOCKED_IPS: list[str] = [] CACHED_BLOCKED_IPS_KEY = "cached_blocked_ips" CACHED_BLOCKED_IPS_TIME = 60 * 5 # 5 minutes @@ -993,7 +993,7 @@ def load_broad_sound_taxonomy_from_csv(path): CSV_PATH = os.path.join(DATA_PATH, "csv/") ANALYSIS_PATH = os.path.join(DATA_PATH, "analysis/") FILE_UPLOAD_TEMP_DIR = os.path.join(DATA_PATH, "tmp_uploads/") -PROCESSING_TEMP_DIR = os.path.join(DATA_PATH, "tmp_processing/") +PROCESSING_TEMP_DIR: str = os.path.join(DATA_PATH, "tmp_processing/") PROCESSING_BEFORE_DESCRIPTION_DIR = os.path.join(DATA_PATH, "processing_before_description/") # URLs (depend on DATA_URL potentially re-defined in local_settings.py) diff --git a/fscollections/models.py b/fscollections/models.py index 68eaf70d1..a0ee2fdf3 100644 --- a/fscollections/models.py +++ b/fscollections/models.py @@ -18,6 +18,8 @@ # See AUTHORS file. # +from __future__ import annotations + from urllib.parse import quote from django.contrib.auth.models import User @@ -45,7 +47,9 @@ class Collection(LicenseSummaryMixin, models.Model): # TODO: description should be required (check how to display it in edit form + collectionsound form) description = models.TextField(blank=True) maintainers = models.ManyToManyField(User, related_name="collection_maintainer") - sounds = models.ManyToManyField(Sound, through="CollectionSound", related_name="collections", blank=True) + sounds: models.ManyToManyField[Sound, "CollectionSound"] = models.ManyToManyField( + Sound, through="CollectionSound", related_name="collections", blank=True + ) num_sounds = models.PositiveIntegerField(default=0) num_downloads = models.PositiveIntegerField(default=0) public = models.BooleanField(default=False) From c083a8bdc4f21e169ff28edfa8dab79200cb6a15 Mon Sep 17 00:00:00 2001 From: Alastair Porter Date: Mon, 20 Jul 2026 14:44:27 +0200 Subject: [PATCH 3/5] Configure mypy and add type stubs --- pyproject.toml | 62 ++++++++++- utils/downloads.py | 5 +- utils/ratelimit.py | 2 +- utils/search/__init__.py | 2 +- uv.lock | 222 ++++++++++++++++++++++++++++++++++++++- 5 files changed, 284 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 224417f85..13b7fffa4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,20 @@ dependencies = [ [dependency-groups] dev = [ - "django-stubs>=5.2.8", + # django-stubs 4.2.x targets Django 4.2 + "django-stubs[compatible-mypy]~=4.2.7", + "djangorestframework-stubs~=3.14.5", + "types-requests", + "types-Markdown", + "types-PyYAML", + "types-bleach", + "types-openpyxl", + "types-oauthlib", + "types-psutil", + "types-python-dateutil", + "types-Pillow", + "types-beautifulsoup4", + "types-six", "pip-licenses>=5.5.0", "pyright>=1.1.409", "ty>=0.0.7", @@ -135,3 +148,50 @@ exclude = [ "_docs/*", "**/migrations/*", ] + +[tool.mypy] +python_version = "3.10" +plugins = ["mypy_django_plugin.main"] +exclude = [ + '/migrations/', + '^_docs/', + '^tagrecommendation/', + '^similarity/', + '^textencoder/', +] + +[tool.django-stubs] +django_settings_module = "freesound.test_settings" + +# These packages have no types, ignore them +[[tool.mypy.overrides]] +module = [ + "adminsortable.*", + "akismet", + "boto3", + "botocore.*", + "celery", + "community", + "debug_toolbar.*", + "debugpy", + "django_object_actions", + "django_ratelimit.*", + "django_recaptcha.*", + "feedparser", + "hotshot", + "luqum.*", + "mapbox", + "multiupload_plus.*", + "networkx.*", + "oauth2_provider.*", + "past.*", + "pysndfile", + "pysolr", + "silk.*", + "sklearn.*", + "stripe", + "whitenoise.*", + "xlrd", + "zenpy.*", +] +ignore_missing_imports = true diff --git a/utils/downloads.py b/utils/downloads.py index ea6f72a36..f3025730e 100644 --- a/utils/downloads.py +++ b/utils/downloads.py @@ -23,11 +23,14 @@ import datetime import random import zlib +from typing import TYPE_CHECKING -from django.db.models import QuerySet from django.http import HttpResponse from django.utils import timezone +if TYPE_CHECKING: + from django.db.models import QuerySet + from donations.models import DonationsModalSettings from sounds.models import Download, PackDownload from utils.nginxsendfile import prepare_sendfile_arguments_for_sound_download diff --git a/utils/ratelimit.py b/utils/ratelimit.py index a601dc77e..e098be24e 100644 --- a/utils/ratelimit.py +++ b/utils/ratelimit.py @@ -35,7 +35,7 @@ console_logger = logging.getLogger("console") last_cached_blocked_ips = [] -last_cached_blocked_ips_timestamp = 0 +last_cached_blocked_ips_timestamp: float = 0 def get_ips_to_block() -> list[str]: diff --git a/utils/search/__init__.py b/utils/search/__init__.py index caed8b7f8..6094c8aaf 100644 --- a/utils/search/__init__.py +++ b/utils/search/__init__.py @@ -168,7 +168,7 @@ class SearchEngineTimeoutException(SearchEngineException): class SearchEngineBase: - solr_base_url = None + solr_base_url: str | None = None # Test SearchEngineBase with `pytest -m "search_engine" utils/search/backends/test_search_engine_backend.py` diff --git a/uv.lock b/uv.lock index 915adafa4..3dca63a66 100644 --- a/uv.lock +++ b/uv.lock @@ -559,18 +559,24 @@ wheels = [ [[package]] name = "django-stubs" -version = "5.2.8" +version = "4.2.7" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "django" }, { name = "django-stubs-ext" }, { name = "tomli" }, + { name = "types-pytz" }, { name = "types-pyyaml" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6c/75/97626224fd8f1787bb6f7f06944efcfddd5da7764bf741cf7f59d102f4a0/django_stubs-5.2.8.tar.gz", hash = "sha256:9bba597c9a8ed8c025cae4696803d5c8be1cf55bfc7648a084cbf864187e2f8b", size = 257709, upload-time = "2025-12-01T08:13:09.569Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/1f/1b1da357d37cbbf21496c17aad5a55416c7c37944a909c4a19ddd61994df/django-stubs-4.2.7.tar.gz", hash = "sha256:8ccd2ff4ee5adf22b9e3b7b1a516d2e1c2191e9d94e672c35cc2bc3dd61e0f6b", size = 258590, upload-time = "2023-12-05T19:02:28.493Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7d/3f/7c9543ad5ade5ce1d33d187a3abd82164570314ebee72c6206ab5c044ebf/django_stubs-5.2.8-py3-none-any.whl", hash = "sha256:a3c63119fd7062ac63d58869698d07c9e5ec0561295c4e700317c54e8d26716c", size = 508136, upload-time = "2025-12-01T08:13:07.963Z" }, + { url = "https://files.pythonhosted.org/packages/31/33/a4024662b31999841bc18ef845a790f6b6a9ad50f5b2aceb029896612d17/django_stubs-4.2.7-py3-none-any.whl", hash = "sha256:4cf4de258fa71adc6f2799e983091b9d46cfc67c6eebc68fe111218c9a62b3b8", size = 456669, upload-time = "2023-12-05T19:02:25.29Z" }, +] + +[package.optional-dependencies] +compatible-mypy = [ + { name = "mypy" }, ] [[package]] @@ -607,6 +613,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e2/08/178ca382236c658cd9d532ad0341ac8adba6d5548e0848320f5893442282/djangorestframework_jsonp-1.0.2-py2.py3-none-any.whl", hash = "sha256:fff4584e1a08f3c92f220eefb6f25907a54cc8fe39214ae058c572530f5c9277", size = 3701, upload-time = "2015-09-10T22:01:40.82Z" }, ] +[[package]] +name = "djangorestframework-stubs" +version = "3.14.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "django-stubs" }, + { name = "requests" }, + { name = "types-pyyaml" }, + { name = "types-requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/51/ef2f2196e81c261490e86c15b78f5a097d825b58f5ceb615bcece612eeb9/djangorestframework-stubs-3.14.5.tar.gz", hash = "sha256:5dd6f638aa5291fb7863e6166128a6ed20bf4986e2fc5cf334e6afc841797a09", size = 34281, upload-time = "2023-12-05T19:21:11.739Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/05/c98a1e3d3824dc9a330df26827935b5d7559a5c01bcfca8e18ab645dc758/djangorestframework_stubs-3.14.5-py3-none-any.whl", hash = "sha256:43d788fd50cda49b922cd411e59c5b8cdc3f3de49c02febae12ce42139f0269b", size = 53481, upload-time = "2023-12-05T19:21:09.215Z" }, +] + [[package]] name = "djangorestframework-xml" version = "2.0.0" @@ -750,10 +772,22 @@ dependencies = [ [package.dev-dependencies] dev = [ - { name = "django-stubs" }, + { name = "django-stubs", extra = ["compatible-mypy"] }, + { name = "djangorestframework-stubs" }, { name = "pip-licenses" }, { name = "pyright" }, { name = "ty" }, + { name = "types-beautifulsoup4" }, + { name = "types-bleach" }, + { name = "types-markdown" }, + { name = "types-oauthlib" }, + { name = "types-openpyxl" }, + { name = "types-pillow" }, + { name = "types-psutil" }, + { name = "types-python-dateutil" }, + { name = "types-pyyaml" }, + { name = "types-requests" }, + { name = "types-six" }, ] [package.metadata] @@ -821,10 +855,22 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ - { name = "django-stubs", specifier = ">=5.2.8" }, + { name = "django-stubs", extras = ["compatible-mypy"], specifier = "~=4.2.7" }, + { name = "djangorestframework-stubs", specifier = "~=3.14.5" }, { name = "pip-licenses", specifier = ">=5.5.0" }, { name = "pyright", specifier = ">=1.1.409" }, { name = "ty", specifier = ">=0.0.7" }, + { name = "types-beautifulsoup4" }, + { name = "types-bleach" }, + { name = "types-markdown" }, + { name = "types-oauthlib" }, + { name = "types-openpyxl" }, + { name = "types-pillow" }, + { name = "types-psutil" }, + { name = "types-python-dateutil" }, + { name = "types-pyyaml" }, + { name = "types-requests" }, + { name = "types-six" }, ] [[package]] @@ -1090,6 +1136,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d0/3e/b4547e3a34210956382eed1c85935fff7e0f9b98be3106b3745d7dec9c5e/msgpack-1.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:db6192777d943bdaaafb6ba66d44bf65aa0e9c5616fa1d2da9bb08828c6b39aa", size = 71293, upload-time = "2025-10-08T09:14:48.665Z" }, ] +[[package]] +name = "mypy" +version = "1.7.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "tomli" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/30/05a7c016431b3fdbaf0bcf663aee7c5e4b3d2293cd4e0568140cecae4967/mypy-1.7.1.tar.gz", hash = "sha256:fcb6d9afb1b6208b4c712af0dafdc650f518836065df0d4fb1d800f5d6773db2", size = 2979742, upload-time = "2023-11-23T18:02:15.227Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/a1/4d821de78ad8c78b2e159359faabd70cc85dcccb0f1c9cd71d5ea5ae332a/mypy-1.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:12cce78e329838d70a204293e7b29af9faa3ab14899aec397798a4b41be7f340", size = 10886501, upload-time = "2023-11-23T18:00:51.868Z" }, + { url = "https://files.pythonhosted.org/packages/a6/8c/034b199b5b07cfa1adbe3b629fced2549cbb1b95919de7a3bd3b349ad425/mypy-1.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1484b8fa2c10adf4474f016e09d7a159602f3239075c7bf9f1627f5acf40ad49", size = 9929685, upload-time = "2023-11-23T18:02:11.124Z" }, + { url = "https://files.pythonhosted.org/packages/44/ae/e45078b06648e42d61461faf1070c7615fe39f904dcef741431beb410b39/mypy-1.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31902408f4bf54108bbfb2e35369877c01c95adc6192958684473658c322c8a5", size = 12486804, upload-time = "2023-11-23T17:59:04.719Z" }, + { url = "https://files.pythonhosted.org/packages/29/6d/8ffee8037d5371008d729d28ae7e700984db96ed2b6b4cbcc49318b73fda/mypy-1.7.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f2c2521a8e4d6d769e3234350ba7b65ff5d527137cdcde13ff4d99114b0c8e7d", size = 12550413, upload-time = "2023-11-23T18:01:30.48Z" }, + { url = "https://files.pythonhosted.org/packages/1a/64/e12688d6bb5acff8d88915faf3e7e54f15eb96d30471ca1e15460c04a08a/mypy-1.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:fcd2572dd4519e8a6642b733cd3a8cfc1ef94bafd0c1ceed9c94fe736cb65b6a", size = 9189111, upload-time = "2023-11-23T18:00:59.398Z" }, + { url = "https://files.pythonhosted.org/packages/10/df/92bb67911c6c1d3faa46e4c9a5d0a93dd343dcf56022d1fb97a0c0ee65eb/mypy-1.7.1-py3-none-any.whl", hash = "sha256:f7c5d642db47376a0cc130f0de6d055056e010debdaf0707cd2b0fc7e7ef30ea", size = 2542516, upload-time = "2023-11-23T18:01:21.21Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + [[package]] name = "networkx" version = "3.2.1" @@ -1777,6 +1851,105 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/f4/c4fc28410c4493982b7481fb23f62bacb02fd2912ebec3b9bc7de18bebb8/ty-0.0.7-py3-none-win_arm64.whl", hash = "sha256:c87d27484dba9fca0053b6a9eee47eecc760aab2bbb8e6eab3d7f81531d1ad0c", size = 9653112, upload-time = "2025-12-24T21:28:31.562Z" }, ] +[[package]] +name = "types-beautifulsoup4" +version = "4.12.0.20250516" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "types-html5lib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/d1/32b410f6d65eda94d3dfb0b3d0ca151f12cb1dc4cef731dcf7cbfd8716ff/types_beautifulsoup4-4.12.0.20250516.tar.gz", hash = "sha256:aa19dd73b33b70d6296adf92da8ab8a0c945c507e6fb7d5db553415cc77b417e", size = 16628, upload-time = "2025-05-16T03:09:09.93Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/79/d84de200a80085b32f12c5820d4fd0addcbe7ba6dce8c1c9d8605e833c8e/types_beautifulsoup4-4.12.0.20250516-py3-none-any.whl", hash = "sha256:5923399d4a1ba9cc8f0096fe334cc732e130269541d66261bb42ab039c0376ee", size = 16879, upload-time = "2025-05-16T03:09:09.051Z" }, +] + +[[package]] +name = "types-bleach" +version = "6.4.0.20260607" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "types-html5lib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/e7/07438d4d0bd58c69e79d0d8aa0320a2005b08d10cac209dd812b3609227c/types_bleach-6.4.0.20260607.tar.gz", hash = "sha256:62ff099b9d25fbcccb1402651c14a465a5f54a24138be987442d2eed38aca91c", size = 11686, upload-time = "2026-06-07T06:11:40.035Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/93/44da16cfa99a3f47fcb35e6b7396c94c3530e2028d2b1027cc4ce6d95d8d/types_bleach-6.4.0.20260607-py3-none-any.whl", hash = "sha256:fc07e2fd0ef9bc0a22026e8abe589d76c5d06ea885d70357263a4ff8c3dc8516", size = 11985, upload-time = "2026-06-07T06:11:38.996Z" }, +] + +[[package]] +name = "types-html5lib" +version = "1.1.11.20260518" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "types-webencodings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b8/5a/0c708d1b0d35ad48b6a223c77c4a882fd016b40c25becb082a92e02a9c00/types_html5lib-1.1.11.20260518.tar.gz", hash = "sha256:4f33c087cb1119d65c4c80eca4323c2b501f9eaf8af9616b8b732ed4d8eae8fa", size = 18420, upload-time = "2026-05-18T06:07:23.662Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/d0/b088b9f11eb69637d6826843f06caaff60247156735a25512922d3dc2c13/types_html5lib-1.1.11.20260518-py3-none-any.whl", hash = "sha256:9baa7912224ebb37027c5ccb7e3768e43ea47b1dfdd977e7ddc4b0a4a550584d", size = 24339, upload-time = "2026-05-18T06:07:22.876Z" }, +] + +[[package]] +name = "types-markdown" +version = "3.10.2.20260712" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/eb/9520ed01687401b47265594955c0a9202a4f547795176de61bff77c8dfda/types_markdown-3.10.2.20260712.tar.gz", hash = "sha256:25aea17282490f35ab6b17a87771abd2f479185aaff8e8f4a0035043f02f2c52", size = 20185, upload-time = "2026-07-12T05:13:53.537Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/ea/a8dc2c8977870d27b8b44c4ba2952d765f9d39626e42d0083c871c1c1ebf/types_markdown-3.10.2.20260712-py3-none-any.whl", hash = "sha256:a551ddedde46e0eaca7059aa3d0e42db71afe2b53d8b58f72f2de63f3284f625", size = 25842, upload-time = "2026-07-12T05:13:52.702Z" }, +] + +[[package]] +name = "types-oauthlib" +version = "3.3.0.20260518" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4f/b9/272610e9b91ce274bd50a34ef9793a1a6e6666fe1acc97462ed5df11e262/types_oauthlib-3.3.0.20260518.tar.gz", hash = "sha256:b713f6c32a0e544d165afed6b953780d99ef6997a6b6ddc07491f72bdaf5307b", size = 26126, upload-time = "2026-05-18T06:02:13.485Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/64/8890a1bd403dc90711c15298e1054d72e8865967d169abaf30fb93a750f6/types_oauthlib-3.3.0.20260518-py3-none-any.whl", hash = "sha256:7e35e6e8731e49fc131fb0a94b5e473ac70722dd039c308501de2a5c137ed024", size = 48964, upload-time = "2026-05-18T06:02:12.098Z" }, +] + +[[package]] +name = "types-openpyxl" +version = "3.1.5.20260518" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/d1/a1e23040a758746ad5bb6b8849a011c3c901775618bc7e1fec3a1a8b7142/types_openpyxl-3.1.5.20260518.tar.gz", hash = "sha256:da9cd644e4e80215a3f60a8c2c2c8e980e941a9b581cffa3876285aa791ca5af", size = 101550, upload-time = "2026-05-18T06:03:57.59Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/0e/d745ce95fc74e34df802010fd0387e33db468179e6ff42b708280ab268c7/types_openpyxl-3.1.5.20260518-py3-none-any.whl", hash = "sha256:e6ca4b116c8b979ed57f3045edcd3d49c25917d6dae99e90358f41322a19d375", size = 165744, upload-time = "2026-05-18T06:03:56.036Z" }, +] + +[[package]] +name = "types-pillow" +version = "10.2.0.20240822" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/4a/4495264dddaa600d65d68bcedb64dcccf9d9da61adff51f7d2ffd8e4c9ce/types-Pillow-10.2.0.20240822.tar.gz", hash = "sha256:559fb52a2ef991c326e4a0d20accb3bb63a7ba8d40eb493e0ecb0310ba52f0d3", size = 35389, upload-time = "2024-08-22T02:32:48.15Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/23/e81a5354859831fcf54d488d33b80ba6133ea84f874a9c0ec40a4881e133/types_Pillow-10.2.0.20240822-py3-none-any.whl", hash = "sha256:d9dab025aba07aeb12fd50a6799d4eac52a9603488eca09d7662543983f16c5d", size = 54354, upload-time = "2024-08-22T02:32:46.664Z" }, +] + +[[package]] +name = "types-psutil" +version = "7.2.2.20260518" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/f1/6901857281d4e8d792492e1495eef6f4f01318a3b6a066486d81000a4511/types_psutil-7.2.2.20260518.tar.gz", hash = "sha256:9f825f631463a5b4d26f19f63aebc9ec25f01140d655026f3ad8a67841f9b331", size = 26660, upload-time = "2026-05-18T06:05:09.389Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/eb/f726339668879819599c74c2e1f0cab760912a4159046942bdae2ad37bd6/types_psutil-7.2.2.20260518-py3-none-any.whl", hash = "sha256:6a3d697665754a60d7b5a41d5a2cff12b53f5e0676d77810cd28ba5e14cb4049", size = 32820, upload-time = "2026-05-18T06:05:08.321Z" }, +] + +[[package]] +name = "types-python-dateutil" +version = "2.9.0.20260716" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/9a/aefb9f80ff79641d36149352afe66ca835c5e6894deb518418f786f4b8ad/types_python_dateutil-2.9.0.20260716.tar.gz", hash = "sha256:1d55d1c3024bdb4861bb6a6622c9ec800c433d87bdc5b16fb84cdd0eed4ef2cb", size = 17516, upload-time = "2026-07-16T04:48:06.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/85/cff97326a81e7e8877803e78e3ea56c840b85bb811934b9c936f7c40f185/types_python_dateutil-2.9.0.20260716-py3-none-any.whl", hash = "sha256:1ae41d51a5c5f6bbeeb7f1f34df7086d55ff1605cf88d2ed71a7a276e1a7794e", size = 18443, upload-time = "2026-07-16T04:48:05.793Z" }, +] + +[[package]] +name = "types-pytz" +version = "2026.2.0.20260518" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9f/d9/9fa4019d2235bd374293e1fd4153879b28b6ae1d2bae98addd352c9713f2/types_pytz-2026.2.0.20260518.tar.gz", hash = "sha256:e5d254329e9c4e91f0781b22c43a4bb2d10bb044d97b24c4b05d45567b0eae16", size = 10871, upload-time = "2026-05-18T06:02:45.789Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/89/41e80670779a223d8bc8bc83019a619988cfa5c432cedac5cec23884fbc4/types_pytz-2026.2.0.20260518-py3-none-any.whl", hash = "sha256:3a12eaa38f476bd650902a9c9bb442f03f3c7dee2be5c5848bce61bd708d205a", size = 10125, upload-time = "2026-05-18T06:02:44.968Z" }, +] + [[package]] name = "types-pyyaml" version = "6.0.12.20250915" @@ -1786,6 +1959,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bd/e0/1eed384f02555dde685fff1a1ac805c1c7dcb6dd019c916fe659b1c1f9ec/types_pyyaml-6.0.12.20250915-py3-none-any.whl", hash = "sha256:e7d4d9e064e89a3b3cae120b4990cd370874d2bf12fa5f46c97018dd5d3c9ab6", size = 20338, upload-time = "2025-09-15T03:00:59.218Z" }, ] +[[package]] +name = "types-requests" +version = "2.31.0.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "types-urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f9/b8/c1e8d39996b4929b918aba10dba5de07a8b3f4c8487bb61bb79882544e69/types-requests-2.31.0.6.tar.gz", hash = "sha256:cd74ce3b53c461f1228a9b783929ac73a666658f223e28ed29753771477b3bd0", size = 15535, upload-time = "2023-09-27T06:19:38.443Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/a1/6f8dc74d9069e790d604ddae70cb46dcbac668f1bb08136e7b0f2f5cd3bf/types_requests-2.31.0.6-py3-none-any.whl", hash = "sha256:a2db9cb228a81da8348b49ad6db3f5519452dd20a9c1e1a868c83c5fe88fd1a9", size = 14516, upload-time = "2023-09-27T06:19:36.373Z" }, +] + +[[package]] +name = "types-six" +version = "1.17.0.20260518" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/14/ec/31f3a70f3b4f7dbbf28025a5b2ed0b3a3eb237c01bc3357e45547c04defd/types_six-1.17.0.20260518.tar.gz", hash = "sha256:b0196d5188bd589bc5ab92901edc1a4ff3c5fd4b0dc19d074a5f1f8213e5213a", size = 15787, upload-time = "2026-05-18T06:04:07.387Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/2e/0e64ff848fa3d3ee1563ee3e6b3f9f226c25e5ce17020fb0be1f832a7e32/types_six-1.17.0.20260518-py3-none-any.whl", hash = "sha256:2ed782cafcef9614e51292ae39bae93e42ff7b20c1c0a00c6be35e1c0e493a70", size = 19940, upload-time = "2026-05-18T06:04:06.565Z" }, +] + +[[package]] +name = "types-urllib3" +version = "1.26.25.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/73/de/b9d7a68ad39092368fb21dd6194b362b98a1daeea5dcfef5e1adb5031c7e/types-urllib3-1.26.25.14.tar.gz", hash = "sha256:229b7f577c951b8c1b92c1bc2b2fdb0b49847bd2af6d1cc2a2e3dd340f3bda8f", size = 11239, upload-time = "2023-07-20T15:19:31.307Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/7b/3fc711b2efea5e85a7a0bbfe269ea944aa767bbba5ec52f9ee45d362ccf3/types_urllib3-1.26.25.14-py3-none-any.whl", hash = "sha256:9683bbb7fb72e32bfe9d2be6e04875fbe1b3eeec3cbb4ea231435aa7fd6b4f0e", size = 15377, upload-time = "2023-07-20T15:19:30.379Z" }, +] + +[[package]] +name = "types-webencodings" +version = "0.5.0.20260408" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/d2/21567fac142315580ce3ee37d08a4e8819921dae833bbcd27b9f8b373799/types_webencodings-0.5.0.20260408.tar.gz", hash = "sha256:28c596619f367e43eee393d85f63e8d2fdb6874c654a8d441c37f8afe29c6d0d", size = 7504, upload-time = "2026-04-08T04:28:51.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/e4/f13be8f6d9a561166f7d963012d0ccc833e13aee3044c4f1a8fb1fee462a/types_webencodings-0.5.0.20260408-py3-none-any.whl", hash = "sha256:19a2afe5c22d9b1e880b49ff823c7b531f473a390fe47ac903c0bdb5cd677dd9", size = 8717, upload-time = "2026-04-08T04:28:50.943Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" From dee949918625a06bba0c8d81b146b4c19ce20719 Mon Sep 17 00:00:00 2001 From: Alastair Porter Date: Tue, 21 Jul 2026 09:50:00 +0200 Subject: [PATCH 4/5] Annotate staff-check predicate for monitor views Replace the inline user_passes_test(lambda u: u.is_staff) with a module-level user_is_staff predicate. django-stubs types the callable's argument as AbstractBaseUser | AnonymousUser, and AbstractBaseUser has no is_staff attribute, so mypy could not check the lambda. The predicate takes that same union and narrows to User before reading is_staff. Behaviour is unchanged (AnonymousUser.is_staff is already False). --- monitor/views.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/monitor/views.py b/monitor/views.py index 5357f6d8b..40c8cee2b 100644 --- a/monitor/views.py +++ b/monitor/views.py @@ -24,7 +24,7 @@ import requests from django.conf import settings from django.contrib.auth.decorators import login_required, user_passes_test -from django.contrib.auth.models import User +from django.contrib.auth.models import AbstractBaseUser, AnonymousUser, User from django.core.cache import caches from django.db.models import Count from django.http import HttpResponse, HttpResponseRedirect, JsonResponse @@ -45,8 +45,12 @@ ) +def user_is_staff(user: AbstractBaseUser | AnonymousUser) -> bool: + return isinstance(user, User) and user.is_staff + + @login_required -@user_passes_test(lambda u: u.is_staff, login_url="/") +@user_passes_test(user_is_staff, login_url="/") def get_queues_status(request): try: celery_task_counts = get_queues_task_counts() @@ -56,7 +60,7 @@ def get_queues_status(request): @login_required -@user_passes_test(lambda u: u.is_staff, login_url="/") +@user_passes_test(user_is_staff, login_url="/") def monitor_home(request): # Test instrumentation: bump a counter so we can confirm it appears on /metrics. MONITOR_HOME_VIEWS.inc() @@ -64,7 +68,7 @@ def monitor_home(request): @login_required -@user_passes_test(lambda u: u.is_staff, login_url="/") +@user_passes_test(user_is_staff, login_url="/") def monitor_processing(request): # Processing sounds_queued_count = Sound.objects.filter(processing_ongoing_state="QU").count() @@ -90,7 +94,7 @@ def monitor_processing(request): @login_required -@user_passes_test(lambda u: u.is_staff, login_url="/") +@user_passes_test(user_is_staff, login_url="/") def monitor_analysis(request): # Analysis analyzers_data = {} @@ -142,7 +146,7 @@ def monitor_analysis(request): @login_required -@user_passes_test(lambda u: u.is_staff, login_url="/") +@user_passes_test(user_is_staff, login_url="/") def monitor_similarity(request): # Get stats about similarity vectors indexed in Solr try: @@ -169,7 +173,7 @@ def monitor_similarity(request): @login_required -@user_passes_test(lambda u: u.is_staff, login_url="/") +@user_passes_test(user_is_staff, login_url="/") def monitor_moderation(request): sounds_in_moderators_queue_count = tickets.views._get_sounds_in_moderators_queue_count(request.user) new_upload_count = tickets.views.new_sound_tickets_count() @@ -198,14 +202,14 @@ def monitor_moderation(request): @login_required -@user_passes_test(lambda u: u.is_staff, login_url="/") +@user_passes_test(user_is_staff, login_url="/") def monitor_stats(request): tvars = {"activePage": "stats"} return render(request, "monitor/stats.html", tvars) @login_required -@user_passes_test(lambda u: u.is_staff, login_url="/") +@user_passes_test(user_is_staff, login_url="/") def moderators_stats(request): return HttpResponseRedirect(reverse("monitor-moderation")) @@ -266,7 +270,7 @@ def totals_stats_ajax(request): @login_required -@user_passes_test(lambda u: u.is_staff, login_url="/") +@user_passes_test(user_is_staff, login_url="/") def process_sounds(request): # Send sounds to processing according to their processing_state processing_status = request.GET.get("prs", None) From 687d46523f7628ff43f4d09313bf1acad6f0b43a Mon Sep 17 00:00:00 2001 From: Alastair Porter Date: Tue, 21 Jul 2026 10:41:00 +0200 Subject: [PATCH 5/5] Fix types on user admin object-actions with @action decorator --- accounts/admin.py | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/accounts/admin.py b/accounts/admin.py index 0596c8f01..8fcf2dbda 100644 --- a/accounts/admin.py +++ b/accounts/admin.py @@ -29,7 +29,7 @@ from django.shortcuts import render from django.urls import reverse from django.utils.safestring import mark_safe -from django_object_actions import DjangoObjectActions +from django_object_actions import DjangoObjectActions, action from accounts.forms import username_taken_by_other_user from accounts.models import ( @@ -190,7 +190,7 @@ def get_actions(self, request): del actions["delete_selected"] return actions - @admin.action(description="Delete the user but keep the sounds available") + @action(description="Delete the user but keep the sounds available", label="Delete user only") def delete_preserve_sounds(self, request, obj): username = obj.username if request.method == "POST": @@ -224,9 +224,7 @@ def delete_preserve_sounds(self, request, obj): tvars = {"users_to_delete": [user_info], "type": "delete_preserve_sounds"} return render(request, "accounts/admin_delete_confirmation.html", tvars) - delete_preserve_sounds.label = "Delete user only" - - @admin.action(description="Delete the user and the sounds") + @action(description="Delete the user and the sounds", label="Delete user and sounds") def delete_include_sounds(self, request, obj): username = obj.username if request.method == "POST": @@ -267,9 +265,7 @@ def delete_include_sounds(self, request, obj): return render(request, "accounts/admin_delete_confirmation.html", tvars) - delete_include_sounds.label = "Delete user and sounds" - - @admin.action(description="Delete the user and the sounds, mark deleted user as spammer") + @action(description="Delete the user and the sounds, mark deleted user as spammer", label="Delete as spammer") def delete_spammer(self, request, obj): username = obj.username if request.method == "POST": @@ -308,9 +304,7 @@ def delete_spammer(self, request, obj): return render(request, "accounts/admin_delete_confirmation.html", tvars) - delete_spammer.label = "Delete as spammer" - - @admin.action(description="Completely delete user from db") + @action(description="Completely delete user from db", label="Full delete") def full_delete(self, request, obj): username = obj.username if request.method == "POST": @@ -349,9 +343,7 @@ def full_delete(self, request, obj): return render(request, "accounts/admin_delete_confirmation.html", tvars) - full_delete.label = "Full delete" - - @admin.action(description="Clear all user flags for of spam reports and akismet") + @action(description="Clear all user flags for of spam reports and akismet", label="Clear spam flags") def clear_spam_flags(self, request, obj): num_akismet, _ = obj.akismetspam_set.all().delete() num_reports, _ = obj.flags.all().delete() @@ -363,20 +355,14 @@ def clear_spam_flags(self, request, obj): ) return HttpResponseRedirect(reverse("admin:auth_user_change", args=[obj.id])) - clear_spam_flags.label = "Clear spam flags" - - @admin.action(description="Open user on site") + @action(description="Open user on site", label="View on site") def view_on_site_action(self, request, obj): return HttpResponseRedirect(reverse("account", args=[obj.username])) - view_on_site_action.label = "View on site" - - @admin.action(description="Edit profile in admin") + @action(description="Edit profile in admin", label="Edit profile in admin") def edit_profile_admin(self, request, obj): return HttpResponseRedirect(reverse("admin:accounts_profile_change", args=[obj.profile.id])) - edit_profile_admin.label = "Edit profile in admin" - # NOTE: in the line below we removed the 'full_delete' option as ideally we should never need to use it. In for # some unexpected reason we happen to need it, we can call the .delete() method on a user object using the terminal. # If we observe a real need for that, we can re-add the option to the admin.