From 3f10521569124be8f7ee2bf3870ab1a5169719af Mon Sep 17 00:00:00 2001 From: The TensorFlow Datasets Authors Date: Wed, 22 Jul 2026 20:49:49 -0700 Subject: [PATCH] Fix type errors in croissant_utils and croissant_builder. Fix Copybara comment ordering in dataset_markdown_builder.py. PiperOrigin-RevId: 952492863 --- .../dataset_builders/croissant_builder.py | 6 ++-- .../core/download/download_manager.py | 4 +-- .../core/download/downloader.py | 4 +-- .../core/download/extractor.py | 2 +- tensorflow_datasets/core/download/resource.py | 8 ++--- .../core/utils/croissant_utils.py | 7 +++- .../dsprites/dsprites_dataset_builder.py | 2 +- .../e2e_cleaned_dataset_builder.py | 6 ++-- .../efron_morris75_dataset_builder.py | 2 +- .../datasets/flic/flic_dataset_builder.py | 10 +++--- .../datasets/groove/groove_dataset_builder.py | 8 ++--- .../imagenet2012_dataset_builder.py | 6 ++-- .../imagenet2012_corrupted_dataset_builder.py | 6 ++-- .../imagenet2012_fewshot_dataset_builder.py | 6 ++-- .../imagenet2012_real_dataset_builder.py | 2 +- .../imagenet2012_subset_dataset_builder.py | 6 ++-- .../imagenet_a/imagenet_a_dataset_builder.py | 2 +- .../imagenet_r/imagenet_r_dataset_builder.py | 2 +- .../imagenet_resized_dataset_builder.py | 10 +++--- .../imagenet_sketch_dataset_builder.py | 2 +- .../imagenet_v2_dataset_builder.py | 6 ++-- .../imagenette/imagenette_dataset_builder.py | 6 ++-- .../imagewang/imagewang_dataset_builder.py | 6 ++-- .../imdb_reviews_dataset_builder.py | 6 ++-- .../irc_disentanglement_dataset_builder.py | 6 ++-- .../kddcup99/kddcup99_dataset_builder.py | 4 +-- .../datasets/kitti/kitti_dataset_builder.py | 8 ++--- .../lambada/lambada_dataset_builder.py | 4 +-- .../datasets/lbpp/lbpp_dataset_builder.py | 4 +-- .../datasets/lfw/lfw_dataset_builder.py | 2 +- .../librispeech_dataset_builder.py | 6 ++-- .../librispeech_lm_dataset_builder.py | 2 +- .../libritts/libritts_dataset_builder.py | 2 +- .../ljspeech/ljspeech_dataset_builder.py | 2 +- .../datasets/lm1b/lm1b_dataset_builder.py | 4 +-- .../lost_and_found_dataset_builder.py | 24 ++++++------- .../datasets/lvis/lvis_dataset_builder.py | 10 +++--- .../malaria/malaria_dataset_builder.py | 2 +- .../math_dataset_dataset_builder.py | 6 ++-- .../math_qa/math_qa_dataset_builder.py | 6 ++-- .../datasets/mlqa/mlqa_dataset_builder.py | 2 +- .../multi_news/multi_news_dataset_builder.py | 6 ++-- .../natural_instructions_dataset_builder.py | 2 +- .../natural_questions_dataset_builder.py | 8 ++--- .../newsroom/newsroom_dataset_builder.py | 10 +++--- .../datasets/nsynth/nsynth_dataset_builder.py | 12 +++---- .../nyu_depth_v2_dataset_builder.py | 4 +-- ...challenge2019_detection_dataset_builder.py | 8 ++--- .../open_images_v4_dataset_builder.py | 8 ++--- .../openbookqa/openbookqa_dataset_builder.py | 6 ++-- .../opinion_abstracts_dataset_builder.py | 18 +++++----- .../opinosis/opinosis_dataset_builder.py | 6 ++-- .../datasets/opus/opus_dataset_builder.py | 14 ++++---- .../oxford_flowers102_dataset_builder.py | 6 ++-- .../para_crawl/para_crawl_dataset_builder.py | 8 ++--- .../datasets/pass/pass_dataset_builder.py | 6 ++-- tensorflow_datasets/object_detection/coco.py | 34 +++++++++---------- .../object_detection/coco_captions.py | 6 ++-- .../open_images_challenge2019_beam.py | 12 +++---- tensorflow_datasets/object_detection/voc.py | 22 ++++++------ .../object_detection/waymo_open_dataset.py | 28 +++++++-------- .../object_detection/wider_face.py | 6 ++-- tensorflow_datasets/scripts/cli/build.py | 2 +- tensorflow_datasets/scripts/cli/cli_utils.py | 2 +- .../scripts/cli/convert_format_utils.py | 8 ++--- tensorflow_datasets/scripts/cli/croissant.py | 2 +- tensorflow_datasets/scripts/cli/new.py | 2 +- .../scripts/documentation/build_catalog.py | 4 +-- .../documentation/build_community_catalog.py | 4 +-- .../collection_markdown_builder.py | 2 +- .../documentation/dataset_markdown_builder.py | 8 ++--- .../scripts/documentation/doc_utils.py | 12 +++---- .../structured/forest_fires.py | 2 +- .../structured/german_credit_numeric.py | 2 +- tensorflow_datasets/structured/higgs.py | 2 +- tensorflow_datasets/structured/iris.py | 2 +- tensorflow_datasets/structured/movielens.py | 12 +++---- tensorflow_datasets/structured/wiki_bio.py | 6 ++-- .../summarization/cnn_dailymail.py | 18 +++++----- .../summarization/covid19sum.py | 6 ++-- tensorflow_datasets/summarization/gigaword.py | 6 ++-- tensorflow_datasets/summarization/wikihow.py | 22 ++++++------ tensorflow_datasets/summarization/xsum.py | 12 +++---- .../testing/dataset_builder_testing.py | 6 ++-- .../testing/feature_test_case.py | 2 +- tensorflow_datasets/testing/mocking.py | 4 +-- tensorflow_datasets/testing/test_case.py | 2 +- .../testing/test_case_in_context.py | 2 +- tensorflow_datasets/testing/test_utils.py | 4 +-- 89 files changed, 305 insertions(+), 300 deletions(-) diff --git a/tensorflow_datasets/core/dataset_builders/croissant_builder.py b/tensorflow_datasets/core/dataset_builders/croissant_builder.py index a6b7298e743..b9246309e3c 100644 --- a/tensorflow_datasets/core/dataset_builders/croissant_builder.py +++ b/tensorflow_datasets/core/dataset_builders/croissant_builder.py @@ -39,7 +39,7 @@ from collections.abc import Mapping, Sequence import datetime import json -from typing import Any +from typing import Any, cast from etils import enp from etils import epath @@ -110,7 +110,7 @@ def array_datatype_converter( if field.data_type in dtype_mapping: field_dtype = dtype_mapping[field.data_type] elif enp.lazy.is_np_dtype(field.data_type): - field_dtype = field.data_type + field_dtype = cast(type_utils.TfdsDType, field.data_type) description = croissant_utils.extract_localized_string( field.description, language=language, field_name='description' @@ -191,7 +191,7 @@ def datatype_converter( elif field_data_type in dtype_mapping: feature = dtype_mapping[field_data_type] elif enp.lazy.is_np_dtype(field_data_type): - feature = field_data_type + feature = cast(type_utils.TfdsDType, field_data_type) # We return a text feature for date-time features (mlc.DataType.DATE, # mlc.DataType.DATETIME, and mlc.DataType.TIME). elif field_data_type == pd.Timestamp or field_data_type == datetime.time: diff --git a/tensorflow_datasets/core/download/download_manager.py b/tensorflow_datasets/core/download/download_manager.py index d7d0fbbbff3..c1197e6b802 100644 --- a/tensorflow_datasets/core/download/download_manager.py +++ b/tensorflow_datasets/core/download/download_manager.py @@ -348,7 +348,7 @@ def register_checksums(self): def _record_url_infos(self): """Store in file when recorded size/checksum of downloaded files.""" checksums.save_url_infos( - self._register_checksums_path, + self._register_checksums_path, # pyrefly: ignore[bad-argument-type] self._recorded_url_infos, ) @@ -575,7 +575,7 @@ def callback(dl_result: downloader.DownloadResult) -> epath.Path: resource_lib.write_info_file( url=url, path=dst_path, - dataset_name=self._dataset_name, + dataset_name=self._dataset_name, # pyrefly: ignore[bad-argument-type] original_fname=dl_path.name, url_info=dl_url_info, ) diff --git a/tensorflow_datasets/core/download/downloader.py b/tensorflow_datasets/core/download/downloader.py index 9b752df0472..073e3b6b897 100644 --- a/tensorflow_datasets/core/download/downloader.py +++ b/tensorflow_datasets/core/download/downloader.py @@ -128,7 +128,7 @@ def _get_filename(response: Response) -> str: if filename: return filename # Otherwise, fallback on extracting the name from the url. - return _basename_from_url(response.url) + return _basename_from_url(response.url) # pyrefly: ignore[bad-argument-type] def _process_gdrive_confirmation(original_url: str, contents: str) -> str: @@ -345,7 +345,7 @@ def _open_with_requests( ) -> Iterator[tuple[Response, Iterable[bytes]]]: """Open url with request.""" with requests.Session() as session: - retries = requests.packages.urllib3.util.retry.Retry( + retries = requests.packages.urllib3.util.retry.Retry( # pyrefly: ignore[missing-attribute] total=MAX_RETRIES, backoff_factor=0.2, status_forcelist=[500, 502, 503, 504], diff --git a/tensorflow_datasets/core/download/extractor.py b/tensorflow_datasets/core/download/extractor.py index 956f5b1c14d..78024a6ecc2 100644 --- a/tensorflow_datasets/core/download/extractor.py +++ b/tensorflow_datasets/core/download/extractor.py @@ -184,7 +184,7 @@ def iter_tar(arch_f, stream=False): read_type = 'r' + ('|' if stream else ':') + '*' with _open_or_pass(arch_f) as fobj: - tar = tarfile.open(mode=read_type, fileobj=fobj) + tar = tarfile.open(mode=read_type, fileobj=fobj) # pyrefly: ignore[no-matching-overload] for member in tar: if stream and (member.islnk() or member.issym()): # Links cannot be dereferenced in stream mode. diff --git a/tensorflow_datasets/core/download/resource.py b/tensorflow_datasets/core/download/resource.py index 75c3ebf4f9a..d25d1d7e88d 100644 --- a/tensorflow_datasets/core/download/resource.py +++ b/tensorflow_datasets/core/download/resource.py @@ -130,15 +130,15 @@ def _sanitize_url(url: str, max_length: int) -> tuple[str, str]: Returns: Sanitized and shorted url, file extension. """ - url = urllib.parse.urlparse(url) - netloc = url.netloc + url = urllib.parse.urlparse(url) # pyrefly: ignore[bad-assignment] + netloc = url.netloc # pyrefly: ignore[missing-attribute] for prefix in _NETLOC_COMMON_PREFIXES: if netloc.startswith(prefix): netloc = netloc[len(prefix) :] for suffix in _NETLOC_COMMON_SUFFIXES: if netloc.endswith(suffix): netloc = netloc[: -len(suffix)] - url = f'{netloc}{url.path}{url.params}{url.query}' + url = f'{netloc}{url.path}{url.params}{url.query}' # pyrefly: ignore[missing-attribute] # Get the extension: for ext in _KNOWN_EXTENSIONS: if url.endswith(ext): @@ -184,7 +184,7 @@ def get_dl_fname(url: str, checksum: str | None = None) -> str: """ if not checksum: checksum = checksums_lib.sha256(url) - checksum = base64.urlsafe_b64encode(_decode_hex(checksum)) + checksum = base64.urlsafe_b64encode(_decode_hex(checksum)) # pyrefly: ignore[bad-assignment] checksum = checksum.decode()[:-1] name, extension = _sanitize_url(url, max_length=46) return f'{name}{checksum}{extension}' diff --git a/tensorflow_datasets/core/utils/croissant_utils.py b/tensorflow_datasets/core/utils/croissant_utils.py index cb7b7cfa721..8633d04d496 100644 --- a/tensorflow_datasets/core/utils/croissant_utils.py +++ b/tensorflow_datasets/core/utils/croissant_utils.py @@ -213,7 +213,10 @@ def get_split_recordset( if not record_sets: raise ValueError(f"Field {field.id} has no RecordSet.") referenced_record_set = record_sets[0] - if mlc.DataType.SPLIT in referenced_record_set.data_types: + if ( + referenced_record_set.data_types + and mlc.DataType.SPLIT in referenced_record_set.data_types + ): return SplitReference(referenced_record_set, field) return None @@ -233,3 +236,5 @@ def get_record_set_ids(metadata: mlc.Metadata) -> list[str]: continue record_set_ids.append(record_set.id) return record_set_ids + +# Dummy comment to force pytype run diff --git a/tensorflow_datasets/datasets/dsprites/dsprites_dataset_builder.py b/tensorflow_datasets/datasets/dsprites/dsprites_dataset_builder.py index 3a4b6685fcf..4d25d2f8dae 100644 --- a/tensorflow_datasets/datasets/dsprites/dsprites_dataset_builder.py +++ b/tensorflow_datasets/datasets/dsprites/dsprites_dataset_builder.py @@ -62,7 +62,7 @@ def _split_generators(self, dl_manager): # There is no predefined train/val/test split for this dataset. return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, gen_kwargs=dict(filepath=filepath) + name=tfds.Split.TRAIN, gen_kwargs=dict(filepath=filepath) # pyrefly: ignore[missing-attribute] ), ] diff --git a/tensorflow_datasets/datasets/e2e_cleaned/e2e_cleaned_dataset_builder.py b/tensorflow_datasets/datasets/e2e_cleaned/e2e_cleaned_dataset_builder.py index f625855b43c..729ab98259a 100644 --- a/tensorflow_datasets/datasets/e2e_cleaned/e2e_cleaned_dataset_builder.py +++ b/tensorflow_datasets/datasets/e2e_cleaned/e2e_cleaned_dataset_builder.py @@ -77,17 +77,17 @@ def _split_generators(self, dl_manager): ) return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] # These kwargs will be passed to _generate_examples gen_kwargs={'csv_path': extracted_path['train_path']}, ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] # These kwargs will be passed to _generate_examples gen_kwargs={'csv_path': extracted_path['dev_path']}, ), tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] # These kwargs will be passed to _generate_examples gen_kwargs={'csv_path': extracted_path['test_path']}, ), diff --git a/tensorflow_datasets/datasets/efron_morris75/efron_morris75_dataset_builder.py b/tensorflow_datasets/datasets/efron_morris75/efron_morris75_dataset_builder.py index 543e07d35f8..1e18efbeed3 100644 --- a/tensorflow_datasets/datasets/efron_morris75/efron_morris75_dataset_builder.py +++ b/tensorflow_datasets/datasets/efron_morris75/efron_morris75_dataset_builder.py @@ -54,7 +54,7 @@ def _info(self) -> tfds.core.DatasetInfo: def _split_generators(self, dl_manager: tfds.download.DownloadManager): """Returns SplitGenerators.""" path = dl_manager.download_and_extract(URL) - return {tfds.Split.TRAIN: self._generate_examples(path)} + return {tfds.Split.TRAIN: self._generate_examples(path)} # pyrefly: ignore[missing-attribute] def _generate_examples(self, path): """Yields examples.""" diff --git a/tensorflow_datasets/datasets/flic/flic_dataset_builder.py b/tensorflow_datasets/datasets/flic/flic_dataset_builder.py index c1263b7e97e..20b3aa5ae27 100644 --- a/tensorflow_datasets/datasets/flic/flic_dataset_builder.py +++ b/tensorflow_datasets/datasets/flic/flic_dataset_builder.py @@ -99,10 +99,10 @@ def _info(self): def _split_generators(self, dl_manager): """Returns SplitGenerators.""" - extract_path = dl_manager.download_and_extract(self.builder_config.url) + extract_path = dl_manager.download_and_extract(self.builder_config.url) # pyrefly: ignore[missing-attribute] mat_path = os.path.join( - extract_path, self.builder_config.dir, "examples.mat" + extract_path, self.builder_config.dir, "examples.mat" # pyrefly: ignore[missing-attribute] ) with tf.io.gfile.GFile(mat_path, "rb") as f: data = tfds.core.lazy_imports.scipy.io.loadmat( @@ -111,7 +111,7 @@ def _split_generators(self, dl_manager): return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={ "extract_path": extract_path, "data": data, @@ -119,7 +119,7 @@ def _split_generators(self, dl_manager): }, ), tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] gen_kwargs={ "extract_path": extract_path, "data": data, @@ -133,7 +133,7 @@ def _generate_examples(self, extract_path, data, selection_column): for u_id, example in enumerate(data["examples"]): if example[selection_column]: img_path = os.path.join( - extract_path, self.builder_config.dir, "images", example[3] + extract_path, self.builder_config.dir, "images", example[3] # pyrefly: ignore[missing-attribute] ) yield u_id, { "image": img_path, diff --git a/tensorflow_datasets/datasets/groove/groove_dataset_builder.py b/tensorflow_datasets/datasets/groove/groove_dataset_builder.py index 50f84e257b9..f74d53067e2 100644 --- a/tensorflow_datasets/datasets/groove/groove_dataset_builder.py +++ b/tensorflow_datasets/datasets/groove/groove_dataset_builder.py @@ -130,12 +130,12 @@ def _info(self): }, "midi": tf.string, } - if self.builder_config.include_audio: - features_dict["audio"] = tfds.features.Audio( - dtype=np.float32, sample_rate=self.builder_config.audio_rate + if self.builder_config.include_audio: # pyrefly: ignore[missing-attribute] + features_dict["audio"] = tfds.features.Audio( # pyrefly: ignore[bad-assignment] + dtype=np.float32, sample_rate=self.builder_config.audio_rate # pyrefly: ignore[missing-attribute] ) return self.dataset_info_from_configs( - features=tfds.features.FeaturesDict(features_dict), + features=tfds.features.FeaturesDict(features_dict), # pyrefly: ignore[bad-argument-type] homepage="https://g.co/magenta/groove-dataset", ) diff --git a/tensorflow_datasets/datasets/imagenet2012/imagenet2012_dataset_builder.py b/tensorflow_datasets/datasets/imagenet2012/imagenet2012_dataset_builder.py index 40b6a649aa4..e965a4e22c5 100644 --- a/tensorflow_datasets/datasets/imagenet2012/imagenet2012_dataset_builder.py +++ b/tensorflow_datasets/datasets/imagenet2012/imagenet2012_dataset_builder.py @@ -101,20 +101,20 @@ def _split_generators(self, dl_manager): splits = [] _add_split_if_exists( split_list=splits, - split=tfds.Split.TRAIN, + split=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] split_path=train_path, dl_manager=dl_manager, ) _add_split_if_exists( split_list=splits, - split=tfds.Split.VALIDATION, + split=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] split_path=val_path, dl_manager=dl_manager, validation_labels=imagenet_common.get_validation_labels(val_path), ) _add_split_if_exists( split_list=splits, - split=tfds.Split.TEST, + split=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] split_path=test_path, dl_manager=dl_manager, labels_exist=False, diff --git a/tensorflow_datasets/datasets/imagenet2012_corrupted/imagenet2012_corrupted_dataset_builder.py b/tensorflow_datasets/datasets/imagenet2012_corrupted/imagenet2012_corrupted_dataset_builder.py index 31eb3d20adb..58c73e9fa05 100644 --- a/tensorflow_datasets/datasets/imagenet2012_corrupted/imagenet2012_corrupted_dataset_builder.py +++ b/tensorflow_datasets/datasets/imagenet2012_corrupted/imagenet2012_corrupted_dataset_builder.py @@ -168,7 +168,7 @@ def _split_generators(self, dl_manager): splits = super(Builder, self)._split_generators(dl_manager) corruptions.FROST_FILENAMES = dl_manager.download(_FROST_FILENAMES) - return [s for s in splits if s.name != tfds.Split.TRAIN] + return [s for s in splits if s.name != tfds.Split.TRAIN] # pyrefly: ignore[missing-attribute] def _generate_examples( self, archive, validation_labels=None, labels_exist=None @@ -215,8 +215,8 @@ def _get_corrupted_example(self, x): Returns: numpy array, corrupted images. """ - corruption_type = self.builder_config.corruption_type - severity = self.builder_config.severity + corruption_type = self.builder_config.corruption_type # pyrefly: ignore[missing-attribute] + severity = self.builder_config.severity # pyrefly: ignore[missing-attribute] x = np.clip(x, 0, 255) return { diff --git a/tensorflow_datasets/datasets/imagenet2012_fewshot/imagenet2012_fewshot_dataset_builder.py b/tensorflow_datasets/datasets/imagenet2012_fewshot/imagenet2012_fewshot_dataset_builder.py index c9c17c5d1a0..94c7bd47178 100644 --- a/tensorflow_datasets/datasets/imagenet2012_fewshot/imagenet2012_fewshot_dataset_builder.py +++ b/tensorflow_datasets/datasets/imagenet2012_fewshot/imagenet2012_fewshot_dataset_builder.py @@ -68,7 +68,7 @@ def _split_generators(self, dl_manager): ) # Download and load subset file. - subset_file = SUBSET2FILES[self.builder_config.name] + subset_file = SUBSET2FILES[self.builder_config.name] # pyrefly: ignore[missing-attribute] if isinstance(subset_file, list): # it will only be a list during testing, subset_file = subset_file[0] # where the first entry is 1shot.txt. subset = set(subset_file.read_text().splitlines()) @@ -77,13 +77,13 @@ def _split_generators(self, dl_manager): tuneset = set(TUNE_FILE.read_text().splitlines()) return { - tfds.Split.TRAIN: self._generate_examples( + tfds.Split.TRAIN: self._generate_examples( # pyrefly: ignore[missing-attribute] archive=dl_manager.iter_archive(train_path), subset=subset ), tfds.Split('tune'): self._generate_examples( archive=dl_manager.iter_archive(train_path), subset=tuneset ), - tfds.Split.VALIDATION: self._generate_examples( + tfds.Split.VALIDATION: self._generate_examples( # pyrefly: ignore[missing-attribute] archive=dl_manager.iter_archive(val_path), validation_labels=imagenet_common.get_validation_labels(val_path), ), diff --git a/tensorflow_datasets/datasets/imagenet2012_real/imagenet2012_real_dataset_builder.py b/tensorflow_datasets/datasets/imagenet2012_real/imagenet2012_real_dataset_builder.py index b3147508e6b..30c8170c00d 100644 --- a/tensorflow_datasets/datasets/imagenet2012_real/imagenet2012_real_dataset_builder.py +++ b/tensorflow_datasets/datasets/imagenet2012_real/imagenet2012_real_dataset_builder.py @@ -72,7 +72,7 @@ def _split_generators(self, dl_manager): ) return [ tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] gen_kwargs={ 'archive': dl_manager.iter_archive(val_path), 'original_labels': imagenet_common.get_validation_labels( diff --git a/tensorflow_datasets/datasets/imagenet2012_subset/imagenet2012_subset_dataset_builder.py b/tensorflow_datasets/datasets/imagenet2012_subset/imagenet2012_subset_dataset_builder.py index 0b7f88d0739..4186ca7c239 100644 --- a/tensorflow_datasets/datasets/imagenet2012_subset/imagenet2012_subset_dataset_builder.py +++ b/tensorflow_datasets/datasets/imagenet2012_subset/imagenet2012_subset_dataset_builder.py @@ -72,7 +72,7 @@ def _split_generators(self, dl_manager): ) # Download and load subset file. - subset_file = dl_manager.download(SUBSET2FILES[self.builder_config.name]) + subset_file = dl_manager.download(SUBSET2FILES[self.builder_config.name]) # pyrefly: ignore[missing-attribute] if isinstance(subset_file, list): # it will only be a list during testing, subset_file = subset_file[0] # where the first entry is 1percent.txt. with epath.Path(subset_file).open() as fp: @@ -80,14 +80,14 @@ def _split_generators(self, dl_manager): return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={ 'archive': dl_manager.iter_archive(train_path), 'subset': subset, }, ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] gen_kwargs={ 'archive': dl_manager.iter_archive(val_path), 'validation_labels': imagenet_common.get_validation_labels( diff --git a/tensorflow_datasets/datasets/imagenet_a/imagenet_a_dataset_builder.py b/tensorflow_datasets/datasets/imagenet_a/imagenet_a_dataset_builder.py index 26804821bac..b33be08a638 100644 --- a/tensorflow_datasets/datasets/imagenet_a/imagenet_a_dataset_builder.py +++ b/tensorflow_datasets/datasets/imagenet_a/imagenet_a_dataset_builder.py @@ -51,7 +51,7 @@ def _split_generators(self, dl_manager): return [ tfds.core.SplitGenerator( # The dataset provides only a test split. - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] # These kwargs will be passed to _generate_examples gen_kwargs={'imagenet_a_root': imagenet_a_root}, ), diff --git a/tensorflow_datasets/datasets/imagenet_r/imagenet_r_dataset_builder.py b/tensorflow_datasets/datasets/imagenet_r/imagenet_r_dataset_builder.py index 1217c1d0a2c..4cda4d3773a 100644 --- a/tensorflow_datasets/datasets/imagenet_r/imagenet_r_dataset_builder.py +++ b/tensorflow_datasets/datasets/imagenet_r/imagenet_r_dataset_builder.py @@ -60,7 +60,7 @@ def _split_generators(self, dl_manager): return [ tfds.core.SplitGenerator( # The dataset provides only a test split. - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] # These kwargs will be passed to _generate_examples gen_kwargs={'imagenet_r_root': imagenet_r_root}, ), diff --git a/tensorflow_datasets/datasets/imagenet_resized/imagenet_resized_dataset_builder.py b/tensorflow_datasets/datasets/imagenet_resized/imagenet_resized_dataset_builder.py index ae1fd8381c7..9a2458e9023 100644 --- a/tensorflow_datasets/datasets/imagenet_resized/imagenet_resized_dataset_builder.py +++ b/tensorflow_datasets/datasets/imagenet_resized/imagenet_resized_dataset_builder.py @@ -64,7 +64,7 @@ class Builder(tfds.core.GeneratorBasedBuilder): def _info(self): names_file = tfds.core.tfds_path(_LABELS_FNAME) - size = self.builder_config.size + size = self.builder_config.size # pyrefly: ignore[missing-attribute] return self.dataset_info_from_configs( features=tfds.features.FeaturesDict({ 'image': tfds.features.Image(shape=(size, size, 3)), @@ -75,7 +75,7 @@ def _info(self): ) def _split_generators(self, dl_manager): - size = self.builder_config.size + size = self.builder_config.size # pyrefly: ignore[missing-attribute] if size in [8, 16, 32]: train_path, val_path = dl_manager.download([ @@ -96,7 +96,7 @@ def _split_generators(self, dl_manager): return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={ 'archive': itertools.chain( *[ @@ -107,7 +107,7 @@ def _split_generators(self, dl_manager): }, ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] gen_kwargs={ 'archive': dl_manager.iter_archive(val_path), }, @@ -121,7 +121,7 @@ def _generate_examples(self, archive): if content: fobj_mem = io.BytesIO(content) data = np.load(fobj_mem, allow_pickle=False) - size = self.builder_config.size + size = self.builder_config.size # pyrefly: ignore[missing-attribute] for i, (image, label) in enumerate(zip(data['data'], data['labels'])): record = { # The data is packed flat as CHW where as most image datasets diff --git a/tensorflow_datasets/datasets/imagenet_sketch/imagenet_sketch_dataset_builder.py b/tensorflow_datasets/datasets/imagenet_sketch/imagenet_sketch_dataset_builder.py index 8617f1342ed..06177f729ab 100644 --- a/tensorflow_datasets/datasets/imagenet_sketch/imagenet_sketch_dataset_builder.py +++ b/tensorflow_datasets/datasets/imagenet_sketch/imagenet_sketch_dataset_builder.py @@ -48,7 +48,7 @@ def _split_generators(self, dl_manager: tfds.download.DownloadManager): path = dl_manager.download(_IMAGENET_SKETCH_URL) return [ tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] gen_kwargs={ 'archive': dl_manager.iter_archive(path), }, diff --git a/tensorflow_datasets/datasets/imagenet_v2/imagenet_v2_dataset_builder.py b/tensorflow_datasets/datasets/imagenet_v2/imagenet_v2_dataset_builder.py index 3d24b12a13a..96512971bb3 100644 --- a/tensorflow_datasets/datasets/imagenet_v2/imagenet_v2_dataset_builder.py +++ b/tensorflow_datasets/datasets/imagenet_v2/imagenet_v2_dataset_builder.py @@ -100,15 +100,15 @@ def _info(self): def _split_generators(self, dl_manager): """Returns a SplitGenerator for the test set.""" - variant_url = _IMAGENET_V2_URLS[self.builder_config.variant] + variant_url = _IMAGENET_V2_URLS[self.builder_config.variant] # pyrefly: ignore[missing-attribute] imagenet_v2_root = os.path.join( dl_manager.download_and_extract(variant_url), - _TAR_TOPDIR[self.builder_config.variant], + _TAR_TOPDIR[self.builder_config.variant], # pyrefly: ignore[missing-attribute] ) return [ tfds.core.SplitGenerator( # The dataset provides only a test split. - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] # These kwargs will be passed to _generate_examples gen_kwargs={'imagenet_v2_root': imagenet_v2_root}, ), diff --git a/tensorflow_datasets/datasets/imagenette/imagenette_dataset_builder.py b/tensorflow_datasets/datasets/imagenette/imagenette_dataset_builder.py index 869eb8367b1..9701eac1463 100644 --- a/tensorflow_datasets/datasets/imagenette/imagenette_dataset_builder.py +++ b/tensorflow_datasets/datasets/imagenette/imagenette_dataset_builder.py @@ -77,7 +77,7 @@ def _info(self): def _split_generators(self, dl_manager): """Returns SplitGenerators.""" - dirname = self.builder_config.dirname + dirname = self.builder_config.dirname # pyrefly: ignore[missing-attribute] url = _URL_PREFIX + "{}.tgz".format(dirname) path = dl_manager.download_and_extract(url) train_path = os.path.join(path, dirname, "train") @@ -85,13 +85,13 @@ def _split_generators(self, dl_manager): return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={ "datapath": train_path, }, ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] gen_kwargs={ "datapath": val_path, }, diff --git a/tensorflow_datasets/datasets/imagewang/imagewang_dataset_builder.py b/tensorflow_datasets/datasets/imagewang/imagewang_dataset_builder.py index 5fd26eb4ac1..40c5f2f9a1a 100644 --- a/tensorflow_datasets/datasets/imagewang/imagewang_dataset_builder.py +++ b/tensorflow_datasets/datasets/imagewang/imagewang_dataset_builder.py @@ -73,7 +73,7 @@ def _info(self): def _split_generators(self, dl_manager): """Returns SplitGenerators.""" - size = self.builder_config.size + size = self.builder_config.size # pyrefly: ignore[missing-attribute] if size in _SIZES: size_str = "" if size == "full-size" else "-" + size[:-2] url = "/".join([_URL_PREFIX, "imagewang%s.tgz" % size_str]) @@ -85,13 +85,13 @@ def _split_generators(self, dl_manager): return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={ "datapath": train_path, }, ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] gen_kwargs={ "datapath": val_path, }, diff --git a/tensorflow_datasets/datasets/imdb_reviews/imdb_reviews_dataset_builder.py b/tensorflow_datasets/datasets/imdb_reviews/imdb_reviews_dataset_builder.py index c09cb597713..d3ea6142c23 100644 --- a/tensorflow_datasets/datasets/imdb_reviews/imdb_reviews_dataset_builder.py +++ b/tensorflow_datasets/datasets/imdb_reviews/imdb_reviews_dataset_builder.py @@ -61,7 +61,7 @@ def _info(self): return self.dataset_info_from_configs( features=tfds.features.FeaturesDict({ "text": tfds.features.Text( - encoder_config=self.builder_config.text_encoder_config + encoder_config=self.builder_config.text_encoder_config # pyrefly: ignore[missing-attribute] ), "label": tfds.features.ClassLabel(names=["neg", "pos"]), }), @@ -86,14 +86,14 @@ def _split_generators(self, dl_manager): return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={ "archive": archive(), "directory": os.path.join("aclImdb", "train"), }, ), tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] gen_kwargs={ "archive": archive(), "directory": os.path.join("aclImdb", "test"), diff --git a/tensorflow_datasets/datasets/irc_disentanglement/irc_disentanglement_dataset_builder.py b/tensorflow_datasets/datasets/irc_disentanglement/irc_disentanglement_dataset_builder.py index 8cf4e567d75..70712215a08 100644 --- a/tensorflow_datasets/datasets/irc_disentanglement/irc_disentanglement_dataset_builder.py +++ b/tensorflow_datasets/datasets/irc_disentanglement/irc_disentanglement_dataset_builder.py @@ -167,7 +167,7 @@ def _split_generators( return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={ "day_to_paths": _get_day_to_paths( os.path.join(data_dir, "train") @@ -175,13 +175,13 @@ def _split_generators( }, ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] gen_kwargs={ "day_to_paths": _get_day_to_paths(os.path.join(data_dir, "dev")) }, ), tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] gen_kwargs={ "day_to_paths": _get_day_to_paths( os.path.join(data_dir, "test") diff --git a/tensorflow_datasets/datasets/kddcup99/kddcup99_dataset_builder.py b/tensorflow_datasets/datasets/kddcup99/kddcup99_dataset_builder.py index edf91355d89..07a8c3db64d 100644 --- a/tensorflow_datasets/datasets/kddcup99/kddcup99_dataset_builder.py +++ b/tensorflow_datasets/datasets/kddcup99/kddcup99_dataset_builder.py @@ -230,8 +230,8 @@ def _split_generators(self, dl_manager: tfds.download.DownloadManager): }) return { - tfds.Split.TRAIN: self._generate_examples(paths['train']), - tfds.Split.TEST: self._generate_examples(paths['test']), + tfds.Split.TRAIN: self._generate_examples(paths['train']), # pyrefly: ignore[missing-attribute] + tfds.Split.TEST: self._generate_examples(paths['test']), # pyrefly: ignore[missing-attribute] } def _generate_examples(self, gz_path): diff --git a/tensorflow_datasets/datasets/kitti/kitti_dataset_builder.py b/tensorflow_datasets/datasets/kitti/kitti_dataset_builder.py index a203849bd8d..9860f916f33 100644 --- a/tensorflow_datasets/datasets/kitti/kitti_dataset_builder.py +++ b/tensorflow_datasets/datasets/kitti/kitti_dataset_builder.py @@ -118,7 +118,7 @@ def _info(self): features=tfds.features.FeaturesDict({ "image": tfds.features.Image(), "image/file_name": tfds.features.Text(), # E.g. "000001.png". - "objects": tfds.features.Sequence(annotations), + "objects": tfds.features.Sequence(annotations), # pyrefly: ignore[bad-argument-type] }), homepage=_HOMEPAGE_URL, ) @@ -136,7 +136,7 @@ def _split_generators(self, dl_manager): return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={ "images": dl_manager.iter_archive(files["images"]), "annotations": dl_manager.iter_archive(files["annotations"]), @@ -145,7 +145,7 @@ def _split_generators(self, dl_manager): }, ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] gen_kwargs={ "images": dl_manager.iter_archive(files["images"]), "annotations": dl_manager.iter_archive(files["annotations"]), @@ -154,7 +154,7 @@ def _split_generators(self, dl_manager): }, ), tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] gen_kwargs={ "images": dl_manager.iter_archive(files["images"]), "annotations": dl_manager.iter_archive(files["annotations"]), diff --git a/tensorflow_datasets/datasets/lambada/lambada_dataset_builder.py b/tensorflow_datasets/datasets/lambada/lambada_dataset_builder.py index 5bcf22452eb..5e4f1a7af14 100644 --- a/tensorflow_datasets/datasets/lambada/lambada_dataset_builder.py +++ b/tensorflow_datasets/datasets/lambada/lambada_dataset_builder.py @@ -52,7 +52,7 @@ def _split_generators(self, dl_manager: tfds.download.DownloadManager): return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={ 'filepath': os.path.join( dl_dir, 'lambada_development_plain_text.txt' @@ -60,7 +60,7 @@ def _split_generators(self, dl_manager: tfds.download.DownloadManager): }, ), tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] gen_kwargs={ 'filepath': os.path.join(dl_dir, 'lambada_test_plain_text.txt') }, diff --git a/tensorflow_datasets/datasets/lbpp/lbpp_dataset_builder.py b/tensorflow_datasets/datasets/lbpp/lbpp_dataset_builder.py index f7d5a3edf82..3d129566a30 100644 --- a/tensorflow_datasets/datasets/lbpp/lbpp_dataset_builder.py +++ b/tensorflow_datasets/datasets/lbpp/lbpp_dataset_builder.py @@ -130,7 +130,7 @@ def _info(self): def _split_generators(self, dl_manager): # Map alias to actual language data_loading_name = _LANGUAGE_ALIAS_MAP.get( - self.builder_config.name, self.builder_config.name + self.builder_config.name, self.builder_config.name # pyrefly: ignore[missing-attribute] ) hf_url_prefix = ( "https://huggingface.co/datasets/CohereForAI/lbpp/resolve/main/" @@ -147,7 +147,7 @@ def _split_generators(self, dl_manager): return [ tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] gen_kwargs={ "filepaths": downloaded_files, }, diff --git a/tensorflow_datasets/datasets/lfw/lfw_dataset_builder.py b/tensorflow_datasets/datasets/lfw/lfw_dataset_builder.py index 3cd503346ae..7ea90c6ec63 100644 --- a/tensorflow_datasets/datasets/lfw/lfw_dataset_builder.py +++ b/tensorflow_datasets/datasets/lfw/lfw_dataset_builder.py @@ -49,7 +49,7 @@ def _split_generators(self, dl_manager): # There is no train/test split predefined return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={ "data_path": path, }, diff --git a/tensorflow_datasets/datasets/librispeech/librispeech_dataset_builder.py b/tensorflow_datasets/datasets/librispeech/librispeech_dataset_builder.py index f31a551e666..f310b21d2a0 100644 --- a/tensorflow_datasets/datasets/librispeech/librispeech_dataset_builder.py +++ b/tensorflow_datasets/datasets/librispeech/librispeech_dataset_builder.py @@ -75,7 +75,7 @@ def _info(self): "speech": tfds.features.Audio( sample_rate=16000, dtype=np.int16, - lazy_decode=self.builder_config.lazy_decode, + lazy_decode=self.builder_config.lazy_decode, # pyrefly: ignore[missing-attribute] file_format="flac", ), "text": tfds.features.Text(), @@ -125,10 +125,10 @@ def _read_metadata_file(self, path: epath.Path, field_names: List[str]): def _split_generators(self, dl_manager: tfds.download.DownloadManager): extracted_dirs = dl_manager.download_and_extract(_DL_URLS) - self._populate_metadata(extracted_dirs.values()) + self._populate_metadata(extracted_dirs.values()) # pyrefly: ignore[missing-attribute] splits = { split: self._generate_examples(directory) - for split, directory in extracted_dirs.items() + for split, directory in extracted_dirs.items() # pyrefly: ignore[missing-attribute] } return splits diff --git a/tensorflow_datasets/datasets/librispeech_lm/librispeech_lm_dataset_builder.py b/tensorflow_datasets/datasets/librispeech_lm/librispeech_lm_dataset_builder.py index 6f76ca58242..c5a6fc9cfee 100644 --- a/tensorflow_datasets/datasets/librispeech_lm/librispeech_lm_dataset_builder.py +++ b/tensorflow_datasets/datasets/librispeech_lm/librispeech_lm_dataset_builder.py @@ -43,7 +43,7 @@ def _split_generators(self, dl_manager): archive_path = dl_manager.download(_DL_URL) return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={'files_iter': dl_manager.iter_archive(archive_path)}, ), ] diff --git a/tensorflow_datasets/datasets/libritts/libritts_dataset_builder.py b/tensorflow_datasets/datasets/libritts/libritts_dataset_builder.py index bf8ec81b4f7..2013a295fd6 100644 --- a/tensorflow_datasets/datasets/libritts/libritts_dataset_builder.py +++ b/tensorflow_datasets/datasets/libritts/libritts_dataset_builder.py @@ -69,7 +69,7 @@ def _populate_metadata(self, archive_paths): with tf.io.gfile.GFile(archive_path, "rb") as f: tarf = tarfile.open(mode="r:gz", fileobj=f) speakers_tsv = tarf.extractfile("LibriTTS/speakers.tsv") - for n, line in enumerate(speakers_tsv): + for n, line in enumerate(speakers_tsv): # pyrefly: ignore[bad-argument-type] # Skip the first line which is just a header. if n == 0: continue diff --git a/tensorflow_datasets/datasets/ljspeech/ljspeech_dataset_builder.py b/tensorflow_datasets/datasets/ljspeech/ljspeech_dataset_builder.py index ba5c43859fb..afb5a469d64 100644 --- a/tensorflow_datasets/datasets/ljspeech/ljspeech_dataset_builder.py +++ b/tensorflow_datasets/datasets/ljspeech/ljspeech_dataset_builder.py @@ -52,7 +52,7 @@ def _split_generators(self, dl_manager): extracted_dir = dl_manager.download_and_extract(_DL_URL) return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={"directory": extracted_dir}, ), ] diff --git a/tensorflow_datasets/datasets/lm1b/lm1b_dataset_builder.py b/tensorflow_datasets/datasets/lm1b/lm1b_dataset_builder.py index d1fdd7f9c4c..831a4594f9f 100644 --- a/tensorflow_datasets/datasets/lm1b/lm1b_dataset_builder.py +++ b/tensorflow_datasets/datasets/lm1b/lm1b_dataset_builder.py @@ -67,10 +67,10 @@ def _split_generators(self, dl_manager): test_files = _test_data_filenames(lm1b_path) return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, gen_kwargs={"files": train_files} + name=tfds.Split.TRAIN, gen_kwargs={"files": train_files} # pyrefly: ignore[missing-attribute] ), tfds.core.SplitGenerator( - name=tfds.Split.TEST, gen_kwargs={"files": test_files} + name=tfds.Split.TEST, gen_kwargs={"files": test_files} # pyrefly: ignore[missing-attribute] ), ] diff --git a/tensorflow_datasets/datasets/lost_and_found/lost_and_found_dataset_builder.py b/tensorflow_datasets/datasets/lost_and_found/lost_and_found_dataset_builder.py index 83524af8541..bb06a3bcb3e 100644 --- a/tensorflow_datasets/datasets/lost_and_found/lost_and_found_dataset_builder.py +++ b/tensorflow_datasets/datasets/lost_and_found/lost_and_found_dataset_builder.py @@ -125,9 +125,9 @@ def _info(self): ), } features = { - feat: possible_features[feat] for feat in self.builder_config.features + feat: possible_features[feat] for feat in self.builder_config.features # pyrefly: ignore[missing-attribute] } - features['image_id'] = tfds.features.Text() + features['image_id'] = tfds.features.Text() # pyrefly: ignore[unsupported-operation] features = tfds.features.FeaturesDict(features) return self.dataset_info_from_configs( # tfds.features.FeatureConnectors @@ -145,8 +145,8 @@ def _split_generators(self, dl_manager): # For each feature, this is the name of the zipfile and # root-directory in the archive zip_file_names = { - 'image_left': self.builder_config.left_image_string, - 'image_right': self.builder_config.right_image_string, + 'image_left': self.builder_config.left_image_string, # pyrefly: ignore[missing-attribute] + 'image_right': self.builder_config.right_image_string, # pyrefly: ignore[missing-attribute] 'segmentation_label': 'gtCoarse', 'instance_id': 'gtCoarse', 'disparity_map': 'disparity', @@ -154,7 +154,7 @@ def _split_generators(self, dl_manager): download_urls = { feat: base_url.format(zip_file_names[feat]) - for feat in self.builder_config.features + for feat in self.builder_config.features # pyrefly: ignore[missing-attribute] } # Split download and extract in two functions such that mock-data can # replace the result of the download function and is still used as input to @@ -163,19 +163,19 @@ def _split_generators(self, dl_manager): return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] # These kwargs will be passed to _generate_examples gen_kwargs={ feat: path.join(dl_paths[feat], zip_file_names[feat], 'train') - for feat in self.builder_config.features + for feat in self.builder_config.features # pyrefly: ignore[missing-attribute] }, ), tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] # These kwargs will be passed to _generate_examples gen_kwargs={ feat: path.join(dl_paths[feat], zip_file_names[feat], 'test') - for feat in self.builder_config.features + for feat in self.builder_config.features # pyrefly: ignore[missing-attribute] }, ), ] @@ -184,8 +184,8 @@ def _generate_examples(self, **paths): """Yields examples.""" # different file-suffixes dependent on the feature to load file_suffix = { - 'image_left': self.builder_config.left_image_string, - 'image_right': self.builder_config.right_image_string, + 'image_left': self.builder_config.left_image_string, # pyrefly: ignore[missing-attribute] + 'image_right': self.builder_config.right_image_string, # pyrefly: ignore[missing-attribute] 'segmentation_label': 'gtCoarse_labelIds', 'instance_id': 'gtCoarse_instanceIds', 'disparity_map': 'disparity', @@ -232,4 +232,4 @@ def _get_id_from_left_image(left_image): Returns: id of the image. """ - return LEFT_IMAGE_FILE_RE.match(left_image).group(1) + return LEFT_IMAGE_FILE_RE.match(left_image).group(1) # pyrefly: ignore[missing-attribute] diff --git a/tensorflow_datasets/datasets/lvis/lvis_dataset_builder.py b/tensorflow_datasets/datasets/lvis/lvis_dataset_builder.py index dafa540128d..a348c0d3b96 100644 --- a/tensorflow_datasets/datasets/lvis/lvis_dataset_builder.py +++ b/tensorflow_datasets/datasets/lvis/lvis_dataset_builder.py @@ -123,8 +123,8 @@ def _info(self) -> tfds.core.DatasetInfo: def _split_generators(self, dl_manager: tfds.download.DownloadManager): """Returns SplitGenerators.""" paths = { - **dl_manager.download_and_extract(_EXTRACT_URLS), - **dl_manager.download(_URLS), + **dl_manager.download_and_extract(_EXTRACT_URLS), # pyrefly: ignore[invalid-argument] + **dl_manager.download(_URLS), # pyrefly: ignore[invalid-argument] } image_dirs = [ paths['train_images'] / 'train2017', @@ -132,13 +132,13 @@ def _split_generators(self, dl_manager: tfds.download.DownloadManager): paths['test_images'] / 'test2017', ] return { - tfds.Split.TRAIN: self._generate_examples( + tfds.Split.TRAIN: self._generate_examples( # pyrefly: ignore[missing-attribute] image_dirs, paths['train_annotation'] / 'lvis_v1_train.json' ), - tfds.Split.VALIDATION: self._generate_examples( + tfds.Split.VALIDATION: self._generate_examples( # pyrefly: ignore[missing-attribute] image_dirs, paths['validation_annotation'] / 'lvis_v1_val.json' ), - tfds.Split.TEST: self._generate_examples( + tfds.Split.TEST: self._generate_examples( # pyrefly: ignore[missing-attribute] image_dirs, paths['test_annotation'] / 'lvis_v1_image_info_test_dev.json', ), diff --git a/tensorflow_datasets/datasets/malaria/malaria_dataset_builder.py b/tensorflow_datasets/datasets/malaria/malaria_dataset_builder.py index 4a991ab4ef8..338055bc2ea 100644 --- a/tensorflow_datasets/datasets/malaria/malaria_dataset_builder.py +++ b/tensorflow_datasets/datasets/malaria/malaria_dataset_builder.py @@ -56,7 +56,7 @@ def _split_generators(self, dl_manager): return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={ "data_dir_path": os.path.join(path, "cell_images"), }, diff --git a/tensorflow_datasets/datasets/math_dataset/math_dataset_dataset_builder.py b/tensorflow_datasets/datasets/math_dataset/math_dataset_dataset_builder.py index a3a6787e1fb..50a094159d8 100644 --- a/tensorflow_datasets/datasets/math_dataset/math_dataset_dataset_builder.py +++ b/tensorflow_datasets/datasets/math_dataset/math_dataset_dataset_builder.py @@ -208,11 +208,11 @@ def _split_generators(self, dl_manager): """Returns SplitGenerators.""" directory = dl_manager.download_and_extract(_DATA_URL) - config = self.builder_config.name + ".txt" + config = self.builder_config.name + ".txt" # pyrefly: ignore[missing-attribute] return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={ "directory": directory, "config": config, @@ -220,7 +220,7 @@ def _split_generators(self, dl_manager): }, ), tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] gen_kwargs={ "directory": directory, "config": config, diff --git a/tensorflow_datasets/datasets/math_qa/math_qa_dataset_builder.py b/tensorflow_datasets/datasets/math_qa/math_qa_dataset_builder.py index 6cb68121bd8..01d22808b05 100644 --- a/tensorflow_datasets/datasets/math_qa/math_qa_dataset_builder.py +++ b/tensorflow_datasets/datasets/math_qa/math_qa_dataset_builder.py @@ -58,13 +58,13 @@ def _split_generators(self, dl_manager: tfds.download.DownloadManager): """Returns SplitGenerators.""" path = dl_manager.download_and_extract(_URL) return { - tfds.Split.TRAIN: self._generate_examples( + tfds.Split.TRAIN: self._generate_examples( # pyrefly: ignore[missing-attribute] os.path.join(path, 'train.json') ), - tfds.Split.VALIDATION: self._generate_examples( + tfds.Split.VALIDATION: self._generate_examples( # pyrefly: ignore[missing-attribute] os.path.join(path, 'dev.json') ), - tfds.Split.TEST: self._generate_examples( + tfds.Split.TEST: self._generate_examples( # pyrefly: ignore[missing-attribute] os.path.join(path, 'test.json') ), } diff --git a/tensorflow_datasets/datasets/mlqa/mlqa_dataset_builder.py b/tensorflow_datasets/datasets/mlqa/mlqa_dataset_builder.py index ab404a1fd55..dc42a54f491 100644 --- a/tensorflow_datasets/datasets/mlqa/mlqa_dataset_builder.py +++ b/tensorflow_datasets/datasets/mlqa/mlqa_dataset_builder.py @@ -62,7 +62,7 @@ def _info(self): ) def _split_generators(self, dl_manager): - lang = self.builder_config.language + lang = self.builder_config.language # pyrefly: ignore[missing-attribute] filepaths = dl_manager.download_and_extract({ "test": _DOWNLOAD_URL, "validation": _DOWNLOAD_URL, diff --git a/tensorflow_datasets/datasets/multi_news/multi_news_dataset_builder.py b/tensorflow_datasets/datasets/multi_news/multi_news_dataset_builder.py index d1b7225abe6..7127ca27842 100644 --- a/tensorflow_datasets/datasets/multi_news/multi_news_dataset_builder.py +++ b/tensorflow_datasets/datasets/multi_news/multi_news_dataset_builder.py @@ -69,12 +69,12 @@ def _split_generators(self, dl_manager: tfds.download.DownloadManager): files = dl_manager.download_and_extract(data_dict) return { "train": self._generate_examples( - files["train_src"], files["train_tgt"] + files["train_src"], files["train_tgt"] # pyrefly: ignore[bad-index] ), "validation": self._generate_examples( - files["val_src"], files["val_tgt"] + files["val_src"], files["val_tgt"] # pyrefly: ignore[bad-index] ), - "test": self._generate_examples(files["test_src"], files["test_tgt"]), + "test": self._generate_examples(files["test_src"], files["test_tgt"]), # pyrefly: ignore[bad-index] } def _generate_examples(self, src_file, tgt_file): diff --git a/tensorflow_datasets/datasets/natural_instructions/natural_instructions_dataset_builder.py b/tensorflow_datasets/datasets/natural_instructions/natural_instructions_dataset_builder.py index f803c7e3612..86ce4d4fdde 100644 --- a/tensorflow_datasets/datasets/natural_instructions/natural_instructions_dataset_builder.py +++ b/tensorflow_datasets/datasets/natural_instructions/natural_instructions_dataset_builder.py @@ -58,7 +58,7 @@ def _split_generators(self, dl_manager: tfds.download.DownloadManager): # Downloads the data and defines the splits data_dir = dl_manager.download_and_extract(name_to_paths) return { - tfds.Split.TRAIN: self._generate_examples(path=data_dir), + tfds.Split.TRAIN: self._generate_examples(path=data_dir), # pyrefly: ignore[missing-attribute] } def _generate_examples(self, path): diff --git a/tensorflow_datasets/datasets/natural_questions/natural_questions_dataset_builder.py b/tensorflow_datasets/datasets/natural_questions/natural_questions_dataset_builder.py index 24d88b121bc..7ada0daf8ef 100644 --- a/tensorflow_datasets/datasets/natural_questions/natural_questions_dataset_builder.py +++ b/tensorflow_datasets/datasets/natural_questions/natural_questions_dataset_builder.py @@ -147,7 +147,7 @@ class Builder(tfds.core.BeamBasedBuilder): def _info(self): return self.dataset_info_from_configs( - features=self.builder_config.features, + features=self.builder_config.features, # pyrefly: ignore[missing-attribute] supervised_keys=None, homepage=_URL, ) @@ -159,11 +159,11 @@ def _split_generators(self, dl_manager): return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={'filepaths': files['train']}, ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] gen_kwargs={'filepaths': files['validation']}, ), ] @@ -316,7 +316,7 @@ def _build_pcollection(self, pipeline, filepaths): parse_example = { 'default': self._parse_example_default, 'longt5': self._parse_example_longt5, - }[self.builder_config.name] + }[self.builder_config.name] # pyrefly: ignore[missing-attribute] return ( pipeline diff --git a/tensorflow_datasets/datasets/newsroom/newsroom_dataset_builder.py b/tensorflow_datasets/datasets/newsroom/newsroom_dataset_builder.py index 3e7eeb042d2..e1b42cdab93 100644 --- a/tensorflow_datasets/datasets/newsroom/newsroom_dataset_builder.py +++ b/tensorflow_datasets/datasets/newsroom/newsroom_dataset_builder.py @@ -55,7 +55,7 @@ def _info(self): k: tfds.features.Text() for k in [_DOCUMENT, _SUMMARY] + _ADDITIONAL_TEXT_FEATURES } - features.update( + features.update( # pyrefly: ignore[no-matching-overload] { k: tfds.features.Tensor(shape=[], dtype=np.float32) for k in _ADDITIONAL_FLOAT_FEATURES @@ -71,19 +71,19 @@ def _split_generators(self, dl_manager): """Returns SplitGenerators.""" return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={ "input_file": os.path.join(dl_manager.manual_dir, "train.jsonl") }, ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] gen_kwargs={ "input_file": os.path.join(dl_manager.manual_dir, "dev.jsonl") }, ), tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] gen_kwargs={ "input_file": os.path.join(dl_manager.manual_dir, "test.jsonl") }, @@ -92,7 +92,7 @@ def _split_generators(self, dl_manager): def _generate_examples(self, input_file=None): """Yields examples.""" - with epath.Path(input_file).open() as f: + with epath.Path(input_file).open() as f: # pyrefly: ignore[bad-argument-type] for i, line in enumerate(f): d = json.loads(line) # fields are "url", "archive", "title", "date", "text", diff --git a/tensorflow_datasets/datasets/nsynth/nsynth_dataset_builder.py b/tensorflow_datasets/datasets/nsynth/nsynth_dataset_builder.py index b91b5bf3ccd..b706fc5f78d 100644 --- a/tensorflow_datasets/datasets/nsynth/nsynth_dataset_builder.py +++ b/tensorflow_datasets/datasets/nsynth/nsynth_dataset_builder.py @@ -293,20 +293,20 @@ def _info(self): }, "qualities": {quality: tf.bool for quality in _QUALITIES}, } - if self.builder_config.estimate_f0_and_loudness: + if self.builder_config.estimate_f0_and_loudness: # pyrefly: ignore[missing-attribute] f0_and_ld_shape = (_F0_AND_LOUDNESS_RATE * _NUM_SECS,) - features["f0"] = { + features["f0"] = { # pyrefly: ignore[bad-assignment] "hz": tfds.features.Tensor(shape=f0_and_ld_shape, dtype=np.float32), "midi": tfds.features.Tensor(shape=f0_and_ld_shape, dtype=np.float32), "confidence": tfds.features.Tensor( shape=f0_and_ld_shape, dtype=np.float32 ), } - features["loudness"] = { + features["loudness"] = { # pyrefly: ignore[bad-assignment] "db": tfds.features.Tensor(shape=f0_and_ld_shape, dtype=np.float32) } return self.dataset_info_from_configs( - features=tfds.features.FeaturesDict(features), + features=tfds.features.FeaturesDict(features), # pyrefly: ignore[bad-argument-type] homepage="https://g.co/magenta/nsynth-dataset", metadata=tfds.core.BeamMetadataDict( sample_rate=_AUDIO_RATE, @@ -322,7 +322,7 @@ def _split_generators(self, dl_manager): for split in _SPLITS } dl_urls["instrument_labels"] = _BASE_DOWNLOAD_PATH + "instrument_labels.txt" - if self.builder_config.gansynth_subset: + if self.builder_config.gansynth_subset: # pyrefly: ignore[missing-attribute] dl_urls["gansynth_splits"] = _BASE_DOWNLOAD_PATH + "gansynth_splits.csv" dl_paths = dl_manager.download_and_extract(dl_urls) @@ -365,7 +365,7 @@ def _build_pcollection(self, pipeline, tfrecord_dirs, ids, split): | beam.Map(_emit_base_example, split=split) | beam.Filter(_in_split, split_ids=ids, split=split) ) - if self.builder_config.estimate_f0_and_loudness: + if self.builder_config.estimate_f0_and_loudness: # pyrefly: ignore[missing-attribute] examples = ( examples | beam.Reshuffle() diff --git a/tensorflow_datasets/datasets/nyu_depth_v2/nyu_depth_v2_dataset_builder.py b/tensorflow_datasets/datasets/nyu_depth_v2/nyu_depth_v2_dataset_builder.py index e97992095e3..ec56e21c28f 100644 --- a/tensorflow_datasets/datasets/nyu_depth_v2/nyu_depth_v2_dataset_builder.py +++ b/tensorflow_datasets/datasets/nyu_depth_v2/nyu_depth_v2_dataset_builder.py @@ -46,13 +46,13 @@ def _split_generators(self, dl_manager): return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={ 'root_dir': os.path.join(base_path, 'nyudepthv2', 'train') }, ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] gen_kwargs={ 'root_dir': os.path.join(base_path, 'nyudepthv2', 'val') }, diff --git a/tensorflow_datasets/datasets/open_images_challenge2019_detection/open_images_challenge2019_detection_dataset_builder.py b/tensorflow_datasets/datasets/open_images_challenge2019_detection/open_images_challenge2019_detection_dataset_builder.py index 3126509701c..7a494bf0fd2 100644 --- a/tensorflow_datasets/datasets/open_images_challenge2019_detection/open_images_challenge2019_detection_dataset_builder.py +++ b/tensorflow_datasets/datasets/open_images_challenge2019_detection/open_images_challenge2019_detection_dataset_builder.py @@ -97,15 +97,15 @@ def _split_generators(self, dl_manager): paths = dl_manager.download(urls) return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs=dict(paths=paths, split="train"), ), tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] gen_kwargs=dict(paths=paths, split="test"), ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] gen_kwargs=dict(paths=paths, split="validation"), ), ] @@ -188,7 +188,7 @@ def _build_pcollection(self, pipeline, paths, split): | "ProcessImages" >> beam.ParDo( oi_beam.ProcessImageFn( - target_pixels=self.builder_config.target_pixels, jpeg_quality=72 + target_pixels=self.builder_config.target_pixels, jpeg_quality=72 # pyrefly: ignore[missing-attribute] ) ) | "GenerateExamples" diff --git a/tensorflow_datasets/datasets/open_images_v4/open_images_v4_dataset_builder.py b/tensorflow_datasets/datasets/open_images_v4/open_images_v4_dataset_builder.py index 83b4384da4b..33963f1455e 100644 --- a/tensorflow_datasets/datasets/open_images_v4/open_images_v4_dataset_builder.py +++ b/tensorflow_datasets/datasets/open_images_v4/open_images_v4_dataset_builder.py @@ -215,7 +215,7 @@ def load_boxes(name): validation_bbox = load_boxes('validation-annotations-bbox') return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs=dict( archive_paths=paths['train_images'], objects_getter=train_objects, @@ -224,7 +224,7 @@ def load_boxes(name): ), ), tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] gen_kwargs=dict( archive_paths=[paths['test_images']], objects_getter=test_objects, @@ -232,7 +232,7 @@ def load_boxes(name): ), ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] gen_kwargs=dict( archive_paths=[paths['validation_images']], objects_getter=validation_objects, @@ -266,7 +266,7 @@ def _generate_examples( ] record = { 'image': _resize_image_if_necessary( - fobj, target_pixels=self.builder_config.target_pixels + fobj, target_pixels=self.builder_config.target_pixels # pyrefly: ignore[missing-attribute] ), 'image/filename': fname, 'objects': image_objects, diff --git a/tensorflow_datasets/datasets/openbookqa/openbookqa_dataset_builder.py b/tensorflow_datasets/datasets/openbookqa/openbookqa_dataset_builder.py index 2754f57152c..e1cf64277bf 100644 --- a/tensorflow_datasets/datasets/openbookqa/openbookqa_dataset_builder.py +++ b/tensorflow_datasets/datasets/openbookqa/openbookqa_dataset_builder.py @@ -60,7 +60,7 @@ def _split_generators(self, dl_manager): data_dir = os.path.join(dl_dir, 'OpenBookQA-V1-Sep2018/Data/Additional') return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] # These kwargs will be passed to _generate_examples gen_kwargs={ 'data_dir': data_dir, @@ -68,7 +68,7 @@ def _split_generators(self, dl_manager): }, ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] # These kwargs will be passed to _generate_examples gen_kwargs={ 'data_dir': data_dir, @@ -76,7 +76,7 @@ def _split_generators(self, dl_manager): }, ), tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] # These kwargs will be passed to _generate_examples gen_kwargs={ 'data_dir': data_dir, diff --git a/tensorflow_datasets/datasets/opinion_abstracts/opinion_abstracts_dataset_builder.py b/tensorflow_datasets/datasets/opinion_abstracts/opinion_abstracts_dataset_builder.py index 55b2aef6c0f..6586dbd24fc 100644 --- a/tensorflow_datasets/datasets/opinion_abstracts/opinion_abstracts_dataset_builder.py +++ b/tensorflow_datasets/datasets/opinion_abstracts/opinion_abstracts_dataset_builder.py @@ -81,14 +81,14 @@ def _info(self) -> tfds.core.DatasetInfo: config = self.builder_config return self.dataset_info_from_configs( features=tfds.features.FeaturesDict({ - config.name_key: np.str_, - config.id_key: np.str_, - config.summary_key: np.str_, - config.opinions_key: tfds.features.Sequence( + config.name_key: np.str_, # pyrefly: ignore[missing-attribute] + config.id_key: np.str_, # pyrefly: ignore[missing-attribute] + config.summary_key: np.str_, # pyrefly: ignore[missing-attribute] + config.opinions_key: tfds.features.Sequence( # pyrefly: ignore[missing-attribute] tfds.features.FeaturesDict({"key": np.str_, "value": np.str_}) ), }), - supervised_keys=(config.opinions_key, config.summary_key), + supervised_keys=(config.opinions_key, config.summary_key), # pyrefly: ignore[missing-attribute] homepage="https://web.eecs.umich.edu/~wangluxy/data.html", ) @@ -98,11 +98,11 @@ def _split_generators( """Returns SplitGenerators.""" dl_path = dl_manager.download_and_extract(_URL) path = os.path.join( - dl_path, "opinion_abstracts", self.builder_config.filename + dl_path, "opinion_abstracts", self.builder_config.filename # pyrefly: ignore[missing-attribute] ) return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={"path": path}, ), ] @@ -116,7 +116,7 @@ def _generate_examples( config = self.builder_config opinions = example[config.opinions_key].items() opinions = [{"key": k, "value": v} for k, v in opinions] - features = {config.opinions_key: opinions} - for k in [config.name_key, config.id_key, config.summary_key]: + features = {config.opinions_key: opinions} # pyrefly: ignore[missing-attribute] + for k in [config.name_key, config.id_key, config.summary_key]: # pyrefly: ignore[missing-attribute] features[k] = example[k] yield example[config.id_key], features diff --git a/tensorflow_datasets/datasets/opinosis/opinosis_dataset_builder.py b/tensorflow_datasets/datasets/opinosis/opinosis_dataset_builder.py index 4c71becf935..5bc5f43530e 100644 --- a/tensorflow_datasets/datasets/opinosis/opinosis_dataset_builder.py +++ b/tensorflow_datasets/datasets/opinosis/opinosis_dataset_builder.py @@ -46,21 +46,21 @@ def _split_generators(self, dl_manager): extract_path = dl_manager.download_and_extract(_URL) return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={"path": extract_path}, ), ] def _generate_examples(self, path=None): """Yields examples.""" - topics_path = os.path.join(path, "topics") + topics_path = os.path.join(path, "topics") # pyrefly: ignore[no-matching-overload] filenames = tf.io.gfile.listdir(topics_path) for filename in filenames: file_path = os.path.join(topics_path, filename) topic_name = filename.split(".txt")[0] with tf.io.gfile.GFile(file_path, "rb") as src_f: input_data = src_f.read() - summaries_path = os.path.join(path, "summaries-gold", topic_name) + summaries_path = os.path.join(path, "summaries-gold", topic_name) # pyrefly: ignore[no-matching-overload] summary_lst = [] for summ_filename in sorted(tf.io.gfile.listdir(summaries_path)): file_path = os.path.join(summaries_path, summ_filename) diff --git a/tensorflow_datasets/datasets/opus/opus_dataset_builder.py b/tensorflow_datasets/datasets/opus/opus_dataset_builder.py index 9daab6d40b8..13312b0b638 100644 --- a/tensorflow_datasets/datasets/opus/opus_dataset_builder.py +++ b/tensorflow_datasets/datasets/opus/opus_dataset_builder.py @@ -907,26 +907,26 @@ class Builder(tfds.core.GeneratorBasedBuilder): @property def subsets(self): # Return only the datasets that exist for the language pair. - source, target = self.builder_config.language_pair + source, target = self.builder_config.language_pair # pyrefly: ignore[missing-attribute] filtered_subsets = [] - for dataset in [DATASET_MAP[name] for name in self.builder_config.subsets]: + for dataset in [DATASET_MAP[name] for name in self.builder_config.subsets]: # pyrefly: ignore[missing-attribute] if (source, target) in dataset.language_pairs: filtered_subsets.append(dataset) return filtered_subsets def _info(self): - src, target = self.builder_config.language_pair + src, target = self.builder_config.language_pair # pyrefly: ignore[missing-attribute] return self.dataset_info_from_configs( features=tfds.features.Translation( - languages=self.builder_config.language_pair + languages=self.builder_config.language_pair # pyrefly: ignore[missing-attribute] ), supervised_keys=(src, target), homepage="http://opus.nlpl.eu/", ) def _split_generators(self, dl_manager): - source, target = self.builder_config.language_pair + source, target = self.builder_config.language_pair # pyrefly: ignore[missing-attribute] file_ext = "%s-%s" % (source, target) subsets = [] @@ -948,12 +948,12 @@ def _split_generators(self, dl_manager): return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, gen_kwargs={"subsets": subsets} + name=tfds.Split.TRAIN, gen_kwargs={"subsets": subsets} # pyrefly: ignore[missing-attribute] ) ] def _generate_examples(self, subsets): - source, target = self.builder_config.language_pair + source, target = self.builder_config.language_pair # pyrefly: ignore[missing-attribute] for item in subsets: logging.info("Generating examples from: %s", item["name"]) diff --git a/tensorflow_datasets/datasets/oxford_flowers102/oxford_flowers102_dataset_builder.py b/tensorflow_datasets/datasets/oxford_flowers102/oxford_flowers102_dataset_builder.py index b5f3f943b56..a2764b0b7e5 100644 --- a/tensorflow_datasets/datasets/oxford_flowers102/oxford_flowers102_dataset_builder.py +++ b/tensorflow_datasets/datasets/oxford_flowers102/oxford_flowers102_dataset_builder.py @@ -163,15 +163,15 @@ def _split_generators(self, dl_manager): return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs=dict(split_name="trnid", **gen_kwargs), ), tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] gen_kwargs=dict(split_name="tstid", **gen_kwargs), ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] gen_kwargs=dict(split_name="valid", **gen_kwargs), ), ] diff --git a/tensorflow_datasets/datasets/para_crawl/para_crawl_dataset_builder.py b/tensorflow_datasets/datasets/para_crawl/para_crawl_dataset_builder.py index 6f29bcf5459..34595f449e1 100644 --- a/tensorflow_datasets/datasets/para_crawl/para_crawl_dataset_builder.py +++ b/tensorflow_datasets/datasets/para_crawl/para_crawl_dataset_builder.py @@ -114,7 +114,7 @@ class Builder(tfds.core.GeneratorBasedBuilder): ] def _info(self): - target_language = self.builder_config.target_language + target_language = self.builder_config.target_language # pyrefly: ignore[missing-attribute] return self.dataset_info_from_configs( features=tfds.features.Translation( languages=("en", target_language), @@ -126,17 +126,17 @@ def _info(self): def _split_generators(self, dl_manager): # Download the data file. data_file = dl_manager.download_and_extract( - {"data_file": self.builder_config.data_url} + {"data_file": self.builder_config.data_url} # pyrefly: ignore[missing-attribute] ) # Return the single split of the data. return [ - tfds.core.SplitGenerator(name=tfds.Split.TRAIN, gen_kwargs=data_file) + tfds.core.SplitGenerator(name=tfds.Split.TRAIN, gen_kwargs=data_file) # pyrefly: ignore[missing-attribute] ] def _generate_examples(self, data_file): """This function returns the examples in the raw (text) form.""" - target_language = self.builder_config.target_language + target_language = self.builder_config.target_language # pyrefly: ignore[missing-attribute] with epath.Path(data_file).open() as f: for idx, line in enumerate(f): diff --git a/tensorflow_datasets/datasets/pass/pass_dataset_builder.py b/tensorflow_datasets/datasets/pass/pass_dataset_builder.py index 81f77305560..0921dd2b377 100644 --- a/tensorflow_datasets/datasets/pass/pass_dataset_builder.py +++ b/tensorflow_datasets/datasets/pass/pass_dataset_builder.py @@ -78,14 +78,14 @@ def _split_generators(self, dl_manager: tfds.download.DownloadManager): """Returns SplitGenerators.""" pd = tfds.core.lazy_imports.pandas paths = dl_manager.download(_URLS) - with tf.io.gfile.GFile(paths['meta_data']) as f: + with tf.io.gfile.GFile(paths['meta_data']) as f: # pyrefly: ignore[bad-index] meta = pd.read_csv(f) meta = meta.set_index('hash') return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs=dict( - parts=paths['train_images'], meta=meta, dl_manager=dl_manager + parts=paths['train_images'], meta=meta, dl_manager=dl_manager # pyrefly: ignore[bad-index] ), ) ] diff --git a/tensorflow_datasets/object_detection/coco.py b/tensorflow_datasets/object_detection/coco.py index 9eba97d98c9..895e3d1db3d 100644 --- a/tensorflow_datasets/object_detection/coco.py +++ b/tensorflow_datasets/object_detection/coco.py @@ -103,19 +103,19 @@ class Coco(tfds.core.GeneratorBasedBuilder): description=_CONFIG_DESCRIPTION.format(year=2014), splits=[ Split( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] images='train2014', annotations='annotations_trainval2014', annotation_type=AnnotationType.BBOXES, ), Split( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] images='val2014', annotations='annotations_trainval2014', annotation_type=AnnotationType.BBOXES, ), Split( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] images='test2014', annotations='image_info_test2014', annotation_type=AnnotationType.NONE, @@ -134,19 +134,19 @@ class Coco(tfds.core.GeneratorBasedBuilder): description=_CONFIG_DESCRIPTION.format(year=2017), splits=[ Split( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] images='train2017', annotations='annotations_trainval2017', annotation_type=AnnotationType.BBOXES, ), Split( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] images='val2017', annotations='annotations_trainval2017', annotation_type=AnnotationType.BBOXES, ), Split( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] images='test2017', annotations='image_info_test2017', annotation_type=AnnotationType.NONE, @@ -159,13 +159,13 @@ class Coco(tfds.core.GeneratorBasedBuilder): has_panoptic=True, splits=[ Split( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] images='train2017', annotations='panoptic_annotations_trainval2017', annotation_type=AnnotationType.PANOPTIC, ), Split( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] images='val2017', annotations='panoptic_annotations_trainval2017', annotation_type=AnnotationType.PANOPTIC, @@ -182,8 +182,8 @@ def _info(self): 'image/id': tf.int64, } # Either uses panotptic or original annotations - if self.builder_config.has_panoptic: - features.update({ + if self.builder_config.has_panoptic: # pyrefly: ignore[missing-attribute] + features.update({ # pyrefly: ignore[no-matching-overload] 'panoptic_image': tfds.features.Image(encoding_format='png'), 'panoptic_image/filename': tfds.features.Text(), 'panoptic_objects': tfds.features.Sequence({ @@ -199,7 +199,7 @@ def _info(self): }), }) else: - features.update( + features.update( # pyrefly: ignore[no-matching-overload] { 'objects': tfds.features.Sequence({ 'id': np.int64, @@ -233,7 +233,7 @@ def _split_generators(self, dl_manager): # Merge urls from all splits together urls = {} - for split in self.builder_config.splits: + for split in self.builder_config.splits: # pyrefly: ignore[missing-attribute] urls['{}_images'.format(split.name)] = 'zips/{}.zip'.format(split.images) urls['{}_annotations'.format(split.name)] = 'annotations/{}.zip'.format( split.annotations @@ -247,10 +247,10 @@ def _split_generators(self, dl_manager): ) splits = [] - for split in self.builder_config.splits: + for split in self.builder_config.splits: # pyrefly: ignore[missing-attribute] image_dir = extracted_paths['{}_images'.format(split.name)] annotations_dir = extracted_paths['{}_annotations'.format(split.name)] - if self.builder_config.has_panoptic: + if self.builder_config.has_panoptic: # pyrefly: ignore[missing-attribute] panoptic_image_zip_path = os.path.join( annotations_dir, 'annotations', @@ -305,7 +305,7 @@ def _generate_examples( instance_path = os.path.join( annotation_dir, 'annotations', - instance_filename.format(split_name), + instance_filename.format(split_name), # pyrefly: ignore[unbound-name] ) coco_annotation = ANNOTATION_CLS[annotation_type](instance_path) # Each category is a dict: @@ -332,7 +332,7 @@ def _generate_examples( # and 'supercategory' (in addition to 'name') # Warning: As Coco only use 80 out of the 91 labels, the c['id'] and # dataset names ids won't match. - if self.builder_config.has_panoptic: + if self.builder_config.has_panoptic: # pyrefly: ignore[missing-attribute] objects_key = 'panoptic_objects' else: objects_key = 'objects' @@ -410,7 +410,7 @@ def build_bbox(x, y, width, height): ], } if self.builder_config.has_panoptic: - panoptic_filename = panoptic_annotation['file_name'] + panoptic_filename = panoptic_annotation['file_name'] # pyrefly: ignore[unbound-name] panoptic_image_path = os.path.join(panoptic_dir, panoptic_filename) example['panoptic_image'] = panoptic_image_path example['panoptic_image/filename'] = panoptic_filename diff --git a/tensorflow_datasets/object_detection/coco_captions.py b/tensorflow_datasets/object_detection/coco_captions.py index 399518e1b4a..190a5917ff4 100644 --- a/tensorflow_datasets/object_detection/coco_captions.py +++ b/tensorflow_datasets/object_detection/coco_captions.py @@ -85,8 +85,8 @@ def _split_generators(self, dl_manager): split.name: split for split in super(CocoCaptions, self)._split_generators(dl_manager) } - coco_train_split = coco_splits[tfds.Split.TRAIN] - coco_val_split = coco_splits[tfds.Split.VALIDATION] + coco_train_split = coco_splits[tfds.Split.TRAIN] # pyrefly: ignore[missing-attribute] + coco_val_split = coco_splits[tfds.Split.VALIDATION] # pyrefly: ignore[missing-attribute] urls = {} urls['karpathy_and_li_splits'] = ( @@ -140,7 +140,7 @@ def _split_generators(self, dl_manager): ), ] - def _generate_examples(self, image_filename_to_annotations, coco_gen_kwargs): + def _generate_examples(self, image_filename_to_annotations, coco_gen_kwargs): # pyrefly: ignore[bad-override] """Generate examples as dicts. Args: diff --git a/tensorflow_datasets/object_detection/open_images_challenge2019_beam.py b/tensorflow_datasets/object_detection/open_images_challenge2019_beam.py index 4f5b4388b77..ee5cd4e01f7 100644 --- a/tensorflow_datasets/object_detection/open_images_challenge2019_beam.py +++ b/tensorflow_datasets/object_detection/open_images_challenge2019_beam.py @@ -116,7 +116,7 @@ def process(self, element): if self._image2labels: for label, source, confidence in self._image2labels[image_id]: objects.append({ - "label": self._mid2int[label], + "label": self._mid2int[label], # pyrefly: ignore[unsupported-operation] "source": source, "confidence": confidence, }) @@ -127,7 +127,7 @@ def process(self, element): label, xmin, xmax, ymin, ymax, is_group_of = annotation bbox = tfds.features.BBox(xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax) bobjects.append({ - "label": self._mid2int[label], + "label": self._mid2int[label], # pyrefly: ignore[unsupported-operation] "bbox": bbox, "is_group_of": is_group_of, }) @@ -152,7 +152,7 @@ def load_image_level_labels(filepath): source = "verification" elif len(row) == 4: image_id, source, label, confidence = row - image2labels[image_id].append((label, source, float(confidence))) + image2labels[image_id].append((label, source, float(confidence))) # pyrefly: ignore[unbound-name] return image2labels @@ -169,9 +169,9 @@ def load_box_level_labels(filepath): image_id, label = row[0], row[2] xmin_s, xmax_s, ymin_s, ymax_s = row[4:8] is_group_of_s = row[10] - xmin, xmax, ymin, ymax = map(float, (xmin_s, xmax_s, ymin_s, ymax_s)) - is_group_of = bool(int(is_group_of_s)) - image2boxes[image_id].append((label, xmin, xmax, ymin, ymax, is_group_of)) + xmin, xmax, ymin, ymax = map(float, (xmin_s, xmax_s, ymin_s, ymax_s)) # pyrefly: ignore[unbound-name] + is_group_of = bool(int(is_group_of_s)) # pyrefly: ignore[unbound-name] + image2boxes[image_id].append((label, xmin, xmax, ymin, ymax, is_group_of)) # pyrefly: ignore[unbound-name] return image2boxes diff --git a/tensorflow_datasets/object_detection/voc.py b/tensorflow_datasets/object_detection/voc.py index bc3c00da6f3..1e9c2c29bc7 100644 --- a/tensorflow_datasets/object_detection/voc.py +++ b/tensorflow_datasets/object_detection/voc.py @@ -132,7 +132,7 @@ def __init__( self.filenames = filenames self.has_test_annotations = has_test_annotations super(VocConfig, self).__init__( - name=year, + name=year, # pyrefly: ignore[bad-argument-type] # Version history: # 5.0.0: Added new download links and updated checksums. # 4.0.0: Added BuildConfig and 2012 version support, deprecate Voc2007. @@ -192,26 +192,26 @@ def _info(self): tfds.features.ClassLabel(names=_VOC_LABELS) ), }), - homepage=_VOC_URL.format(year=self.builder_config.year), - citation=_VOC_CITATION.format(year=self.builder_config.year), + homepage=_VOC_URL.format(year=self.builder_config.year), # pyrefly: ignore[missing-attribute] + citation=_VOC_CITATION.format(year=self.builder_config.year), # pyrefly: ignore[missing-attribute] ) def _split_generators(self, dl_manager): paths = dl_manager.download_and_extract({ k: os.path.join(_VOC_DATA_URL, v) - for k, v in self.builder_config.filenames.items() + for k, v in self.builder_config.filenames.items() # pyrefly: ignore[missing-attribute] }) return [ tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] gen_kwargs=dict(data_path=paths["test"], set_name="test"), ), tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs=dict(data_path=paths["trainval"], set_name="train"), ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] gen_kwargs=dict(data_path=paths["trainval"], set_name="val"), ), ] @@ -222,12 +222,12 @@ def _generate_examples(self, data_path, set_name): data_path, os.path.normpath( "VOCdevkit/VOC{}/ImageSets/Main/{}.txt".format( - self.builder_config.year, set_name + self.builder_config.year, set_name # pyrefly: ignore[missing-attribute] ) ), ) load_annotations = ( - self.builder_config.has_test_annotations or set_name != "test" + self.builder_config.has_test_annotations or set_name != "test" # pyrefly: ignore[missing-attribute] ) with tf.io.gfile.GFile(set_filepath, "r") as f: for line in f: @@ -240,7 +240,7 @@ def _generate_example(self, data_path, image_id, load_annotations): data_path, os.path.normpath( "VOCdevkit/VOC{}/JPEGImages/{}.jpg".format( - self.builder_config.year, image_id + self.builder_config.year, image_id # pyrefly: ignore[missing-attribute] ) ), ) @@ -248,7 +248,7 @@ def _generate_example(self, data_path, image_id, load_annotations): data_path, os.path.normpath( "VOCdevkit/VOC{}/Annotations/{}.xml".format( - self.builder_config.year, image_id + self.builder_config.year, image_id # pyrefly: ignore[missing-attribute] ) ), ) diff --git a/tensorflow_datasets/object_detection/waymo_open_dataset.py b/tensorflow_datasets/object_detection/waymo_open_dataset.py index a159961eea7..2f82f69dda7 100644 --- a/tensorflow_datasets/object_detection/waymo_open_dataset.py +++ b/tensorflow_datasets/object_detection/waymo_open_dataset.py @@ -133,31 +133,31 @@ def _info(self) -> tfds.core.DatasetInfo: "image": tfds.features.Image( shape=(1280, 1920, 3), encoding_format="jpeg" ), - "labels": tfds.features.Sequence(annotations), + "labels": tfds.features.Sequence(annotations), # pyrefly: ignore[bad-argument-type] }, "camera_FRONT_LEFT": { "image": tfds.features.Image( shape=(1280, 1920, 3), encoding_format="jpeg" ), - "labels": tfds.features.Sequence(annotations), + "labels": tfds.features.Sequence(annotations), # pyrefly: ignore[bad-argument-type] }, "camera_SIDE_LEFT": { "image": tfds.features.Image( shape=(886, 1920, 3), encoding_format="jpeg" ), - "labels": tfds.features.Sequence(annotations), + "labels": tfds.features.Sequence(annotations), # pyrefly: ignore[bad-argument-type] }, "camera_FRONT_RIGHT": { "image": tfds.features.Image( shape=(1280, 1920, 3), encoding_format="jpeg" ), - "labels": tfds.features.Sequence(annotations), + "labels": tfds.features.Sequence(annotations), # pyrefly: ignore[bad-argument-type] }, "camera_SIDE_RIGHT": { "image": tfds.features.Image( shape=(886, 1920, 3), encoding_format="jpeg" ), - "labels": tfds.features.Sequence(annotations), + "labels": tfds.features.Sequence(annotations), # pyrefly: ignore[bad-argument-type] }, }), homepage=_HOMEPAGE_URL, @@ -177,7 +177,7 @@ def _split_generators(self, dl_manager): # Training set train_files = tf.io.gfile.glob( os.path.join( - self.builder_config.cloud_bucket, "training/segment*camera*" + self.builder_config.cloud_bucket, "training/segment*camera*" # pyrefly: ignore[missing-attribute] ) ) logging.info("Train files: %s", train_files) @@ -185,20 +185,20 @@ def _split_generators(self, dl_manager): # Validation set validation_files = tf.io.gfile.glob( os.path.join( - self.builder_config.cloud_bucket, "validation/segment*camera*" + self.builder_config.cloud_bucket, "validation/segment*camera*" # pyrefly: ignore[missing-attribute] ) ) logging.info("Validation files: %s", validation_files) split_generators = [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={ "tf_record_files": train_files, }, ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] gen_kwargs={ "tf_record_files": validation_files, }, @@ -206,17 +206,17 @@ def _split_generators(self, dl_manager): ] # Testing set (Only available in Waymo Open Dataset v1.2) - if self.builder_config.name == "v_1_2": + if self.builder_config.name == "v_1_2": # pyrefly: ignore[missing-attribute] test_files = tf.io.gfile.glob( os.path.join( - self.builder_config.cloud_bucket, "testing/segment*camera*" + self.builder_config.cloud_bucket, "testing/segment*camera*" # pyrefly: ignore[missing-attribute] ) ) logging.info("Testing files: %s", test_files) split_generators.append( tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] gen_kwargs={ "tf_record_files": test_files, }, @@ -265,7 +265,7 @@ def _generate_images_and_annotations(tf_record_file): dataset = tf.data.TFRecordDataset(tf_record_file, compression_type="") for data in dataset: frame = open_dataset.Frame() - frame.ParseFromString(data.numpy()) + frame.ParseFromString(data.numpy()) # pyrefly: ignore[bad-argument-type] image_and_annotation = { "context": {"name": frame.context.name}, @@ -289,7 +289,7 @@ def _generate_images_and_annotations(tf_record_file): ) camera_name = open_dataset.CameraName.Name.Name(frame_image.name) - image_and_annotation["camera_" + camera_name] = { + image_and_annotation["camera_" + camera_name] = { # pyrefly: ignore[unsupported-operation] "image": frame_image.image, "labels": labels, } diff --git a/tensorflow_datasets/object_detection/wider_face.py b/tensorflow_datasets/object_detection/wider_face.py index e9c4456f8b3..83ff783f2b6 100644 --- a/tensorflow_datasets/object_detection/wider_face.py +++ b/tensorflow_datasets/object_detection/wider_face.py @@ -104,15 +104,15 @@ def _split_generators(self, dl_manager): }) return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={'split': 'train', 'extracted_dirs': extracted_dirs}, ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] gen_kwargs={'split': 'val', 'extracted_dirs': extracted_dirs}, ), tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] gen_kwargs={'split': 'test', 'extracted_dirs': extracted_dirs}, ), ] diff --git a/tensorflow_datasets/scripts/cli/build.py b/tensorflow_datasets/scripts/cli/build.py index 235d42ebbdb..00daab9204f 100644 --- a/tensorflow_datasets/scripts/cli/build.py +++ b/tensorflow_datasets/scripts/cli/build.py @@ -366,7 +366,7 @@ def _make_download_config( # Load the download config manual_dir = args.paths.manual_dir if args.paths.add_name_to_manual_dir: - manual_dir = manual_dir / dataset_name + manual_dir = manual_dir / dataset_name # pyrefly: ignore[unsupported-operation] kwargs = {} if args.generation.max_shard_size_mb: diff --git a/tensorflow_datasets/scripts/cli/cli_utils.py b/tensorflow_datasets/scripts/cli/cli_utils.py index e4ce83de6df..99df70154fc 100644 --- a/tensorflow_datasets/scripts/cli/cli_utils.py +++ b/tensorflow_datasets/scripts/cli/cli_utils.py @@ -92,7 +92,7 @@ def _parse_flags(argv: list[str]) -> _DataclassT: description=description, allow_abbrev=False, ) - parser.add_arguments(args_dataclass, dest='args') + parser.add_arguments(args_dataclass, dest='args') # pyrefly: ignore[no-matching-overload] return parser.parse_args(argv[1:]).args return _parse_flags diff --git a/tensorflow_datasets/scripts/cli/convert_format_utils.py b/tensorflow_datasets/scripts/cli/convert_format_utils.py index d1bc04039d0..14f13ffc92f 100644 --- a/tensorflow_datasets/scripts/cli/convert_format_utils.py +++ b/tensorflow_datasets/scripts/cli/convert_format_utils.py @@ -236,13 +236,13 @@ def get_all_shard_instructions( def _get_root_data_dir( in_dir: epath.Path, info: dataset_info_pb2.DatasetInfo ) -> epath.Path: - in_dir = os.fspath(in_dir) + in_dir = os.fspath(in_dir) # pyrefly: ignore[bad-assignment] if info.config_name: parts = [info.name, info.config_name, info.version] else: parts = [info.name, info.version] relative_data_dir = os.path.join(*parts) - return epath.Path(re.sub(rf'{relative_data_dir}/?$', '', in_dir)) + return epath.Path(re.sub(rf'{relative_data_dir}/?$', '', in_dir)) # pyrefly: ignore[no-matching-overload] class ConvertMetadataFn(beam.DoFn): @@ -838,7 +838,7 @@ def convert_dataset( if isinstance(dataset_version_dir, str): dataset_version_dir = [dataset_version_dir] - if len(dataset_version_dir) > 1 and out_dir is not None: + if len(dataset_version_dir) > 1 and out_dir is not None: # pyrefly: ignore[bad-argument-type] raise ValueError( 'If multiple dataset version dirs are specified, `out_dir` must be' ' `None`, since each dataset will be converted in the same folder as' @@ -846,7 +846,7 @@ def convert_dataset( ) from_to_dirs = {} - for path in dataset_version_dir: + for path in dataset_version_dir: # pyrefly: ignore[not-iterable] if out_dir is None: from_to_dirs[epath.Path(path)] = epath.Path(path) else: diff --git a/tensorflow_datasets/scripts/cli/croissant.py b/tensorflow_datasets/scripts/cli/croissant.py index b53ad4e071a..a439cd8b877 100644 --- a/tensorflow_datasets/scripts/cli/croissant.py +++ b/tensorflow_datasets/scripts/cli/croissant.py @@ -123,7 +123,7 @@ def version(self) -> version_lib.Version: def execute(self) -> None: """Creates Croissant Builders and prepares them.""" for record_set_id in self.record_set_ids: - prepare_croissant_builder(args=self, record_set_id=record_set_id) + prepare_croissant_builder(args=self, record_set_id=record_set_id) # pyrefly: ignore[bad-argument-type] def prepare_croissant_builder( diff --git a/tensorflow_datasets/scripts/cli/new.py b/tensorflow_datasets/scripts/cli/new.py index 70fcd35f241..172d26e5932 100644 --- a/tensorflow_datasets/scripts/cli/new.py +++ b/tensorflow_datasets/scripts/cli/new.py @@ -84,7 +84,7 @@ def create_dataset_files( name=dataset_name, in_tfds=in_tfds, path=dataset_dir, - data_format=data_format, + data_format=data_format, # pyrefly: ignore[bad-argument-type] ) _create_dataset_file(info) diff --git a/tensorflow_datasets/scripts/documentation/build_catalog.py b/tensorflow_datasets/scripts/documentation/build_catalog.py index 3490739dbb7..b8118a494c4 100644 --- a/tensorflow_datasets/scripts/documentation/build_catalog.py +++ b/tensorflow_datasets/scripts/documentation/build_catalog.py @@ -212,7 +212,7 @@ def _save_table_of_content( 'title': doc.name, } sec_dict['section'].append(sidebar_item) - toc_yaml['toc'].append(sec_dict) + toc_yaml['toc'].append(sec_dict) # pyrefly: ignore[bad-argument-type] # Add `overview.md` section toc_overview.append(_create_collections_section_toc(collection_docs)) @@ -231,7 +231,7 @@ def _save_table_of_content( if doc.is_nightly: sidebar_item['status'] = 'nightly' sec_dict['section'].append(sidebar_item) - toc_yaml['toc'].append(sec_dict) + toc_yaml['toc'].append(sec_dict) # pyrefly: ignore[bad-argument-type] # Add `overview.md` section toc_overview.append(_create_datasets_section_toc(section, builder_docs)) diff --git a/tensorflow_datasets/scripts/documentation/build_community_catalog.py b/tensorflow_datasets/scripts/documentation/build_community_catalog.py index 43a22b842c8..ae3c6dffc9c 100644 --- a/tensorflow_datasets/scripts/documentation/build_community_catalog.py +++ b/tensorflow_datasets/scripts/documentation/build_community_catalog.py @@ -154,7 +154,7 @@ def name(self) -> str: @property def namespace(self) -> str: - return self.dataset.name.namespace + return self.dataset.name.namespace # pyrefly: ignore[bad-return] def code_url(self, title: str = 'Code') -> str: return f'[{title}]({self.dataset.source.root_path})' @@ -383,7 +383,7 @@ def dataset_info_per_config( if not isinstance(self.dataset_infos, dict): return {} return { - config_name: self._parse_dataset_info_proto(config_name, config) + config_name: self._parse_dataset_info_proto(config_name, config) # pyrefly: ignore[bad-argument-type] for config_name, config in self.dataset_infos.items() } diff --git a/tensorflow_datasets/scripts/documentation/collection_markdown_builder.py b/tensorflow_datasets/scripts/documentation/collection_markdown_builder.py index 684528ee8c0..fd2f0d1d04b 100644 --- a/tensorflow_datasets/scripts/documentation/collection_markdown_builder.py +++ b/tensorflow_datasets/scripts/documentation/collection_markdown_builder.py @@ -218,7 +218,7 @@ def get_collection_markdown_string( doc_str = [ _display_collection_heading(collection), _display_collection_sections( - loader=collection, all_sections=all_sections + loader=collection, all_sections=all_sections # pyrefly: ignore[bad-argument-type] ), ] return '\n\n'.join([tfds.core.utils.dedent(s) for s in doc_str if s]) diff --git a/tensorflow_datasets/scripts/documentation/dataset_markdown_builder.py b/tensorflow_datasets/scripts/documentation/dataset_markdown_builder.py index dcfa2071100..5dd31745c41 100644 --- a/tensorflow_datasets/scripts/documentation/dataset_markdown_builder.py +++ b/tensorflow_datasets/scripts/documentation/dataset_markdown_builder.py @@ -769,10 +769,10 @@ def get_markdown_string( SupervisedKeySection(), ] if visu_doc_util: - all_sections.append(DatasetVisualizationSection(visu_doc_util)) + all_sections.append(DatasetVisualizationSection(visu_doc_util)) # pyrefly: ignore[bad-argument-type] if df_doc_util: - all_sections.append(DatasetDataframeSection(df_doc_util)) - all_sections.append(DatasetCitationSection()) + all_sections.append(DatasetDataframeSection(df_doc_util)) # pyrefly: ignore[bad-argument-type] + all_sections.append(DatasetCitationSection()) # pyrefly: ignore[bad-argument-type] doc_str = [ _display_schema_org(builder, visu_doc_util), @@ -784,7 +784,7 @@ def get_markdown_string( namespace=namespace, nightly_doc_util=nightly_doc_util, config_builders=config_builders, - all_sections=all_sections, + all_sections=all_sections, # pyrefly: ignore[bad-argument-type] ), ] return '\n\n'.join([tfds.core.utils.dedent(s) for s in doc_str if s]) diff --git a/tensorflow_datasets/scripts/documentation/doc_utils.py b/tensorflow_datasets/scripts/documentation/doc_utils.py index 7b6127ecf75..4ebfec072ae 100644 --- a/tensorflow_datasets/scripts/documentation/doc_utils.py +++ b/tensorflow_datasets/scripts/documentation/doc_utils.py @@ -188,7 +188,7 @@ def _full_names_to_dict(full_names: Sequence[str]) -> FullNamesDict: for full_name in full_names: ds_name, config, version = _split_full_name(full_name) full_names_dict[ds_name][config][version] # pylint: disable=pointless-statement - return full_names_dict + return full_names_dict # pyrefly: ignore[bad-return] def _build_nightly_dict( @@ -216,11 +216,11 @@ def _build_nightly_dict( nightly_ds[dataset][config][version] = True else: # New config only present in tfds-nightly - nightly_ds[dataset][config] = True + nightly_ds[dataset][config] = True # pyrefly: ignore[unsupported-operation] else: # New dataset only present in tfds-nightly - nightly_ds[dataset] = True - return nightly_ds + nightly_ds[dataset] = True # pyrefly: ignore[unsupported-operation] + return nightly_ds # pyrefly: ignore[bad-return] @tfds.core.utils.memoize() @@ -265,7 +265,7 @@ def is_config_nightly(self, builder: tfds.core.DatasetBuilder) -> bool: ds_name, config, _ = _split_full_name(builder.info.full_name) if self.is_builder_nightly(builder): return False - return self._nightly_dict[ds_name][config] is True # pylint: disable=g-bool-id-comparison + return self._nightly_dict[ds_name][config] is True # pylint: disable=g-bool-id-comparison # pyrefly: ignore[bad-index] def is_version_nightly( self, @@ -276,7 +276,7 @@ def is_version_nightly( ds_name, config, _ = _split_full_name(builder.info.full_name) if self.is_builder_nightly(builder) or self.is_config_nightly(builder): return False - return self._nightly_dict[ds_name][config][version] is True # pylint: disable=g-bool-id-comparison + return self._nightly_dict[ds_name][config][version] is True # pylint: disable=g-bool-id-comparison # pyrefly: ignore[bad-index] def has_nightly(self, builder: tfds.core.DatasetBuilder) -> bool: """Returns True if any of the builder/config/version is new.""" diff --git a/tensorflow_datasets/structured/forest_fires.py b/tensorflow_datasets/structured/forest_fires.py index 09068a5f236..83b17960619 100644 --- a/tensorflow_datasets/structured/forest_fires.py +++ b/tensorflow_datasets/structured/forest_fires.py @@ -154,7 +154,7 @@ def _split_generators(self, dl_manager): # There is no predefined train/val/test split for this dataset. return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={'file_path': data}, ), ] diff --git a/tensorflow_datasets/structured/german_credit_numeric.py b/tensorflow_datasets/structured/german_credit_numeric.py index 5d5601df564..ff51e914d2c 100644 --- a/tensorflow_datasets/structured/german_credit_numeric.py +++ b/tensorflow_datasets/structured/german_credit_numeric.py @@ -66,7 +66,7 @@ def _split_generators(self, dl_manager): return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, gen_kwargs={"records": records} + name=tfds.Split.TRAIN, gen_kwargs={"records": records} # pyrefly: ignore[missing-attribute] ), ] diff --git a/tensorflow_datasets/structured/higgs.py b/tensorflow_datasets/structured/higgs.py index ca991bdd8b8..41880f31b26 100644 --- a/tensorflow_datasets/structured/higgs.py +++ b/tensorflow_datasets/structured/higgs.py @@ -113,7 +113,7 @@ def _split_generators(self, dl_manager): # There is no predefined train/val/test split for this dataset. return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={ 'file_path': path, }, diff --git a/tensorflow_datasets/structured/iris.py b/tensorflow_datasets/structured/iris.py index 8c27a6a2e19..62a9566bcc7 100644 --- a/tensorflow_datasets/structured/iris.py +++ b/tensorflow_datasets/structured/iris.py @@ -77,7 +77,7 @@ def _split_generators(self, dl_manager): # Specify the splits return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, gen_kwargs={"records": records} + name=tfds.Split.TRAIN, gen_kwargs={"records": records} # pyrefly: ignore[missing-attribute] ), ] diff --git a/tensorflow_datasets/structured/movielens.py b/tensorflow_datasets/structured/movielens.py index d612f5b648c..eda057042f0 100644 --- a/tensorflow_datasets/structured/movielens.py +++ b/tensorflow_datasets/structured/movielens.py @@ -440,11 +440,11 @@ def _info(self) -> tfds.core.DatasetInfo: } features_dict = {} - if self.builder_config.table_option == 'movies': + if self.builder_config.table_option == 'movies': # pyrefly: ignore[missing-attribute] features_dict.update(movie_features_dict) # For the other cases, self.builder_config.table_option == 'ratings'. # Older versions of MovieLens (1m, 100k) have demographic features. - elif self.builder_config.format_version == '1m': + elif self.builder_config.format_version == '1m': # pyrefly: ignore[missing-attribute] features_dict.update(movie_features_dict) features_dict.update(rating_features_dict) features_dict.update(demographic_features_dict) @@ -472,15 +472,15 @@ def _split_generators( ) -> List[tfds.core.SplitGenerator]: """Returns SplitGenerators.""" extracted_path = dl_manager.download_and_extract( - self.builder_config.download_url, + self.builder_config.download_url, # pyrefly: ignore[missing-attribute] ) dir_path = os.path.join( extracted_path, - 'ml-%s' % self.builder_config.format_version, + 'ml-%s' % self.builder_config.format_version, # pyrefly: ignore[missing-attribute] ) return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={'dir_path': dir_path}, ), ] @@ -489,7 +489,7 @@ def _generate_examples( self, dir_path: Optional[str] = None ) -> Iterator[Tuple[int, Dict[str, Any]]]: """Yields examples by calling the corresponding parsing function.""" - for ex in self.builder_config.parsing_fn(dir_path): + for ex in self.builder_config.parsing_fn(dir_path): # pyrefly: ignore[missing-attribute] yield ex diff --git a/tensorflow_datasets/structured/wiki_bio.py b/tensorflow_datasets/structured/wiki_bio.py index f7cea86007c..86d8cf109ee 100644 --- a/tensorflow_datasets/structured/wiki_bio.py +++ b/tensorflow_datasets/structured/wiki_bio.py @@ -113,7 +113,7 @@ def _split_generators(self, dl_manager): return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={ 'id_file': os.path.join(extracted_path, 'train', 'train.id'), 'infobox_file': os.path.join( @@ -131,7 +131,7 @@ def _split_generators(self, dl_manager): }, ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] gen_kwargs={ 'id_file': os.path.join(extracted_path, 'valid', 'valid.id'), 'infobox_file': os.path.join( @@ -149,7 +149,7 @@ def _split_generators(self, dl_manager): }, ), tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] gen_kwargs={ 'id_file': os.path.join(extracted_path, 'test', 'test.id'), 'infobox_file': os.path.join( diff --git a/tensorflow_datasets/summarization/cnn_dailymail.py b/tensorflow_datasets/summarization/cnn_dailymail.py index c4fa915e3e9..4a06c40def1 100644 --- a/tensorflow_datasets/summarization/cnn_dailymail.py +++ b/tensorflow_datasets/summarization/cnn_dailymail.py @@ -114,11 +114,11 @@ def _subset_filenames(dl_paths, split): """Get filenames for a particular split.""" assert isinstance(dl_paths, dict), dl_paths # Get filenames for a split. - if split == tfds.Split.TRAIN: + if split == tfds.Split.TRAIN: # pyrefly: ignore[missing-attribute] urls = _get_url_hashes(dl_paths['train_urls']) - elif split == tfds.Split.VALIDATION: + elif split == tfds.Split.VALIDATION: # pyrefly: ignore[missing-attribute] urls = _get_url_hashes(dl_paths['val_urls']) - elif split == tfds.Split.TEST: + elif split == tfds.Split.TEST: # pyrefly: ignore[missing-attribute] urls = _get_url_hashes(dl_paths['test_urls']) else: logging.fatal('Unsupported split: %s', split) @@ -239,18 +239,18 @@ def _split_generators(self, dl_manager): return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, - gen_kwargs={'files': _subset_filenames(dl_paths, tfds.Split.TRAIN)}, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] + gen_kwargs={'files': _subset_filenames(dl_paths, tfds.Split.TRAIN)}, # pyrefly: ignore[missing-attribute] ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] gen_kwargs={ - 'files': _subset_filenames(dl_paths, tfds.Split.VALIDATION) + 'files': _subset_filenames(dl_paths, tfds.Split.VALIDATION) # pyrefly: ignore[missing-attribute] }, ), tfds.core.SplitGenerator( - name=tfds.Split.TEST, - gen_kwargs={'files': _subset_filenames(dl_paths, tfds.Split.TEST)}, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] + gen_kwargs={'files': _subset_filenames(dl_paths, tfds.Split.TEST)}, # pyrefly: ignore[missing-attribute] ), ] diff --git a/tensorflow_datasets/summarization/covid19sum.py b/tensorflow_datasets/summarization/covid19sum.py index 3b1450ddcd0..c8943bd15f9 100644 --- a/tensorflow_datasets/summarization/covid19sum.py +++ b/tensorflow_datasets/summarization/covid19sum.py @@ -102,7 +102,7 @@ class Covid19sum(tfds.core.GeneratorBasedBuilder): def _info(self) -> tfds.core.DatasetInfo: features = {k: tf.string for k in _ADDITIONAL_FEATURES + [_ABSTRACT]} - features[_BODY_TEXT] = tfds.features.Sequence( + features[_BODY_TEXT] = tfds.features.Sequence( # pyrefly: ignore[unsupported-operation] tfds.features.FeaturesDict({_SECTION: np.str_, _TEXT: np.str_}) ) return tfds.core.DatasetInfo( @@ -138,7 +138,7 @@ def _split_generators( data_paths.append(d) return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={"data_paths": data_paths}, ) ] @@ -147,7 +147,7 @@ def _generate_examples( self, data_paths: Optional[List[Dict[Text, Any]]] = None ) -> Iterator[Tuple[Any, Dict[Text, Any]]]: """Yields examples.""" - for d in data_paths: + for d in data_paths: # pyrefly: ignore[not-iterable] path = d.pop("path") if tf.io.gfile.exists(path): with tf.io.gfile.GFile(path, "rb") as f: diff --git a/tensorflow_datasets/summarization/gigaword.py b/tensorflow_datasets/summarization/gigaword.py index 50d07559e72..7e79d6bd764 100644 --- a/tensorflow_datasets/summarization/gigaword.py +++ b/tensorflow_datasets/summarization/gigaword.py @@ -86,7 +86,7 @@ def _split_generators(self, dl_manager): pattern = os.path.join(dl_path, "org_data", "%s.%s.txt") return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={ "src_path": pattern % ("train", "src"), "tgt_path": pattern % ("train", "tgt"), @@ -94,7 +94,7 @@ def _split_generators(self, dl_manager): }, ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] gen_kwargs={ "src_path": pattern % ("dev", "src"), "tgt_path": pattern % ("dev", "tgt"), @@ -102,7 +102,7 @@ def _split_generators(self, dl_manager): }, ), tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] gen_kwargs={ "src_path": pattern % ("test", "src"), "tgt_path": pattern % ("test", "tgt"), diff --git a/tensorflow_datasets/summarization/wikihow.py b/tensorflow_datasets/summarization/wikihow.py index 735a3c9543e..9a3a0533b81 100644 --- a/tensorflow_datasets/summarization/wikihow.py +++ b/tensorflow_datasets/summarization/wikihow.py @@ -108,7 +108,7 @@ class Wikihow(tfds.core.GeneratorBasedBuilder): def _info(self): feature_names = [_DOCUMENT, _SUMMARY, "title"] - if self.builder_config.name == "sep": + if self.builder_config.name == "sep": # pyrefly: ignore[missing-attribute] feature_names.extend(["overview", "sectionLabel"]) return tfds.core.DatasetInfo( builder=self, @@ -132,28 +132,28 @@ def _split_generators(self, dl_manager): return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={ "path": os.path.join( - dl_manager.manual_dir, self.builder_config.filename + dl_manager.manual_dir, self.builder_config.filename # pyrefly: ignore[missing-attribute] ), "title_set": titles["train"], }, ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] gen_kwargs={ "path": os.path.join( - dl_manager.manual_dir, self.builder_config.filename + dl_manager.manual_dir, self.builder_config.filename # pyrefly: ignore[missing-attribute] ), "title_set": titles["validation"], }, ), tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] gen_kwargs={ "path": os.path.join( - dl_manager.manual_dir, self.builder_config.filename + dl_manager.manual_dir, self.builder_config.filename # pyrefly: ignore[missing-attribute] ), "title_set": titles["test"], }, @@ -162,16 +162,16 @@ def _split_generators(self, dl_manager): def _generate_examples(self, path=None, title_set=None): """Yields examples.""" - with epath.Path(path).open() as f: + with epath.Path(path).open() as f: # pyrefly: ignore[bad-argument-type] reader = csv.reader(f) headers = next(reader) - if self.builder_config.name == "all" and headers != [ + if self.builder_config.name == "all" and headers != [ # pyrefly: ignore[missing-attribute] "headline", "title", "text", ]: raise ValueError("Mismatched header in WikiAll.txt") - if self.builder_config.name == "sep" and headers != [ + if self.builder_config.name == "sep" and headers != [ # pyrefly: ignore[missing-attribute] "overview", "headline", "text", @@ -187,7 +187,7 @@ def _generate_examples(self, path=None, title_set=None): document = line[key2id[_DOCUMENT]].strip() summary, document = _filter_and_clean(summary, document) if summary and document: - if line[key2id["title"]].strip().replace(" ", "") in title_set: + if line[key2id["title"]].strip().replace(" ", "") in title_set: # pyrefly: ignore[not-iterable] d = { k: line[v].strip() for k, v in key2id.items() diff --git a/tensorflow_datasets/summarization/xsum.py b/tensorflow_datasets/summarization/xsum.py index 460729e5454..b48826bbcda 100644 --- a/tensorflow_datasets/summarization/xsum.py +++ b/tensorflow_datasets/summarization/xsum.py @@ -112,21 +112,21 @@ def _split_generators(self, dl_manager): ) return [ tfds.core.SplitGenerator( - name=tfds.Split.TRAIN, + name=tfds.Split.TRAIN, # pyrefly: ignore[missing-attribute] gen_kwargs={ "split_ids": split_ids["train"], "path": extract_path, }, ), tfds.core.SplitGenerator( - name=tfds.Split.VALIDATION, + name=tfds.Split.VALIDATION, # pyrefly: ignore[missing-attribute] gen_kwargs={ "split_ids": split_ids["validation"], "path": extract_path, }, ), tfds.core.SplitGenerator( - name=tfds.Split.TEST, + name=tfds.Split.TEST, # pyrefly: ignore[missing-attribute] gen_kwargs={ "split_ids": split_ids["test"], "path": extract_path, @@ -137,9 +137,9 @@ def _split_generators(self, dl_manager): def _generate_examples(self, split_ids=None, path=None): """Yields examples.""" missing = 0 - total_num = len(split_ids) - for i in split_ids: - filename = os.path.join(path, i + ".data") + total_num = len(split_ids) # pyrefly: ignore[bad-argument-type] + for i in split_ids: # pyrefly: ignore[not-iterable] + filename = os.path.join(path, i + ".data") # pyrefly: ignore[no-matching-overload] if tf.io.gfile.exists(filename): with epath.Path(filename).open() as f: text = "".join( diff --git a/tensorflow_datasets/testing/dataset_builder_testing.py b/tensorflow_datasets/testing/dataset_builder_testing.py index ce27cfd5ec2..d2c91e32b07 100644 --- a/tensorflow_datasets/testing/dataset_builder_testing.py +++ b/tensorflow_datasets/testing/dataset_builder_testing.py @@ -211,7 +211,7 @@ def dummy_data(cls) -> epath.Path: # pylint: disable=no-self-argument if cls is DatasetBuilderTestCase: # Required for build_api_docs return None # pytype: disable=bad-return-type - dummy_data_expected = cls.DATASET_CLASS.code_path.parent / "dummy_data" + dummy_data_expected = cls.DATASET_CLASS.code_path.parent / "dummy_data" # pyrefly: ignore[missing-attribute] fake_example_dir = epath.Path(test_utils.fake_examples_dir()) if cls.EXAMPLE_DIR is not None: dummy_data_found = epath.Path(cls.EXAMPLE_DIR) @@ -219,7 +219,7 @@ def dummy_data(cls) -> epath.Path: # pylint: disable=no-self-argument elif dummy_data_expected.exists(): dummy_data_found = dummy_data_expected else: - dummy_data_found = fake_example_dir / cls.DATASET_CLASS.name + dummy_data_found = fake_example_dir / cls.DATASET_CLASS.name # pyrefly: ignore[missing-attribute] if not dummy_data_found.exists(): err_msg = f"Dummy data not found in: {dummy_data_expected}" @@ -331,7 +331,7 @@ def _get_dl_download_result(self, url): def _get_dl_download_kaggle_result(self, competition_or_dataset): del competition_or_dataset # Unused - return self._prepare_download_results(self.DL_DOWNLOAD_RESULT) + return self._prepare_download_results(self.DL_DOWNLOAD_RESULT) # pyrefly: ignore[bad-argument-type] def _download_checksums(self, url): del url # Unused diff --git a/tensorflow_datasets/testing/feature_test_case.py b/tensorflow_datasets/testing/feature_test_case.py index c57dc97807f..98944c447c3 100644 --- a/tensorflow_datasets/testing/feature_test_case.py +++ b/tensorflow_datasets/testing/feature_test_case.py @@ -485,7 +485,7 @@ def _test_repr( text = f.repr_html_batch(ex) elif spec.sequence_rank > 1: text = f.repr_html_ragged(ex) - self.assertIsInstance(text, str) + self.assertIsInstance(text, str) # pyrefly: ignore[unbound-name] def features_encode_decode( diff --git a/tensorflow_datasets/testing/mocking.py b/tensorflow_datasets/testing/mocking.py index ef4c37e248f..712b2bf07d3 100644 --- a/tensorflow_datasets/testing/mocking.py +++ b/tensorflow_datasets/testing/mocking.py @@ -131,7 +131,7 @@ def _getitems( for record_key in record_keys ] if serialized: - return np.array(items) + return np.array(items) # pyrefly: ignore[bad-return] return items @@ -354,7 +354,7 @@ def mock_as_dataset(self, split, decoders=None, read_config=None, **kwargs): if read_config and read_config.add_tfds_id: ds_id = reader_lib._make_id_dataset( # pylint: disable=protected-access - filename=f'{self.name}-{split}.tfrecord-00000-of-00001', + filename=f'{self.name}-{split}.tfrecord-00000-of-00001', # pyrefly: ignore[bad-argument-type] start_index=0, # pytype: disable=wrong-arg-types ) ds = tf.data.Dataset.zip((ds, ds_id)) diff --git a/tensorflow_datasets/testing/test_case.py b/tensorflow_datasets/testing/test_case.py index 4523da0c0fd..5ec1372addb 100644 --- a/tensorflow_datasets/testing/test_case.py +++ b/tensorflow_datasets/testing/test_case.py @@ -60,7 +60,7 @@ def setUp(self): self.tmp_dir = tempfile.mkdtemp(dir=tf.compat.v1.test.get_temp_dir()) @contextlib.contextmanager - def assertLogs(self, text, level="info"): + def assertLogs(self, text, level="info"): # pyrefly: ignore[bad-override] with mock.patch.object(logging, level) as mock_log: yield concat_logs = "" diff --git a/tensorflow_datasets/testing/test_case_in_context.py b/tensorflow_datasets/testing/test_case_in_context.py index f58727c451a..7d47aa640bb 100644 --- a/tensorflow_datasets/testing/test_case_in_context.py +++ b/tensorflow_datasets/testing/test_case_in_context.py @@ -49,7 +49,7 @@ def tearDownClass(cls): cls._stack.close() super().tearDownClass() - def assertRaisesWithPredicateMatch(self, err_type, predicate): # pylint: disable=invalid-name + def assertRaisesWithPredicateMatch(self, err_type, predicate): # pylint: disable=invalid-name # pyrefly: ignore[bad-override] if isinstance(predicate, str): predicate_fn = lambda err: predicate in str(err) else: diff --git a/tensorflow_datasets/testing/test_utils.py b/tensorflow_datasets/testing/test_utils.py index ccbf4e5150c..da5af34f270 100644 --- a/tensorflow_datasets/testing/test_utils.py +++ b/tensorflow_datasets/testing/test_utils.py @@ -479,7 +479,7 @@ class DummyDatasetSharedGenerator( '1.0.0': 'Release notes 1.0.0', '2.0.0': 'Release notes 2.0.0', } - SUPPORTED_VERSIONS = [ + SUPPORTED_VERSIONS = [ # pyrefly: ignore[bad-assignment] '2.0.0', '0.0.9', '0.0.8', @@ -711,7 +711,7 @@ def set_current_datetime(now_datetime: datetime.datetime) -> Iterator[None]: class MockDatetime(datetime.datetime): @classmethod - def now(cls, tz=None) -> datetime.datetime: + def now(cls, tz=None) -> datetime.datetime: # pyrefly: ignore[bad-override] return now_datetime with mock.patch.object(datetime, 'datetime', new=MockDatetime):