diff --git a/fms_dgt/dataloaders/default.py b/fms_dgt/dataloaders/default.py index eae5744a..17147144 100644 --- a/fms_dgt/dataloaders/default.py +++ b/fms_dgt/dataloaders/default.py @@ -10,7 +10,13 @@ @register_dataloader("default") class DefaultDataloader(BaseDataloader): - """Base Class for all dataloaders""" + """ + The default dataloader takes a datastore and iterates over it. + + @param state_datastore : A data store. + @param loop_over_data : If true, it wraps around when it reaches the end of the iterator. + + """ def __init__( self, diff --git a/fms_dgt/datastores/default.py b/fms_dgt/datastores/default.py index 512bcfee..65dbcd10 100644 --- a/fms_dgt/datastores/default.py +++ b/fms_dgt/datastores/default.py @@ -20,7 +20,22 @@ @register_datastore("default") class DefaultDatastore(BaseDatastore): - """Base Class for all data stores""" + """ + The default datastore is a data store that takes a name of a `.jsonl`, + `.json`, `.yaml`, or `parquet` file, loads it, and returns an iterator + over it. + + If none of the filename extension matches, it attempts to give the first + argument to huggingface's `datasets.load_dataset` function to load the data, and returns an iterator over it. + + It also has an ability to dump the data into a file. + + @param output_dir : The destination directory for writing the data. + @param data_format : The data format for writing the data. It is not used for loading; The format is guessed from data_path. + @param data : A list. Data loaded from the files are concatenated with this list. + @param data_path : The path to the file to load. If it is a list, it is interpreted as the arguments to huggingface's `datasets.load_dataset` . + + """ def __init__( self, diff --git a/templates/generator/template.py b/templates/generator/template.py index f83e3948..e0c712ce 100644 --- a/templates/generator/template.py +++ b/templates/generator/template.py @@ -12,7 +12,7 @@ @register_block("template_generator") class TemplateGenerator(BaseGeneratorBlock): - """Base Class for all Generators""" + """TODO: Copy and edit this template to implement your own generator class""" def __init__(self, name: str, config: Dict, **kwargs: Any) -> None: super().__init__(name, config, **kwargs) diff --git a/templates/validator/template.py b/templates/validator/template.py index 92da5e6a..485c403d 100644 --- a/templates/validator/template.py +++ b/templates/validator/template.py @@ -12,7 +12,7 @@ @register_block("template_validator") class TemplateValidator(BaseValidatorBlock): - """Base Class for all Validators""" + """TODO: Copy and edit this template to implement your own validator class""" def __init__(self, name: str, config: Dict) -> None: super().__init__(name, config)