diff --git a/src/cache/cache.py b/src/cache/cache.py index 381de48..ce2ef1d 100644 --- a/src/cache/cache.py +++ b/src/cache/cache.py @@ -12,22 +12,48 @@ def get_digested(candidate_path: str) -> str: + """returns the encrypted the string + + returns the candidate's path encrypted using the FIPS secure hash algorithms sha256 + :param candidate_path: + :return: + """ return hashlib.sha256(candidate_path.encode('utf-8')).hexdigest() def load_from_cache(path: str, prefix: str = ''): + """returns the object loaded from path + + :param path: + :param prefix: + :return: + """ if path is not None: # TODO: what if the file is not there? with open(prefix + get_digested(path) + '.pickle', 'rb') as f: return pickle.load(f) def dump_to_cache(path: str, obj, prefix: str = ''): + """uploads the object obj on path + + :param path: + :param obj: + :param prefix: + """ if path is not None: with open(prefix + get_digested(path) + '.pickle', "wb") as f: pickle.dump(obj, f) def put_loaded_logs(split: Split, train_df, test_df, additional_columns): + """uploads the loaded logs + + uploads the training and test DataFrames loaded + :param split: + :param train_df: + :param test_df: + :param additional_columns: + """ [dump_to_cache(path, data, 'cache/loaded_log_cache/') for (path, data) in [ (split.train_log.name, train_df), (split.test_log.name, test_df), @@ -40,6 +66,13 @@ def put_loaded_logs(split: Split, train_df, test_df, additional_columns): def put_labelled_logs(job: Job, train_df, test_df): + """uploads the labelled logs + + uploads the training and test DataFrames labelled using the given job configuration + :param job: + :param train_df: + :param test_df: + """ train_df_path = "encoding{}-label{}-splitTR{}".format(job.encoding.id, job.labelling.id, job.split.train_log.name) test_df_path = "encoding{}-label{}-splitTE{}".format(job.encoding.id, job.labelling.id, job.split.train_log.name) [dump_to_cache(path, data, 'cache/labeled_log_cache/') for (path, data) in [ @@ -54,6 +87,12 @@ def put_labelled_logs(job: Job, train_df, test_df): def get_loaded_logs(split: Split) -> (DataFrame, DataFrame, DataFrame): + """returns the loaded logs + + returns the training and test DataFrames loaded using + :param split: + :return: + """ logger.info('\t\tFound pre-loaded Dataset in cache, loading..') cache = LoadedLog.objects.filter(split=split)[0] return ( @@ -64,6 +103,12 @@ def get_loaded_logs(split: Split) -> (DataFrame, DataFrame, DataFrame): def get_labelled_logs(job: Job) -> (DataFrame, DataFrame): + """returns the labelled logs + + returns the training and test DataFrames labelled using the given job configuration + :param job: + :return: + """ logger.info('\t\tFound pre-labeled Dataset in cache, loading..') cache = LabelledLog.objects.filter(split=job.split, encoding=job.encoding, diff --git a/src/clustering/clustering.py b/src/clustering/clustering.py index 195a655..a7a6d23 100644 --- a/src/clustering/clustering.py +++ b/src/clustering/clustering.py @@ -69,6 +69,11 @@ def cluster_data(self, input_df: DataFrame) -> dict: } def _choose_clusterer(self, clustering: src.clustering.models.Clustering): + """chooses the clustering method + + chooses the clustering method using the given clustering + :param clustering: + """ self.config.pop('clustering_method', None) if clustering.clustering_method == ClusteringMethods.KMEANS.value: self.clusterer = KMeans(**self.config) @@ -79,6 +84,12 @@ def _choose_clusterer(self, clustering: src.clustering.models.Clustering): @classmethod def load_model(cls, job: Job): + """returns the clustering method from a model + + returns the clustering method using the given job configuration + :param job: + :return: + """ if job.clustering.clustering_method == ClusteringMethods.KMEANS.value: clusterer = joblib.load(job.clustering.model_path) elif job.clustering.clustering_method == ClusteringMethods.NO_CLUSTER.value: @@ -89,6 +100,13 @@ def load_model(cls, job: Job): def init_clusterer(clustering: Clustering, train_data: DataFrame): + """returns a new cluster + + returns a new cluster, fitted on the train_data DataFrame + :param clustering: + :param train_data: + :return: + """ clusterer = Clustering(clustering) clusterer.fit(train_data.drop(['trace_id', 'label'], 1)) return clusterer diff --git a/src/core/core.py b/src/core/core.py index b994f92..147afeb 100644 --- a/src/core/core.py +++ b/src/core/core.py @@ -101,7 +101,7 @@ def run_by_type(training_df: DataFrame, test_df: DataFrame, job: Job) -> (dict, def runtime_calculate(job: Job) -> dict: """calculate the prediction for traces in the uncompleted logs - :param job: job idctionary + :param job: job dictionary :return: runtime results """ @@ -112,7 +112,7 @@ def runtime_calculate(job: Job) -> dict: return results -def replay_prediction_calculate(job: Job, log) -> (dict, dict): +def replay_prediction_calculate(job: Job, log: EventLog) -> (dict, dict): """calculate the prediction for the log coming from replayers :param job: job dictionary @@ -146,6 +146,7 @@ def get_run(job: Job) -> str: def _label_task(input_dataframe: DataFrame) -> dict: """calculates the distribution of labels in the data frame + :param input_dataframe: :return: Dict of string and int {'label1': label1_count, 'label2': label2_count} """ diff --git a/src/encoding/boolean_frequency.py b/src/encoding/boolean_frequency.py index 330aada..89738e8 100644 --- a/src/encoding/boolean_frequency.py +++ b/src/encoding/boolean_frequency.py @@ -9,10 +9,26 @@ def boolean(log: EventLog, event_names: list, label: Labelling, encoding: Encoding) -> DataFrame: + """Encodes the log using boolean encoding + + :param log: + :param event_names: + :param label: + :param encoding: + :return: + """ return _encode_boolean_frequency(log, event_names, label, encoding) def frequency(log: EventLog, event_names: list, label: Labelling, encoding: Encoding) -> DataFrame: + """Encodes the log using frequency encoding + + :param log: + :param event_names: + :param label: + :param encoding: + :return: + """ return _encode_boolean_frequency(log, event_names, label, encoding) @@ -21,6 +37,10 @@ def _encode_boolean_frequency(log: EventLog, event_names: list, labelling: Label """Encodes the log by boolean or frequency trace_id, event_nr, event_names, label stuff + :param log: + :param event_names: + :param label: + :param encoding: :return pandas DataFrame """ columns = _create_columns(event_names, encoding, labelling) @@ -46,7 +66,12 @@ def _encode_boolean_frequency(log: EventLog, event_names: list, labelling: Label def _create_event_happened(event_names: list, encoding: Encoding) -> list: - """Creates list of event happened placeholders""" + """Creates list of event happened placeholders + + :param event_names: + :param encoding: + :return: + """ if encoding.value_encoding == ValueEncodings.BOOLEAN.value: return [False] * len(event_names) return [0] * len(event_names) @@ -57,6 +82,11 @@ def _update_event_happened(event, event_names: list, event_happened: list, encod For boolean set happened to True. For frequency updates happened count. + :param event + :param event_names: + :param event_happened: + :param encoding: + :return: """ event_name = event['concept:name'] if event_name in event_names: @@ -68,6 +98,13 @@ def _update_event_happened(event, event_names: list, event_happened: list, encod def _create_columns(event_names: list, encoding: Encoding, labelling: Labelling) -> list: + """Returns a new columns + + :param event_names: + :param encoding: + :param labelling: + :return: + """ columns = ["trace_id"] columns = list(np.append(columns, event_names).tolist()) return compute_label_columns(columns, encoding, labelling) @@ -75,6 +112,21 @@ def _create_columns(event_names: list, encoding: Encoding, labelling: Labelling) def _trace_to_row(trace: Trace, encoding: Encoding, event_index: int, labelling: Labelling = None, executed_events=None, resources_used=None, new_traces=None, event_names=None, atr_classifier=None): + """Transforms trace into a list comprehension + + :param trace: + :param encoding: + :param labelling: + :param event_index: + :param data_fun: + :param columns_len: + :param atr_classifier: + :param executed_events: + :param resources_used: + :param new_traces: + :param additional_columns + :return: + """ # starts with all False, changes to event event_happened = _create_event_happened(event_names, encoding) trace_row = [] diff --git a/src/encoding/common.py b/src/encoding/common.py index 406acc6..2b194ab 100644 --- a/src/encoding/common.py +++ b/src/encoding/common.py @@ -27,6 +27,16 @@ def encode_label_logs(training_log: EventLog, test_log: EventLog, job: Job, additional_columns=None, encode=True): + """returns the encoded label logs + + returns the training and test DataFrames with encoded column 'label' using the given job configuration + :param training_log: + :param test_log: + :param job: + :param additional_columns: + :param encode: + :return: + """ logger.info('\tDataset not found in cache, building..') training_log, cols = _eventlog_to_dataframe(training_log, job.encoding, job.labelling, additional_columns=additional_columns, cols=None) test_log, _ = _eventlog_to_dataframe(test_log, job.encoding, job.labelling, additional_columns=additional_columns, cols=cols) @@ -61,6 +71,15 @@ def encode_label_logs(training_log: EventLog, test_log: EventLog, job: Job, addi def _eventlog_to_dataframe(log: EventLog, encoding: Encoding, labelling: Labelling, additional_columns=None, cols=None): + """converting eventlog into a DataFrame + + :param training_log: + :param test_log: + :param job: + :param additional_columns: + :param encode: + :return: + """ if encoding.prefix_length < 1: raise ValueError("Prefix length must be greater than 1") if encoding.value_encoding == ValueEncodings.SIMPLE_INDEX.value: @@ -88,7 +107,16 @@ def _eventlog_to_dataframe(log: EventLog, encoding: Encoding, labelling: Labelli return run_df, cols -def _data_encoder_encoder(job: Job, training_log, test_log) -> Encoder: +def _data_encoder_encoder(job: Job, training_log: EventLog, test_log: EventLog) -> Encoder: + """uses data_encoder to encoder DataFrame + + :param training_log: + :param test_log: + :param job: + :param additional_columns: + :param encode: + :return: + """ if job.type != JobTypes.LABELLING.value and \ job.encoding.value_encoding != ValueEncodings.BOOLEAN.value and \ job.predictive_model.predictive_model != PredictiveModels.TIME_SERIES_PREDICTION.value: @@ -107,12 +135,24 @@ def _data_encoder_encoder(job: Job, training_log, test_log) -> Encoder: return encoder -def data_encoder_decoder(job: Job, training_log, test_log) -> None: +def data_encoder_decoder(job: Job, training_log: EventLog, test_log: EventLog) -> None: + """uses data_encoder to decoder DataFrame + + :param training_log: + :param test_log: + :param job: + :return: + """ encoder = retrieve_proper_encoder(job) encoder.decode(training_log, job.encoding), encoder.decode(test_log, job.encoding) def retrieve_proper_encoder(job: Job) -> Encoder: + """find the proper encoder + + :param job: + :return: + """ if job.incremental_train is not None: return retrieve_proper_encoder(job.incremental_train) else: diff --git a/src/encoding/complex_last_payload.py b/src/encoding/complex_last_payload.py index 5ac7540..b6998ba 100644 --- a/src/encoding/complex_last_payload.py +++ b/src/encoding/complex_last_payload.py @@ -13,16 +13,42 @@ def complex(log: EventLog, labelling: Labelling, encoding: Encoding, additional_columns: dict) -> DataFrame: + """Encodes the log using complex encoding + + :param log: + :param labelling: + :param encoding: + :param additional_columns: + :return: + """ return _encode_complex_latest(log, labelling, encoding, additional_columns, _columns_complex, _data_complex) def last_payload(log: EventLog, labelling: Labelling, encoding: Encoding, additional_columns: dict) -> DataFrame: + """Encodes the log using last_payload encoding + + :param log: + :param labelling: + :param encoding: + :param additional_columns: + :return: + """ return _encode_complex_latest(log, labelling, encoding, additional_columns, _columns_last_payload, _data_last_payload) def _encode_complex_latest(log: EventLog, labelling: Labelling, encoding: Encoding, additional_columns: dict, column_fun: Callable, data_fun: Callable) -> DataFrame: + """Encodes the log by complex or last_payload + + :param log: + :param labelling: + :param encoding: + :param additional_columns: + :param column_fun: + :param data_fun: + :return: + """ columns = column_fun(encoding.prefix_length, additional_columns) normal_columns_number = len(columns) columns = compute_label_columns(columns, encoding, labelling) @@ -48,6 +74,13 @@ def _encode_complex_latest(log: EventLog, labelling: Labelling, encoding: Encodi def _columns_complex(prefix_length: int, additional_columns: dict) -> list: + """Creates list in form [event1[concept:name], event1[attributes], event2[concept:name], event2[attributes], ..., eventN[concept:name], eventN[attributes]] + + Appends values in additional_columns + :param prefix_length: + :param additional_columns: + :return: + """ columns = ['trace_id'] columns += additional_columns['trace_attributes'] for i in range(1, prefix_length + 1): @@ -58,6 +91,13 @@ def _columns_complex(prefix_length: int, additional_columns: dict) -> list: def _columns_last_payload(prefix_length: int, additional_columns: dict) -> list: + """Creates list in form [event1[concept:name], event2[concept:name], ..., eventN[concept:name], eventN[attributes]] + + Appends values in additional_columns + :param prefix_length: + :param additional_columns: + :return: + """ columns = ['trace_id'] i = 0 for i in range(1, prefix_length + 1): @@ -71,6 +111,10 @@ def _data_complex(trace: Trace, prefix_length: int, additional_columns: dict) -> """Creates list in form [1, value1, value2, 2, ...] Appends values in additional_columns + :param trace: + :param prefix_length: + :param additional_columns: + :return: """ data = [trace.attributes.get(att, 0) for att in additional_columns['trace_attributes']] for idx, event in enumerate(trace): @@ -90,6 +134,10 @@ def _data_last_payload(trace: list, prefix_length: int, additional_columns: dict Event name index of the position they are in event_names Appends values in additional_columns + :param trace: + :param prefix_length: + :param additional_columns: + :return: """ data = list() for idx, event in enumerate(trace): @@ -112,6 +160,21 @@ def _trace_to_row(trace: Trace, encoding: Encoding, labelling: Labelling, event_ columns_len: int, atr_classifier=None, executed_events=None, resources_used=None, new_traces=None, additional_columns: dict = None) -> list: + """Transforms trace into a list comprehension + + :param trace: + :param encoding: + :param labelling: + :param event_index: + :param data_fun: + :param columns_len: + :param atr_classifier: + :param executed_events: + :param resources_used: + :param new_traces: + :param additional_columns + :return: + """ trace_row = [trace.attributes["concept:name"]] # prefix_length - 1 == index trace_row += data_fun(trace, event_index, additional_columns) diff --git a/src/encoding/declare/declare.py b/src/encoding/declare/declare.py index b37f62c..46105a4 100644 --- a/src/encoding/declare/declare.py +++ b/src/encoding/declare/declare.py @@ -29,6 +29,15 @@ def xes_to_positional(log, label=True): def declare_encoding(log, labelling, encoding, additional_columns, cols=None): #TODO JONAS + """creates and returns the DataFrame encoded using the declare encoding + + :param log: + :param labelling: + :param encoding: + :param additional_columns: + :param cols: + :return: + """ filter_t = True print("Filter_t", filter_t) templates = template_sizes.keys() diff --git a/src/encoding/declare/declare_mining.py b/src/encoding/declare/declare_mining.py index fd4456e..f7c2998 100644 --- a/src/encoding/declare/declare_mining.py +++ b/src/encoding/declare/declare_mining.py @@ -7,6 +7,13 @@ def apply_template_to_log(template, candidate, log): + """returns the log with template applied + + :param template: + :param candidate: + :param log: + :return: + """ results = [] for trace in log: result, vacuity = apply_template(template, log[trace], candidate) @@ -17,6 +24,15 @@ def apply_template_to_log(template, candidate, log): def find_if_satisfied_by_class(constraint_result, transformed_log, labels, support_true, support_false): + """returns two boolean variable show if class is trusted + + :param constraint_result: + :param transformed_log: + :param labels: + :param support_true: + :param support_false: + :return: + """ fulfill_true = 0 fulfill_false = 0 for i, trace in enumerate(transformed_log): @@ -36,6 +52,17 @@ def find_if_satisfied_by_class(constraint_result, transformed_log, labels, suppo def generate_train_candidate_constraints(candidates, templates, transformed_log, labels, constraint_support_true, constraint_support_false, filter_t=True): + """returns the train-candidate's constraints + + :param candidates: + :param templates: + :param transformed_log: + :param labels: + :param constraint_support_true: + :param constraint_support_false: + :param filter_t: + :return: + """ all_results = {} for template in templates: print("Started working on {}".format(template)) @@ -57,7 +84,9 @@ def transform_results_to_numpy(results, labels, transformed_log, cols): """ Transforms results structure into numpy arrays :param results: + :param labels: :param transformed_log: + :param cols: :return: """ labels = [labels[trace] for trace in transformed_log] @@ -85,6 +114,15 @@ def transform_results_to_numpy(results, labels, transformed_log, cols): def filter_candidates_by_support(candidates, transformed_log, labels, support_true, support_false): #TODO JONAS, no idea what this does + """returns candidates filtered using given support_true and support_false + + :param candidates: + :param transformed_log: + :param labels: + :param support_true: + :param support_false: + :return: + """ filtered_candidates = [] for candidate in candidates: count_false = 0 diff --git a/src/encoding/encoder.py b/src/encoding/encoder.py index 4123546..cfcff8c 100644 --- a/src/encoding/encoder.py +++ b/src/encoding/encoder.py @@ -25,6 +25,11 @@ def __init__(self, df: DataFrame, encoding: Encoding): self._init_encoder(df, encoding) def _init_encoder(self, df: DataFrame, encoding: Encoding): + """initializes the encoder object, creating label encoder for each column in the DataFrame + + :param df: + :param encoding: + """ for column in df: if column != 'trace_id': if df[column].dtype != int or (df[column].dtype == int and np.any(df[column] < 0)): @@ -41,6 +46,11 @@ def _init_encoder(self, df: DataFrame, encoding: Encoding): raise ValueError('Please set the encoding technique!') def encode(self, df: DataFrame, encoding: Encoding) -> None: + """Encodes the Dataframe using given encoding technique + + :param df: + :param encoding: + """ for column in df: if column in self._encoder: if encoding.data_encoding == DataEncodings.LABEL_ENCODER.value: @@ -51,6 +61,11 @@ def encode(self, df: DataFrame, encoding: Encoding) -> None: raise ValueError('Please set the encoding technique!') def decode(self, df: DataFrame, encoding: Encoding) -> None: + """Decodes the Dataframe using given encoding technique + + :param df: + :param encoding: + """ for column in df: if column in self._encoder: if encoding.data_encoding == DataEncodings.LABEL_ENCODER.value: diff --git a/src/encoding/encoding_container.py b/src/encoding/encoding_container.py index 45ecf1c..c73452c 100644 --- a/src/encoding/encoding_container.py +++ b/src/encoding/encoding_container.py @@ -53,6 +53,10 @@ def is_complex(self) -> bool: @staticmethod def encode(df: DataFrame) -> None: + """Encodes the Dataframe + + :param df: + """ for column in df: if column in encoder: if ENCODING == DataEncodings.LABEL_ENCODER.value: @@ -67,6 +71,10 @@ def encode(df: DataFrame) -> None: @staticmethod def init_label_encoder(df: DataFrame) -> None: + """creates the encoder + + :param df: + """ for column in df: if column != 'trace_id': if df[column].dtype != int or (df[column].dtype == int and pd.np.any(df[column] < 0)): diff --git a/src/encoding/simple_index.py b/src/encoding/simple_index.py index e6ea6a1..3592cdc 100644 --- a/src/encoding/simple_index.py +++ b/src/encoding/simple_index.py @@ -11,6 +11,12 @@ def simple_index(log: EventLog, labelling: Labelling, encoding: Encoding) -> DataFrame: + """Creates list in form [event1[concept:name], event2[concept:name], ..., eventN[concept:name]] + :param log: + :param encoding: + :param labelling: + :return: + """ columns = _compute_columns(encoding.prefix_length) normal_columns_number = len(columns) columns = compute_label_columns(columns, encoding, labelling) @@ -33,7 +39,19 @@ def simple_index(log: EventLog, labelling: Labelling, encoding: Encoding) -> Dat def add_trace_row(trace: Trace, encoding: Encoding, labelling: Labelling, event_index: int, column_len: int, attribute_classifier=None, executed_events=None, resources_used=None, new_traces=None): - """Row in data frame""" + """Row in data frame + + :param trace: + :param encoding: + :param labelling: + :param event_index: + :param column_len: + :param attribute_classifier: + :param executed_events: + :param resources_used: + :param new_traces: + :return: + """ trace_row = [trace.attributes['concept:name']] trace_row += _trace_prefixes(trace, event_index) if encoding.padding or encoding.task_generation_type == TaskGenerationTypes.ALL_IN_ONE.value: @@ -46,6 +64,9 @@ def add_trace_row(trace: Trace, encoding: Encoding, labelling: Labelling, event_ def _trace_prefixes(trace: Trace, prefix_length: int) -> list: """List of indexes of the position they are in event_names + :param trace: + :param prefix_length: + :return: """ prefixes = [] for idx, event in enumerate(trace): @@ -59,5 +80,7 @@ def _trace_prefixes(trace: Trace, prefix_length: int) -> list: def _compute_columns(prefix_length: int) -> list: """trace_id, prefixes, any other columns, label + :param prefix_length: + :return: """ return ["trace_id"] + [PREFIX_ + str(i + 1) for i in range(0, prefix_length)] diff --git a/src/explanation/anchor_wrapper.py b/src/explanation/anchor_wrapper.py index 80c8b98..6160c82 100644 --- a/src/explanation/anchor_wrapper.py +++ b/src/explanation/anchor_wrapper.py @@ -8,6 +8,15 @@ def explain(anchor_exp: Explanation, training_df, test_df, explanation_target, prefix_target): + """ + + :param anchor_exp: + :param training_df: + :param test_df: + :param explanation_target: + :param prefix_target: + :return: + """ job = Job.objects.filter(pk=anchor_exp.job.id)[0] explainer = anchor_tabular.AnchorTabularExplainer( diff --git a/src/explanation/cm_feedback_wrapper.py b/src/explanation/cm_feedback_wrapper.py index d7a2c76..92ea946 100644 --- a/src/explanation/cm_feedback_wrapper.py +++ b/src/explanation/cm_feedback_wrapper.py @@ -13,7 +13,15 @@ -def retrieve_temporal_stability(training_df, test_df, job_obj, split_obj): +def retrieve_temporal_stability(training_df: DataFrame, test_df: DataFrame, job_obj: Job, split_obj: Job): + """ + + :param training_df: + :param test_df: + :param job_obj: + :param split_obj: + :return: + """ ts_exp_job, _ = Explanation.objects.get_or_create( type=ExplanationTypes.TEMPORAL_STABILITY.value, split=split_obj, diff --git a/src/hyperparameter_optimization/hyperopt_spaces.py b/src/hyperparameter_optimization/hyperopt_spaces.py index cc4a60c..d41ec8c 100755 --- a/src/hyperparameter_optimization/hyperopt_spaces.py +++ b/src/hyperparameter_optimization/hyperopt_spaces.py @@ -17,11 +17,20 @@ def _get_space(job: Job) -> dict: + """Returns the specific space for the used machine learning algorithm for the given job configuration + + :param job: + :return: + """ method_conf_name = "{}.{}".format(job.predictive_model.predictive_model, job.predictive_model.prediction_method) return HYPEROPT_SPACE_MAP[method_conf_name]() def _classification_random_forest() -> dict: + """Returns the random forest prediction classification method + + :return: + """ return { 'n_estimators': hp.choice('n_estimators', np.arange(150, 1000, dtype=int)), 'max_depth': scope.int(hp.quniform('max_depth', 4, 30, 1)), @@ -31,6 +40,10 @@ def _classification_random_forest() -> dict: def _classification_knn() -> dict: + """Returns the KNN prediction classification method + + :return: + """ return { 'n_neighbors': hp.choice('n_neighbors', np.arange(1, 20, dtype=int)), 'weights': hp.choice('weights', ['uniform', 'distance']), @@ -38,6 +51,10 @@ def _classification_knn() -> dict: def _classification_decision_tree() -> dict: + """Returns the decision tree prediction classification method + + :return: + """ return { 'max_depth': scope.int(hp.quniform('max_depth', 4, 30, 1)), 'min_samples_split': hp.choice('min_samples_split', np.arange(2, 10, dtype=int)), @@ -46,6 +63,10 @@ def _classification_decision_tree() -> dict: def _classification_xgboost() -> dict: + """Returns the XGBoost prediction classification method + + :return: + """ return { 'n_estimators': hp.choice('n_estimators', np.arange(150, 1000, dtype=int)), 'max_depth': scope.int(hp.quniform('max_depth', 3, 30, 1)), @@ -53,6 +74,10 @@ def _classification_xgboost() -> dict: def _classification_incremental_naive_bayes() -> dict: + """Returns the incremental naive bayes prediction classification method + + :return: + """ return { 'alpha': hp.uniform('alpha', 0, 10), 'fit_prior': True @@ -60,6 +85,10 @@ def _classification_incremental_naive_bayes() -> dict: def _classification_incremental_adaptive_tree() -> dict: + """Returns the incremental adaptive tree prediction classification method + + :return: + """ return { 'grace_period': hp.uniform('grace_period', 1, 5), 'split_criterion': hp.choice('split_criterion', ['gini', 'info_gain']), @@ -75,6 +104,10 @@ def _classification_incremental_adaptive_tree() -> dict: def _classification_incremental_hoeffding_tree() -> dict: + """Returns the incremental hoeffding tree prediction classification method + + :return: + """ return { 'grace_period': hp.uniform('grace_period', 3, 8), 'split_criterion': hp.choice('split_criterion', ['gini', 'info_gain']), @@ -90,6 +123,10 @@ def _classification_incremental_hoeffding_tree() -> dict: def _classification_incremental_sgd_classifier() -> dict: + """Returns the incremental sgd classifier prediction classification method + + :return: + """ return { 'loss': hp.choice('loss', ['hinge', 'log', 'modified_huber', 'squared_hinge', 'perceptron', 'squared_loss', 'huber', 'epsilon_insensitive', 'squared_epsilon_insensitive']), @@ -110,6 +147,10 @@ def _classification_incremental_sgd_classifier() -> dict: def _classification_incremental_perceptron() -> dict: + """Returns the incremental perceptron prediction classification method + + :return: + """ return { 'penalty': hp.choice('penalty', [None, 'l1', 'l2', 'elasticnet']), 'alpha': hp.uniform('alpha', 0.0001, 0.5), @@ -124,6 +165,10 @@ def _classification_incremental_perceptron() -> dict: def _classification_nn() -> dict: + """Returns the NN prediction classification method + + :return: + """ return { 'hidden_layers': hp.quniform('hidden_layers', 1, 10, 1), 'hidden_units': hp.quniform('hidden_units', 1, 100, 1), @@ -134,6 +179,10 @@ def _classification_nn() -> dict: def _regression_random_forest() -> dict: + """Returns the random forest prediction regression method + + :return: + """ return { 'n_estimators': hp.choice('n_estimators', np.arange(150, 1000, dtype=int)), 'max_features': hp.choice('max_features', ['sqrt', 'log2', 'auto', None]), @@ -142,6 +191,10 @@ def _regression_random_forest() -> dict: def _regression_lasso() -> dict: + """Returns the lasso prediction regression method + + :return: + """ return { 'alpha': hp.uniform('alpha', 0.01, 2.0), 'fit_intercept': hp.choice('fit_intercept', [True, False]), @@ -150,6 +203,10 @@ def _regression_lasso() -> dict: def _regression_linear() -> dict: + """Returns the linear prediction regression method + + :return: + """ return { 'fit_intercept': hp.choice('fit_intercept', [True, False]), 'normalize': hp.choice('normalize', [True, False]) @@ -157,6 +214,10 @@ def _regression_linear() -> dict: def _regression_xgboost() -> dict: + """Returns the XGBoost prediction regression method + + :return: + """ return { 'max_depth': scope.int(hp.quniform('max_depth', 3, 100, 1)), 'n_estimators': hp.choice('n_estimators', np.arange(150, 1000, dtype=int)), @@ -164,6 +225,10 @@ def _regression_xgboost() -> dict: def _regression_nn() -> dict: + """Returns the NN prediction regression method + + :return: + """ return { 'hidden_layers': hp.quniform('hidden_layers', 1, 10, 1), 'hidden_units': hp.quniform('hidden_units', 1, 100, 1), @@ -174,6 +239,10 @@ def _regression_nn() -> dict: def _time_series_prediction_rnn() -> dict: + """TODO: complete commit description + + :return: + """ raise NotImplementedError diff --git a/src/hyperparameter_optimization/hyperopt_wrapper.py b/src/hyperparameter_optimization/hyperopt_wrapper.py index 2cc20f8..23ee640 100755 --- a/src/hyperparameter_optimization/hyperopt_wrapper.py +++ b/src/hyperparameter_optimization/hyperopt_wrapper.py @@ -33,6 +33,14 @@ def _retrieve_train_validate_test(local_train_df, local_test_df): + """returns the train validate test + + :param space: + :param algorithm_suggest: + :param max_evaluations: + :param trials: + :return: + """ validation_df = local_train_df.tail(# uses last 20% of train set to validate result or at least one trace max( int(len(local_train_df) * 20 / 100), 1 ) ) @@ -41,6 +49,14 @@ def _retrieve_train_validate_test(local_train_df, local_test_df): def _run_hyperoptimisation(space, algorithm_suggest, max_evaluations, trials): + """optimizes the given hyperparameter space + + :param space: + :param algorithm_suggest: + :param max_evaluations: + :param trials: + :return: + """ try: return fmin(_calculate_and_evaluate, space, algo=algorithm_suggest, max_evals=max_evaluations, trials=trials) except ValueError: @@ -48,6 +64,13 @@ def _run_hyperoptimisation(space, algorithm_suggest, max_evaluations, trials): def _test_best_candidate(current_best, job_labelling_type, job_type): + """finds the best candidate + + :param current_best: + :param job_labelling_type: + :param job_type: + :return: + """ if job_type == PredictiveModels.CLASSIFICATION.value: return classification_test(current_best['model_split'], test_df.drop(['trace_id'], 1), evaluation=True, is_binary_classifier=_check_is_binary_classifier(job_labelling_type)) @@ -56,6 +79,13 @@ def _test_best_candidate(current_best, job_labelling_type, job_type): def run_hyperopt(job, original_training_df, original_test_df): + """optimizes the given job configuration + + :param job: + :param original_training_df: + :param original_test_df: + :return: + """ global train_df, test_df, global_job, validation_df global_job = job train_df, test_df = original_training_df.copy(), original_test_df.copy() diff --git a/src/jobs/job_creator.py b/src/jobs/job_creator.py index d623529..81873db 100644 --- a/src/jobs/job_creator.py +++ b/src/jobs/job_creator.py @@ -12,6 +12,12 @@ def generate(split, payload): + """Returns a list of job + + :param split: + :param payload: + :return: + """ jobs = [] config = payload['config'] @@ -128,6 +134,13 @@ def check_predictive_model_not_overwrite(job: Job) -> None: def get_prediction_method_config(predictive_model, prediction_method, payload): + """Returns a dict contain the configuration of prediction method + + :param predictive_model: + :param prediction_method: + :param payload: + :return: + """ return { 'predictive_model': predictive_model, 'prediction_method': prediction_method, @@ -136,6 +149,10 @@ def get_prediction_method_config(predictive_model, prediction_method, payload): def set_model_name(job: Job) -> None: + """Sets the model using the given job configuration + + :param job: + """ if job.create_models: if job.predictive_model.model_path != '': # job.predictive_model = duplicate_orm_row(PredictiveModel.objects.filter(pk=job.predictive_model.pk)[0]) #todo: replace with simple CREATE @@ -170,6 +187,12 @@ def set_model_name(job: Job) -> None: def generate_labelling(split, payload): + """Returns a list of job + + :param split: + :param payload: + :return: + """ jobs = [] encoding = payload['config']['encoding'] @@ -242,6 +265,13 @@ def generate_labelling(split, payload): def update(split, payload, generation_type=PredictiveModels.CLASSIFICATION.value): # TODO adapt to allow selecting the predictive_model to update + """Returns a list of job + + :param split: + :param payload: + :param generation_type: + :return: + """ jobs = [] config = payload['config'] diff --git a/src/jobs/json_renderer.py b/src/jobs/json_renderer.py index 51a903c..1294fc4 100644 --- a/src/jobs/json_renderer.py +++ b/src/jobs/json_renderer.py @@ -12,6 +12,10 @@ def iterencode(self, o, _one_shot=False): """Encode the given object and yield each string representation as available. + :param o: + :param _one_shot: + :return: + For example:: for chunk in JSONEncoder().iterencode(bigobject): @@ -30,6 +34,15 @@ def iterencode(self, o, _one_shot=False): _encoder = encode_basestring def floatstr(o, allow_nan=True, _repr=lambda o: format(o, '.4f'), _inf=INFINITY, _neginf=-INFINITY): + """Convert float number into a string + + :param o: + :param allow_nan: + :param _repr: + :param _inf: + :param _neginf: + :return: + """ # Check for specials. Note that this type of test is processor # and/or platform-specific, so do tests which don't depend on the # internals. diff --git a/src/jobs/serializers.py b/src/jobs/serializers.py index ceed5bd..4a4d33d 100644 --- a/src/jobs/serializers.py +++ b/src/jobs/serializers.py @@ -10,6 +10,11 @@ class JobSerializer(serializers.ModelSerializer): config = serializers.SerializerMethodField() def get_config(self, job): + """Returns the the parameter of the given job configuration, converted into dictionary + + :param job: + :return + """ evaluation = Evaluation.objects.filter(pk=job.evaluation.pk).select_subclasses()[ 0] if job.evaluation is not None else None hyperparameter_optimizer = \ diff --git a/src/jobs/tasks.py b/src/jobs/tasks.py index aeeb7d3..f06ae8b 100644 --- a/src/jobs/tasks.py +++ b/src/jobs/tasks.py @@ -18,6 +18,12 @@ @job("default", timeout='100h') def prediction_task(job_id, do_publish_result=True): + """Predict the task using the given job id + + :param job_id: + :param do_publish_result: + """ + logger.info("Start prediction task ID {}".format(job_id)) job = Job.objects.get(id=job_id) @@ -58,6 +64,11 @@ def prediction_task(job_id, do_publish_result=True): def save_models(models: dict, job: Job): + """Saves the given models in the given job configuration + + :param models: + :param job: + """ set_model_name(job) logger.info("\tStart saving models of JOB {}".format(job.id)) diff --git a/src/jobs/ws_publisher.py b/src/jobs/ws_publisher.py index d98f1d1..19265ab 100644 --- a/src/jobs/ws_publisher.py +++ b/src/jobs/ws_publisher.py @@ -11,6 +11,7 @@ def publish(object): """ Publish an object to websocket listeners + :param object: A Django predictive_model :return: {type: object class name, data: OBJECT} """ @@ -19,7 +20,11 @@ def publish(object): def _serializer(object): - """Assumed to be Django models""" + """Assumed to be Django models + + :param object: + :return: + """ name = object.__class__.__name__ if name == 'Log': data = LogSerializer(object).data diff --git a/src/labelling/common.py b/src/labelling/common.py index 5002691..a1913cd 100644 --- a/src/labelling/common.py +++ b/src/labelling/common.py @@ -9,6 +9,10 @@ def get_intercase_attributes(log: EventLog, encoding: Encoding): """Dict of kwargs These intercase attributes are expensive operations!!! + + :param log: + :param encoding: + :return: """ # Expensive operations executed_events = events_by_date(log) if encoding.add_executed_events else None @@ -20,6 +24,13 @@ def get_intercase_attributes(log: EventLog, encoding: Encoding): def compute_label_columns(columns: list, encoding: Encoding, labelling: Labelling) -> list: + """Compute the label columns + + :param columns: + :param encoding: + :param labelling: + :return: + """ if labelling.type == LabelTypes.NO_LABEL.value: return columns if encoding.add_elapsed_time: @@ -40,6 +51,16 @@ def add_labels(encoding: Encoding, labelling: Labelling, prefix_length: int, tra executed_events=None, resources_used=None, new_traces=None): """ Adds any number of label cells with last as label + + :param encoding: + :param labelling: + :param prefix_length: + :param trace: + :param attribute_classifier: + :param executed_events: + :param resources_used: + :param new_traces: + :return: """ labels = [] if labelling.type == LabelTypes.NO_LABEL.value: @@ -70,6 +91,9 @@ def add_labels(encoding: Encoding, labelling: Labelling, prefix_length: int, tra def next_event_name(trace: list, prefix_length: int): """Return the event event name at prefix length or 0 if out of range. + :param trace: + :param prefix_length: + :return: """ if prefix_length < len(trace): next_event = trace[prefix_length] diff --git a/src/logs/log_service.py b/src/logs/log_service.py index a643532..cdfa772 100644 --- a/src/logs/log_service.py +++ b/src/logs/log_service.py @@ -20,6 +20,11 @@ def import_log_csv(path): + """Returns the log, took from the given path + + :param path: + :return: + """ return conversion_factory.apply( import_event_stream(path), # https://pm4py.fit.fraunhofer.de/documentation/1.2 parameters={constants.PARAMETER_CONSTANT_CASEID_KEY: "case:concept:name", # this tells the importer @@ -39,6 +44,14 @@ def import_log_csv(path): def create_log(log, name: str, folder='cache/log_cache/', import_in_cache=True): + """Creates new file log in memory + + :param log: + :param name: + :param folder: + :param import_in_cache: + :return: + """ logger.info('\tCreating new file (' + name + ') in memory') if import_in_cache: name = create_unique_name(name) @@ -57,7 +70,11 @@ def create_log(log, name: str, folder='cache/log_cache/', import_in_cache=True): def create_properties(log: EventLog) -> dict: - """Create read-only dict with methods in this class""" + """Create read-only dict with methods in this class + + :param log: + :return: + """ return { 'events': events_by_date(log), 'resources': resources_by_date(log), @@ -72,6 +89,11 @@ def create_properties(log: EventLog) -> dict: def get_log_trace_attributes(log: EventLog) -> list: + """Returns a list of dict, of traces in the given log + + :param log: + :return: + """ return traces_in_log(log) @@ -79,6 +101,8 @@ def get_log(log: Log) -> EventLog: """Read in event log from disk Uses xes_importer to parse log. + :param log: + :return: """ filepath = log.path logger.info("\t\tReading in log from {}".format(filepath)) diff --git a/src/predictive_model/classification/classification.py b/src/predictive_model/classification/classification.py index a961ce5..0cbc74c 100644 --- a/src/predictive_model/classification/classification.py +++ b/src/predictive_model/classification/classification.py @@ -78,6 +78,13 @@ def classification(training_df: DataFrame, test_df: DataFrame, clusterer: Cluste def update_and_test(training_df: DataFrame, test_df: DataFrame, job: Job): + """ + + :param training_df: + :param test_df: + :param job: + :return: + """ train_data = _drop_columns(training_df) test_data = _drop_columns(test_df) @@ -106,6 +113,13 @@ def update_and_test(training_df: DataFrame, test_df: DataFrame, job: Job): def _train(train_data: DataFrame, classifier: ClassifierMixin, clusterer: Clustering) -> dict: + """Initializes and train the predictive model with the given data + + :param train_data: + :param classifier: + :param clusterer: + :return: + """ models = dict() train_data = clusterer.cluster_data(train_data) @@ -132,6 +146,12 @@ def _train(train_data: DataFrame, classifier: ClassifierMixin, clusterer: Cluste def _update(job: Job, data: DataFrame) -> dict: + """Updates the existing model + + :param job: + :param data: + :return: + """ previous_job = job.incremental_train clusterer = Clustering.load_model(previous_job) @@ -167,6 +187,14 @@ def _update(job: Job, data: DataFrame) -> dict: def _test(model_split: dict, test_data: DataFrame, evaluation: bool, is_binary_classifier: bool) -> (DataFrame, float): + """Tests the given predictive model with the given data + + :param model_split: + :param test_data: + :param evaluation: + :param is_binary_classifier: + :return: + """ clusterer = model_split[ModelType.CLUSTERER.value] classifier = model_split[ModelType.CLASSIFIER.value] @@ -218,6 +246,12 @@ def _test(model_split: dict, test_data: DataFrame, evaluation: bool, is_binary_c def predict(job: Job, data: DataFrame) -> Any: + """Returns the predicted results whit classification + + :param job: + :param data: + :return: + """ data = data.drop(['trace_id'], 1) clusterer = Clustering.load_model(job) data = clusterer.cluster_data(data) @@ -242,6 +276,12 @@ def predict(job: Job, data: DataFrame) -> Any: def predict_proba(job: Job, data: DataFrame) -> Any: + """Returns the probability of predicted results whit classification + + :param job: + :param data: + :return: + """ data = data.drop(['trace_id'], 1) clusterer = Clustering.load_model(job) data = clusterer.cluster_data(data) @@ -266,6 +306,12 @@ def predict_proba(job: Job, data: DataFrame) -> Any: def _prepare_results(results_df: DataFrame, auc: int) -> dict: + """Calculates and returns the results + + :param results_df: + :param auc: + :return: + """ actual = results_df['label'].values predicted = results_df['predicted'].values @@ -275,11 +321,21 @@ def _prepare_results(results_df: DataFrame, auc: int) -> dict: def _drop_columns(df: DataFrame) -> DataFrame: + """Drops the column trace_id of given DataFrame + + :param df: + :return: + """ df = df.drop('trace_id', 1) return df def _choose_classifier(job: Job): + """Chooses the classifier predictive method using the given job configuration + + :param job: + :return: + """ method, config = get_method_config(job) config.pop('classification_method', None) logger.info("Using method {} with config {}".format(method, config)) diff --git a/src/predictive_model/classification/methods_default_config.py b/src/predictive_model/classification/methods_default_config.py index 5b7404b..b7fecc9 100644 --- a/src/predictive_model/classification/methods_default_config.py +++ b/src/predictive_model/classification/methods_default_config.py @@ -1,4 +1,7 @@ def classification_random_forest(): + """" + Returns the default parameters for classification random forest + """ return { ### MANUALLY OPTIMISED PARAMS 'n_estimators': 10, @@ -31,6 +34,9 @@ def classification_random_forest(): def classification_knn(): + """" + Returns the default parameters for classification KNN + """ return { 'n_neighbors': 3, 'n_jobs': -1, @@ -39,6 +45,9 @@ def classification_knn(): def classification_decision_tree(): + """" + Returns the default parameters for classification decision tree + """ return { 'max_depth': None, 'min_samples_split': 2, @@ -48,6 +57,9 @@ def classification_decision_tree(): def classification_incremental_naive_bayes(): + """" + Returns the default parameters for classification incremental naive bayes + """ return { 'alpha': 1.0, 'fit_prior': True, @@ -56,6 +68,9 @@ def classification_incremental_naive_bayes(): def classification_incremental_adaptive_tree(): + """" + Returns the default parameters for classification incremental adaptive tree + """ return { 'max_byte_size': 33554432, 'memory_estimate_period': 1000000, @@ -74,6 +89,9 @@ def classification_incremental_adaptive_tree(): def classification_incremental_hoeffding_tree(): + """" + Returns the default parameters for classification incremental hoeffding tree + """ return { 'max_byte_size': 33554432, 'memory_estimate_period': 1000000, @@ -92,6 +110,9 @@ def classification_incremental_hoeffding_tree(): def classification_incremental_sgd_classifier(): + """" + Returns the default parameters for classification incremental sgd classifier + """ return { 'loss': 'hinge', 'penalty': 'l2', @@ -112,6 +133,9 @@ def classification_incremental_sgd_classifier(): def classification_incremental_perceptron(): + """" + Returns the default parameters for classification incremental perceptron + """ return { 'penalty': None, 'alpha': 0.0001, @@ -127,6 +151,9 @@ def classification_incremental_perceptron(): def classification_xgboost(): + """" + Returns the default parameters for classification XGBoost + """ return { 'max_depth': 3, 'learning_rate': 0.1, @@ -135,6 +162,9 @@ def classification_xgboost(): def classification_nn(): + """" + Returns the default parameters for classification NN + """ return { 'n_hidden_layers': 1, 'n_hidden_units': 10, @@ -145,6 +175,9 @@ def classification_nn(): def _update_incremental_naive_bayes(): + """" + Returns the updated parameters for classification incremental naive bayes + """ return { 'alpha': 1.0, 'fit_prior': True, @@ -153,6 +186,9 @@ def _update_incremental_naive_bayes(): def _update_incremental_adaptive_tree(): + """" + Returns the updated parameters for classification incremental adaptive tree + """ return { 'max_byte_size': 33554432, 'memory_estimate_period': 1000000, @@ -171,6 +207,9 @@ def _update_incremental_adaptive_tree(): def _update_incremental_hoeffding_tree(): + """" + Returns the updated parameters for classification incremental hoeffding tree + """ return { 'max_byte_size': 33554432, 'memory_estimate_period': 1000000, diff --git a/src/predictive_model/regression/methods_default_config.py b/src/predictive_model/regression/methods_default_config.py index 783a696..a5c698b 100644 --- a/src/predictive_model/regression/methods_default_config.py +++ b/src/predictive_model/regression/methods_default_config.py @@ -1,4 +1,7 @@ def regression_random_forest(): + """" + Returns the default parameters for regression random forest + """ return { 'n_estimators': 10, 'max_depth': None, @@ -9,6 +12,9 @@ def regression_random_forest(): def regression_lasso(): + """" + Returns the default parameters for regression lasso + """ return { 'alpha': 1.0, 'fit_intercept': True, @@ -18,6 +24,9 @@ def regression_lasso(): def regression_linear(): + """" + Returns the default parameters for regression linear + """ return { 'fit_intercept': True, 'n_jobs': -1, @@ -26,6 +35,9 @@ def regression_linear(): def regression_xgboost(): + """" + Returns the default parameters for regression XGBoost + """ return { 'n_estimators': 100, 'max_depth': 3 @@ -33,6 +45,9 @@ def regression_xgboost(): def regression_nn(): + """" + Returns the default parameters for regression NN + """ return { 'n_hidden_layers': 1, 'n_hidden_units': 10, diff --git a/src/predictive_model/regression/regression.py b/src/predictive_model/regression/regression.py index c517330..a64db0a 100644 --- a/src/predictive_model/regression/regression.py +++ b/src/predictive_model/regression/regression.py @@ -133,6 +133,14 @@ def regression_single_log(input_df: DataFrame, model: dict) -> DataFrame: def _train(train_data: DataFrame, regressor: RegressorMixin, clusterer: Clustering, do_cv=False) -> dict: + """Initializes and train the predictive model with the given data + + :param train_data: + :param regressor: + :param clusterer: + :param do_cv: + :return: + """ models = dict() train_data = clusterer.cluster_data(train_data) @@ -168,6 +176,12 @@ def _train(train_data: DataFrame, regressor: RegressorMixin, clusterer: Clusteri def _test(model_split: dict, data: DataFrame) -> DataFrame: + """Tests the given predictive model with the given data + + :param model_split: + :param data: + :return: + """ clusterer = model_split[ModelType.CLUSTERER.value] regressor = model_split[ModelType.REGRESSOR.value] @@ -184,6 +198,12 @@ def _test(model_split: dict, data: DataFrame) -> DataFrame: def predict(job: Job, data: DataFrame) -> Any: + """Returns the predicted results whit regression + + :param job: + :param data: + :return: + """ data = data.drop(['trace_id'], 1) clusterer = Clustering.load_model(job) test_data = clusterer.cluster_data(data) @@ -201,6 +221,12 @@ def predict(job: Job, data: DataFrame) -> Any: def _prep_data(training_df: DataFrame, test_df: DataFrame) -> (DataFrame, DataFrame): + """Drops the column trace_id of given DataFrame + + :param training_df: + :param test_df: + :return: + """ train_data = training_df test_data = test_df @@ -210,6 +236,11 @@ def _prep_data(training_df: DataFrame, test_df: DataFrame) -> (DataFrame, DataFr def _choose_regressor(job: Job) -> RegressorMixin: + """Chooses the regressor predictive method using the given job configuration + + :param job: + :return: + """ method, config = get_method_config(job) config.pop('regression_method', None) logger.info("Using method {} with config {}".format(method, config)) diff --git a/src/predictive_model/time_series_prediction/methods_default_config.py b/src/predictive_model/time_series_prediction/methods_default_config.py index 44832a2..65fabf8 100644 --- a/src/predictive_model/time_series_prediction/methods_default_config.py +++ b/src/predictive_model/time_series_prediction/methods_default_config.py @@ -1,4 +1,7 @@ def time_series_prediction_rnn(): + """" + Returns the default parameters for time series prediction RNN + """ return { 'n_units': 16, 'rnn_type': 'lstm', diff --git a/src/predictive_model/time_series_prediction/time_series_prediction.py b/src/predictive_model/time_series_prediction/time_series_prediction.py index f4347d2..3a818e3 100644 --- a/src/predictive_model/time_series_prediction/time_series_prediction.py +++ b/src/predictive_model/time_series_prediction/time_series_prediction.py @@ -52,6 +52,13 @@ def time_series_prediction(training_df: DataFrame, test_df: DataFrame, clusterer def _train(train_data: DataFrame, time_series_predictor: Any, clusterer: Clustering) -> dict: + """Initializes and train the predictive model with the given data + + :param train_data: + :param time_series_predictor: + :param clusterer: + :return: + """ models = dict() train_data = clusterer.cluster_data(train_data) @@ -68,6 +75,13 @@ def _train(train_data: DataFrame, time_series_predictor: Any, clusterer: Cluster def _test(model_split: dict, data: DataFrame, evaluation: bool) -> (DataFrame, float): + """Tests the given predictive model with the given data + + :param model_split: + :param data: + :param evaluation: + :return: + """ clusterer = model_split[ModelType.CLUSTERER.value] time_series_predictor = model_split[ModelType.TIME_SERIES_PREDICTOR.value] @@ -101,6 +115,12 @@ def _test(model_split: dict, data: DataFrame, evaluation: bool) -> (DataFrame, f def predict(job: Job, data: DataFrame) -> Any: + """Returns the predicted results whit time series prediction + + :param job: + :param data: + :return: + """ model_split = joblib.load(job.predictive_model.model_path) clusterer = model_split[ModelType.CLUSTERER.value] test_data = clusterer.cluster_data(data) @@ -122,6 +142,12 @@ def predict(job: Job, data: DataFrame) -> Any: def _prepare_results(df: DataFrame, nlevenshtein: float) -> dict: + """Creates the list of results + + :param df: + :param nlevenshtein: + :return: + """ actual = np.array(df['actual'].values.tolist()) predicted = np.array(df['predicted'].values.tolist()) row = calculate_results_time_series_prediction(actual, predicted) @@ -130,12 +156,23 @@ def _prepare_results(df: DataFrame, nlevenshtein: float) -> dict: def _drop_columns(train_df: DataFrame, test_df: DataFrame) -> (DataFrame, DataFrame): + """Drops the column trace_id of given DataFrame + + :param training_df: + :param test_df: + :return: + """ train_df = train_df.drop(['trace_id', 'label'], 1) test_df = test_df.drop(['trace_id', 'label'], 1) return train_df, test_df def _choose_time_series_predictor(job: Job) -> TimeSeriesPredictorMixin: + """Chooses the time series predictor predictive method using the given job configuration + + :param job: + :return: + """ method, config = get_method_config(job) config.pop('time_series_prediction_method', None) logger.info("Using method {} with config {}".format(method, config)) diff --git a/src/split/serializers.py b/src/split/serializers.py index c04c20e..1a84dff 100644 --- a/src/split/serializers.py +++ b/src/split/serializers.py @@ -13,6 +13,11 @@ class SplitSerializer(serializers.ModelSerializer): training_log = serializers.SerializerMethodField() def get_training_log(self, split): + """Returns the training log using the given split + + :param split: + :return: + """ return split.train_log.pk if split.train_log is not None else None class Meta: diff --git a/src/split/splitting.py b/src/split/splitting.py index 709e1ed..4ef78a0 100644 --- a/src/split/splitting.py +++ b/src/split/splitting.py @@ -13,7 +13,11 @@ def get_train_test_log(split: Split): - """Returns training_log and test_log""" + """Returns training_log and test_log + + :param split: + :return: + """ if split.type == SplitTypes.SPLIT_SINGLE.value and Split.objects.filter( type=SplitTypes.SPLIT_DOUBLE.value, original_log=split.original_log, @@ -63,6 +67,11 @@ def get_train_test_log(split: Split): def _split_single_log(split: Split): + """Returns training_log and test_log, create from log taken using the given split + + :param split: + :return: + """ log = get_log(split.original_log) logger.info("\t\tExecute single split ID {}, split_type {}, test_size {}".format(split.id, split.type, split.test_size)) if split.splitting_method == SplitOrderingMethods.SPLIT_TEMPORAL.value: @@ -78,14 +87,24 @@ def _split_single_log(split: Split): def _temporal_split(log: EventLog, test_size: float): - """sort log by first event timestamp to enforce temporal order""" + """sort log by first event timestamp to enforce temporal order + + :param log: + :param test_size: + :return: + """ log = sorted(log, key=functools.cmp_to_key(_compare_trace_starts)) training_log, test_log = train_test_split(log, test_size=test_size, shuffle=False) return training_log, test_log def _temporal_split_strict(log: EventLog, test_size: float): - """Includes only training traces where it's last event ends before the first in test trace""" + """Includes only training traces where it's last event ends before the first in test trace + + :param log: + :param test_size: + :return: + """ training_log, test_log = _temporal_split(log, test_size) test_first_time = _trace_event_time(test_log[0]) training_log = filter(lambda x: _trace_event_time(x, event_index=-1) < test_first_time, training_log) @@ -93,11 +112,23 @@ def _temporal_split_strict(log: EventLog, test_size: float): def _split_log(log: EventLog, test_size: float, random_state: Union[int, None] = 4, shuffle=True): + """Split a log in training_log and test_log + + :param item1: + :param item2: + :return: + """ training_log, test_log = train_test_split(log, test_size=test_size, random_state=random_state, shuffle=shuffle) return training_log, test_log def _compare_trace_starts(item1, item2): + """Compare 2 traces, taken using the given item + + :param item1: + :param item2: + :return: + """ first = _trace_event_time(item1) second = _trace_event_time(item2) if first < second: @@ -109,5 +140,10 @@ def _compare_trace_starts(item1, item2): def _trace_event_time(trace, event_index=0): - """Event time as Date. By default first event.""" + """Event time as Date. By default first event. + + :param trace: + :param event_index: + :return: + """ return trace[event_index]['time:timestamp'] diff --git a/src/utils/custom_parser.py b/src/utils/custom_parser.py index 981012e..3752195 100644 --- a/src/utils/custom_parser.py +++ b/src/utils/custom_parser.py @@ -10,5 +10,10 @@ class CustomXMLParser(BaseParser): def parse(self, stream, media_type=None, parser_context=None): """ Simply return a string representing the body of the request. + + :param stream: + :param media_type: + :param parser_context: + :return: """ return stream.read() diff --git a/src/utils/django_orm.py b/src/utils/django_orm.py index 1b303b9..60e1f55 100644 --- a/src/utils/django_orm.py +++ b/src/utils/django_orm.py @@ -4,6 +4,11 @@ def duplicate_orm_row(obj: models.Model): + """Returns a copy of the given object + + :param obj: + :return: + """ cloned = copy.deepcopy(obj) cloned.id = None cloned.pk = None diff --git a/src/utils/event_attributes.py b/src/utils/event_attributes.py index 14cfbce..402887b 100644 --- a/src/utils/event_attributes.py +++ b/src/utils/event_attributes.py @@ -7,6 +7,8 @@ def unique_events(log: EventLog): """List of unique events using event concept:name Adds all events into a list and removes duplicates while keeping order. + :param log: + :return: """ event_list = [event['concept:name'] for trace in log for event in trace] @@ -19,6 +21,9 @@ def unique_events2(training_log: EventLog, test_log: EventLog): Renamed to 2 because Python doesn't allow functions with same names. Python is objectively the worst language. + :param training_log: + :param test_log: + :return: """ event_list = unique_events(training_log) + unique_events(test_log) return sorted(set(event_list), key=lambda x: event_list.index(x)) @@ -28,6 +33,8 @@ def get_event_attributes(log: EventLog): """Get log event attributes that are not name or time As log file is a list, it has no global event attributes. Getting from first event of first trace. This may be bad. + :param log: + :return: """ event_attributes = [] for attribute in log[0][0]._dict.keys(): @@ -37,11 +44,21 @@ def get_event_attributes(log: EventLog): def get_additional_columns(log: EventLog): + """Returns additional columns + + :param log: + :return: + """ return {'trace_attributes': get_global_trace_attributes(log), 'event_attributes': get_global_event_attributes(log)} def get_global_trace_attributes(log: EventLog): + """Get log trace attributes that are not name or time + + :param log: + :return: + """ # retrieves all traces in the log and returns their intersection attributes = list(reduce(set.intersection, [set(trace._get_attributes().keys()) for trace in log])) trace_attributes = [attr for attr in attributes if attr not in ["concept:name", "time:timestamp", "label"]] @@ -50,6 +67,9 @@ def get_global_trace_attributes(log: EventLog): def get_global_event_attributes(log): """Get log event attributes that are not name or time + + :param log: + :return: """ # retrieves all events in the log and returns their intersection attributes = list(reduce(set.intersection, [set(event._dict.keys()) for trace in log for event in trace])) diff --git a/src/utils/experiments_utils.py b/src/utils/experiments_utils.py index 86e1cc1..b50698f 100644 --- a/src/utils/experiments_utils.py +++ b/src/utils/experiments_utils.py @@ -40,7 +40,20 @@ def create_classification_payload( }, incremental_train=[], model_hyperparameters={}): - + """ + Returns a default configuration to create a classification model + + :param split: + :param encodings: + :param encoding: + :param labeling: + :param clustering: + :param classification: + :param hyperparameter_optimization: + :param incremental_train: + :param model_hyperparameters: + :return: + """ config = { "clusterings": clustering, "labelling": labeling, @@ -86,6 +99,20 @@ def create_regression_payload( }, incremental_train=[], model_hyperparameters={}): + """ + Returns a default configuration to create a regression model + + :param split: + :param encodings: + :param encoding: + :param labeling: + :param clustering: + :param regression: + :param hyperparameter_optimization: + :param incremental_train: + :param model_hyperparameters: + :return: + """ config = { "clusterings": clustering, @@ -108,6 +135,14 @@ def upload_split( server_name="0.0.0.0", server_port='8000' ): + """Uploads train and test event_log + + :param train: + :param test: + :param server_name: + :param server_port: + :return: + """ r = requests.post( 'http://' + server_name + ':' + server_port + '/splits/multiple', files={'trainingSet': open(train, 'r+'), 'testSet': open(test, 'r+')} @@ -120,6 +155,13 @@ def send_job_request( server_name="0.0.0.0", server_port='8000' ): + """Sends to the server request to schedule a job using given the payload and returns the job id and status + + :param payload: + :param server_name: + :param server_port: + :return: + """ r = requests.post( 'http://' + server_name + ':' + server_port + '/jobs/multiple', json=payload, @@ -133,6 +175,13 @@ def retrieve_job( server_name="0.0.0.0", server_port='8000' ): + """Retrieves a job on the server using the given config and returns the job result + + :param config: + :param server_name: + :param server_port: + :return: + """ r = requests.get( 'http://' + server_name + ':' + server_port + '/jobs/', headers={'Content-type': 'application/json'}, diff --git a/src/utils/file_service.py b/src/utils/file_service.py index 1e49dbd..5ff6ac5 100644 --- a/src/utils/file_service.py +++ b/src/utils/file_service.py @@ -5,6 +5,11 @@ def create_unique_name(name: str) -> str: + """Returns a unique name, using the given name and the time + + :param name: + :return: + """ return name.replace('.', '_' + str(time.time()).replace('.', '') + '.') diff --git a/src/utils/log_metrics.py b/src/utils/log_metrics.py index 26d9656..0be4b7a 100644 --- a/src/utils/log_metrics.py +++ b/src/utils/log_metrics.py @@ -10,6 +10,7 @@ def events_by_date(log: EventLog) -> OrderedDict: """Creates dict of events by date ordered by date + :param log: :return {'2010-12-30': 7, '2011-01-06': 8} :rtype: OrderedDict """ @@ -27,6 +28,8 @@ def resources_by_date(log: EventLog) -> OrderedDict: Resource and timestamp delimited by &&. If this is in resources name, bad stuff will happen. Returns a dict with a date and the number of unique resources used on that day. + + :param log: :return {'2010-12-30': 7, '2011-01-06': 8} """ stamp_dict = defaultdict(lambda: []) @@ -45,6 +48,7 @@ def resources_by_date(log: EventLog) -> OrderedDict: def event_executions(log: EventLog) -> OrderedDict: """Creates dict of event execution count + :param log: :return {'Event A': 7, '2011-01-06': 8} """ executions = defaultdict(lambda: 0) @@ -57,6 +61,7 @@ def event_executions(log: EventLog) -> OrderedDict: def new_trace_start(log: EventLog) -> OrderedDict: """Creates dict of new traces by date + :param log: :return {'2010-12-30': 1, '2011-01-06': 2} """ executions = defaultdict(lambda: 0) @@ -70,6 +75,7 @@ def trace_attributes(log: EventLog) -> list: """Creates an array of dicts that describe trace attributes. Only looks at first trace. Filters out `concept:name`. + :param log: :return [{name: 'name', type: 'string', example: 34}] """ values = [] @@ -84,6 +90,11 @@ def trace_attributes(log: EventLog) -> list: def _is_number(s) -> str: + """Returns whether the parameter is a number or string + + :param s: + :return: + """ if (isinstance(s, (float, int)) or (s.isdigit() if hasattr(s, 'isdigit') else False)) and not isinstance(s, bool): return 'number' return 'string' @@ -92,6 +103,7 @@ def _is_number(s) -> str: def events_in_trace(log: EventLog) -> OrderedDict: """Creates dict of number of events in trace + :param log: :return {'4': 11, '3': 8} """ stamp_dict = defaultdict(lambda: 0) @@ -103,6 +115,7 @@ def events_in_trace(log: EventLog) -> OrderedDict: def max_events_in_log(log: EventLog) -> int: """Returns the maximum number of events in any trace + :param log: :return 3 """ return max([len(trace) for trace in log]) @@ -111,6 +124,7 @@ def max_events_in_log(log: EventLog) -> int: def avg_events_in_log(log: EventLog) -> int: """Returns the average number of events in any trace + :param log: :return 3 """ return statistics.mean([len(trace) for trace in log]) @@ -119,16 +133,27 @@ def avg_events_in_log(log: EventLog) -> int: def std_var_events_in_log(log: EventLog) -> int: """Returns the standard variation of the average number of events in any trace + :param log: :return 3 """ return statistics.stdev([len(trace) for trace in log]) def trace_ids_in_log(log: EventLog) -> list: + """Returns a list of trace's name classifier in the given log + + :param log: + :return: + """ return [trace.attributes[NAME_CLASSIFIER] for trace in log] def traces_in_log(log: EventLog) -> list: + """Returns a list of dict, of traces in the given log + + :param log: + :return: + """ return [{'attributes': trace.attributes, 'events': [event for event in trace]} for trace in log] diff --git a/src/utils/result_metrics.py b/src/utils/result_metrics.py index e75ae29..58d3c45 100644 --- a/src/utils/result_metrics.py +++ b/src/utils/result_metrics.py @@ -11,6 +11,12 @@ def calculate_results_classification(actual: list, predicted: list) -> dict: + """Returns the results of classification predictive model + + :param actual: + :param predicted: + :return: + """ return {**{'f1score': _get_f1(actual, predicted), 'acc': accuracy_score(actual, predicted), 'precision': _get_precision(actual, predicted), @@ -19,10 +25,22 @@ def calculate_results_classification(actual: list, predicted: list) -> dict: def calculate_results_time_series_prediction(actual: ndarray, predicted: ndarray) -> dict: + """Returns the results of classification predictive model + + :param actual: + :param predicted: + :return: + """ return {} def get_confusion_matrix(actual, predicted) -> dict: + """Returns the confusion matrix of model + + :param actual: + :param predicted: + :return: + """ true_negatives, false_positives, false_negatives, true_positives = '--', '--', '--', '--' actual_set = list(sorted(set(actual))) if len(actual_set) <= 2: @@ -41,6 +59,12 @@ def get_confusion_matrix(actual, predicted) -> dict: def _get_f1(actual, predicted) -> float: + """Returns the f1_score of model + + :param actual: + :param predicted: + :return: + """ try: f1score = f1_score(actual, predicted, average='macro') except ZeroDivisionError: @@ -49,6 +73,12 @@ def _get_f1(actual, predicted) -> float: def _get_recall(actual, predicted) -> float: + """Returns the recall of the model + + :param actual: + :param predicted: + :return: + """ try: recall = recall_score(actual, predicted, average='macro') except ZeroDivisionError: @@ -57,6 +87,12 @@ def _get_recall(actual, predicted) -> float: def _get_precision(actual, predicted) -> float: + """Returns the precision of model + + :param actual: + :param predicted: + :return: + """ try: precision = precision_score(actual, predicted, average='macro') except ZeroDivisionError: @@ -65,6 +101,12 @@ def _get_precision(actual, predicted) -> float: def calculate_nlevenshtein(actual: ndarray, predicted: ndarray) -> float: + """Returns the nlevenshtein of model + + :param actual: + :param predicted: + :return: + """ distances = [] for row in range(actual.shape[0]): @@ -74,6 +116,12 @@ def calculate_nlevenshtein(actual: ndarray, predicted: ndarray) -> float: def get_auc(actual, scores) -> float: + """Returns the roc_auc_score of model + + :param actual: + :param scores: + :return: + """ try: auc = roc_auc_score(actual, scores) except ValueError: @@ -82,6 +130,13 @@ def get_auc(actual, scores) -> float: def calculate_auc(actual, scores, auc: int) -> float: + """Calculate and returns the roc_auc_score of model + + :param actual: + :param scores: + :param auc: + :return: + """ if scores.shape[1] == 1: auc += 0 else: @@ -93,6 +148,12 @@ def calculate_auc(actual, scores, auc: int) -> float: def _prepare_results(input_df: DataFrame, label: Labelling) -> dict: + """Prepares the result + + :param input_df: + :param label: + :return: + """ if label.type == LabelTypes.REMAINING_TIME.value: # TODO is the remaining time in seconds or hours? input_df['label'] = input_df['label'] / 3600 @@ -107,6 +168,12 @@ def _prepare_results(input_df: DataFrame, label: Labelling) -> dict: def _mean_absolute_percentage_error(y_true, y_pred): + """Calculates and returns the mean absolute percentage error + + :param y_true: + :param y_pred: + :return: + """ y_true, y_pred = np.array(y_true), np.array(y_pred) if 0 in y_true: return -1 diff --git a/src/utils/tests_utils.py b/src/utils/tests_utils.py index 93e490f..0e63ef1 100644 --- a/src/utils/tests_utils.py +++ b/src/utils/tests_utils.py @@ -26,6 +26,13 @@ def create_test_log(log_name: str = 'general_example.xes', log_path: str = 'cache/log_cache/test_logs/general_example.xes') -> Log: + """ + Gets or creates a new log + + :param log_name: + :param log_path: + :return: + """ log = Log.objects.get_or_create(name=log_name, path=log_path)[0] return log @@ -36,6 +43,17 @@ def create_test_split(split_type: str = SplitTypes.SPLIT_SINGLE.value, original_log: Log = None, train_log: Log = None, test_log: Log = None): + """ + Gets or creates a test split + + :param split_type: + :param split_ordering_method: + :param test_size: + :param original_log: + :param train_log: + :param test_log: + :return: + """ if split_type == SplitTypes.SPLIT_SINGLE.value: if original_log is None: original_log = create_test_log() @@ -66,6 +84,21 @@ def create_test_encoding(prefix_length: int = 1, add_executed_events: bool = False, task_generation_type: str = TaskGenerationTypes.ONLY_THIS.value, data_encoding: str = DataEncodings.LABEL_ENCODER.value) -> Encoding: + """ + Gets or creates the test encoding + + :param prefix_length: + :param padding: + :param value_encoding: + :param add_elapsed_time: + :param add_remaining_time: + :param add_resources_used: + :param add_new_traces: + :param add_executed_events: + :param task_generation_type: + :param data_encoding: + :return: + """ encoding = Encoding.objects.get_or_create( data_encoding=data_encoding, value_encoding=value_encoding, @@ -84,6 +117,15 @@ def create_test_labelling(label_type: str = LabelTypes.NEXT_ACTIVITY.value, attribute_name: str = None, threshold_type: str = ThresholdTypes.THRESHOLD_MEAN.value, threshold: float = 0.0) -> Labelling: + """ + Gets or creates test labelling + + :param label_type: + :param attribute_name: + :param threshold: + :param threshold_type: + :return: + """ labelling = Labelling.objects.get_or_create( type=label_type, attribute_name=attribute_name, @@ -95,6 +137,13 @@ def create_test_labelling(label_type: str = LabelTypes.NEXT_ACTIVITY.value, def create_test_clustering(clustering_type: str = ClusteringMethods.NO_CLUSTER.value, configuration: dict = {}) -> Clustering: + """ + Gets or creates the test clustering + + :param clustering_type: + :param configuration: + :return: + """ clustering = Clustering.init(clustering_type, configuration) return clustering @@ -102,6 +151,14 @@ def create_test_clustering(clustering_type: str = ClusteringMethods.NO_CLUSTER.v def create_test_predictive_model(predictive_model: str = PredictiveModels.CLASSIFICATION.value, prediction_method: str = ClassificationMethods.RANDOM_FOREST.value, configuration: dict = {}) -> PredictiveModel: + """ + Gets or creates the test predictive model + + :param predictive_model: + :param prediction_method: + :param configuration: + :return: + """ pred_model = PredictiveModel.init(get_prediction_method_config(predictive_model, prediction_method, configuration)) return pred_model @@ -109,6 +166,14 @@ def create_test_predictive_model(predictive_model: str = PredictiveModels.CLASSI def create_test_hyperparameter_optimizer(hyperoptim_type: str = HyperparameterOptimizationMethods.HYPEROPT.value, performance_metric: HyperOptLosses = HyperOptLosses.ACC.value, max_evals: int = 10): + """ + Gets or creates the test hyperparemeter optimizer + + :param hyperoptim_type: + :param performance_metric: + :param max_evals: + :return: + """ hyperparameter_optimization = HyperparameterOptimization.init({'type': hyperoptim_type, 'performance_metric': performance_metric, 'max_evals': max_evals}) @@ -124,6 +189,20 @@ def create_test_job(split: Split = None, job_type=JobTypes.PREDICTION.value, hyperparameter_optimizer: HyperparameterOptimization = None, incremental_train : Job = None): + """ + Gets or creates the test job + + :param split: + :param encoding: + :param labelling: + :param clustering: + :param create_models: + :param predictive_model: + :param job_type: + :param hyperparameter_optimizer: + :param incremental_train: + :return: + """ job, _ = Job.objects.get_or_create( status=JobStatuses.CREATED.value, type=job_type, diff --git a/src/utils/time_metrics.py b/src/utils/time_metrics.py index 8f69a96..5eea13e 100644 --- a/src/utils/time_metrics.py +++ b/src/utils/time_metrics.py @@ -4,12 +4,21 @@ def duration(trace): - """Calculate the duration of a trace""" + """Calculate the duration of a trace + + :param trace: + :return: + """ return remaining_time_id(trace, 0) def elapsed_time_id(trace, event_index: int): - """Calculate elapsed time by event index in trace""" + """Calculate elapsed time by event index in trace + + :param trace: + :param event_index: + :return: + """ try: event = trace[event_index] except IndexError: @@ -20,7 +29,12 @@ def elapsed_time_id(trace, event_index: int): def elapsed_time(trace, event): - """Calculate elapsed time by event in trace""" + """Calculate elapsed time by event in trace + + :param trace: + :param event: + :return: + """ # FIXME using no timezone info for calculation event_time = event['time:timestamp'].strftime("%Y-%m-%dT%H:%M:%S") first_time = trace[0]['time:timestamp'].strftime("%Y-%m-%dT%H:%M:%S") @@ -33,7 +47,12 @@ def elapsed_time(trace, event): def remaining_time_id(trace, event_index: int): - """Calculate remaining time by event index in trace""" + """Calculate remaining time by event index in trace + + :param trace: + :param event_index: + :return: + """ try: event = trace[event_index] return remaining_time(trace, event) @@ -44,7 +63,12 @@ def remaining_time_id(trace, event_index: int): def remaining_time(trace, event): - """Calculate remaining time by event in trace""" + """Calculate remaining time by event in trace + + :param trace: + :param event: + :return: + """ # FIXME using no timezone info for calculation event_time = event['time:timestamp'].strftime("%Y-%m-%dT%H:%M:%S") last_time = trace[-1]['time:timestamp'].strftime("%Y-%m-%dT%H:%M:%S") @@ -58,9 +82,11 @@ def remaining_time(trace, event): def count_on_event_day(trace, date_dict: dict, event_id): """Finds the date of event and returns the value from date_dict + :param date_dict one of the dicts from log_metrics.py :param event_id Event id :param trace Log trace + :return: """ try: event = trace[event_id]