progressiveTrainCodeSwitch/load_data.py at main · s1998/progressiveTrainCodeSwitch · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
import csv
import numpy as np
import os
import re
import itertools
from collections import Counter
from os.path import join
from nltk import tokenize
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from base_code.utils import *

def read_file(data_dir, with_evaluation, lbl_to_id_chng=True):
    data = []
    target = []

    if data_dir in ["sail", "enes", "taen", ]:
        train_data = read_examples_from_file("./data" + data_dir, "train")
        valid_data = read_examples_from_file("./data" + data_dir, "validation")
        labels = ["negative", "positive"]
        lbl_to_id = {lbl:i for i, lbl in enumerate(labels)}

        for lin in train_data:
            if lin["label"] == "neutral":
                continue
            data.append(lin["text"])
            if lbl_to_id_chng:
                target.append(lbl_to_id[lin["label"]])
            else:
                target.append(lin["label"])

        for lin in valid_data:
            if lin["label"] == "neutral":
                continue
            data.append(lin["text"])
            if lbl_to_id_chng:
                target.append(lbl_to_id[lin["label"]])
            else:
                target.append(lin["label"])
    else:
        raise NotImplementedError("Data {} not found .... ".format(data_dir))

    if with_evaluation:
        y = np.asarray(target)
        assert len(data) == len(y)
        if lbl_to_id_chng:
            assert set(range(len(np.unique(y)))) == set(np.unique(y))
    else:
        y = None

    return data, y

def clean_str(string):
    string = re.sub(r"[^A-Za-z\u0900-\u097F0-9(),.!?_\"\'\`]", " ", string)
    string = re.sub(r"\'s", " \'s", string)
    string = re.sub(r"\"", " \" ", string)
    string = re.sub(r"\'ve", " \'ve", string)
    string = re.sub(r"n\'t", " n\'t", string)
    string = re.sub(r"\'m", " \'m", string)
    string = re.sub(r"\'re", " \'re", string)
    string = re.sub(r"\'d", " \'d", string)
    string = re.sub(r"\'ll", " \'ll", string)
    string = re.sub(r",", " , ", string)
    string = re.sub(r"\.", " . ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\$", " $ ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)
    return string.strip().lower()

def preprocess_doc(data):
    data = [s.strip() for s in data]
    data = [clean_str(s) for s in data]
    return data