ToxicCommentProject/clean_data_jd_td.py at master · TheoDlmz/ToxicCommentProject · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
import collections
import numpy as np
import pandas as pd
import re

##
from argparse import Namespace

args = Namespace(
    path_csv="C:\\Users\\Theo Delemazure\\Documents\\GitHub\\ToxicCommentProject\\data\\unclean\\",
    train_proportion=0.7,
    val_proportion=0.2,
    test_proportion=0.1,
    path_out="C:\\Users\\Theo Delemazure\\Documents\\GitHub\\ToxicCommentProject\\data\\clean\\",
    seed=1337
)
##


def clean_data(input_path):
    data = pd.read_csv(input_path + 'train.csv')

    # Remove return to line
    data['comment_text'] = data['comment_text'].map(
        lambda x: re.sub('\\n', ' ', str(x)))

    # Remove any text starting with user
    data['comment_text'] = data['comment_text'].map(
        lambda x: re.sub('\[\[User.*', '', str(x)))

    # convert to lowercase
    data['comment_text'] = data['comment_text'].map(lambda x: str(x).lower())

    # remove http links in the text
    data['comment_text'] = data['comment_text'].map(
        lambda x: re.sub("(http://.*?\s)|(http://.*)", '', str(x)))

    # Flag empty comments with na and remove from data
    data['comment_text'].replace('', np.nan, inplace=True)
    data.dropna(inplace=True)

    # Other
    data['comment_text'] = data['comment_text'].map(
        lambda x: re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", str(x)))
    data['comment_text'] = data['comment_text'].map(
        lambda x: re.sub(r"\'s", " \'s", str(x)))
    data['comment_text'] = data['comment_text'].map(
        lambda x: re.sub(r"\'m", " \'m", str(x)))
    data['comment_text'] = data['comment_text'].map(
        lambda x: re.sub(r"\'ve", " \'ve", str(x)))
    data['comment_text'] = data['comment_text'].map(
        lambda x: re.sub(r"n\'t", " n\'t", str(x)))
    data['comment_text'] = data['comment_text'].map(
        lambda x: re.sub(r"\'re", " \'re", str(x)))
    data['comment_text'] = data['comment_text'].map(
        lambda x: re.sub(r"\'d", " \'d", str(x)))
    data['comment_text'] = data['comment_text'].map(
        lambda x: re.sub(r"\'ll", " \'ll", str(x)))
    data['comment_text'] = data['comment_text'].map(
        lambda x: re.sub(r",", " , ", str(x)))
    data['comment_text'] = data['comment_text'].map(
        lambda x: re.sub(r"!", " ! ", str(x)))
    data['comment_text'] = data['comment_text'].map(
        lambda x: re.sub(r"\(", " ( ", str(x)))
    data['comment_text'] = data['comment_text'].map(
        lambda x: re.sub(r"\)", " ) ", str(x)))
    data['comment_text'] = data['comment_text'].map(
        lambda x: re.sub(r"\?", " ? ", str(x)))
    data['comment_text'] = data['comment_text'].map(
        lambda x: re.sub(r"\s{2,}", " ", str(x)))
    data['comment_text'] = data['comment_text'].map(
        lambda x: x.replace('"',""))
    data['comment_text'] = data['comment_text'].map(
        lambda x: x.strip('\"'))
    data['comment_text'] = data['comment_text'].map(
        lambda x: re.sub('"', '', str(x)))

    data = data.drop('id',1)

    COMMENT = 'comment_text'
    LABELS = [
        'toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate'
    ]

    data['clean'] = (data['toxic'] + data['severe_toxic']+data['obscene']+data['threat'] +data['insult']+data['identity_hate']) == 0
    '''
    comments = data[[COMMENT]]
    labels = data[LABELS]

    data['extract'] = data[LABELS].astype('str').agg(' '.join, axis=1)
    data['extract'] = data['extract'] + '\t' + data['comment_text']

    data['extract'].to_csv(input_path + '/train_edited.csv',
                           header=False,
                           index=False)

    data['overall'] = sum(data[LABELS])
    assert (len(comments)) == len(labels)
    comments.to_csv(input_path + '/train_comments.csv', header=True)
    labels.to_csv(input_path + '/train_labels.csv')
    '''
    by_toxicity = collections.defaultdict(list)
    for _, row in data.iterrows():
        by_toxicity[row.clean].append(row.to_dict())


    # Create split data
    final_list = []
    np.random.seed(args.seed)

    for _, item_list in sorted(by_toxicity.items()):

        np.random.shuffle(item_list)

        n_total = len(item_list)
        n_train = int(args.train_proportion * n_total)
        n_val = int(args.val_proportion * n_total)
        n_test = int(args.test_proportion * n_total)

        # Give data point a split attribute
        for item in item_list[:n_train]:
            item['split'] = 'train'

        for item in item_list[n_train:n_train+n_val]:
            item['split'] = 'val'

        for item in item_list[n_train+n_val:n_train+n_val+n_test]:
            item['split'] = 'test'

        # Add to final list
        final_list.extend(item_list)

    data = pd.DataFrame(final_list)
    data.to_csv(args.path_out+"toxic_comments.csv", index=False)
    return data


data = clean_data(args.path_csv)