ToxicCommentProject/data_cleaning.py at master · TheoDlmz/ToxicCommentProject · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
import pandas as pd
import numpy as np

import re
##

data = pd.read_csv('C:\\Users\\Theo Delemazure\\Documents\\GitHub\\ToxicCommentProject\\data\\unclean\\train.csv')

##

data['comment_text'] = data['comment_text'].map(lambda x: re.sub('\\n',' ',str(x)))

    # remove any text starting with User...
data['comment_text'] = data['comment_text'].map(lambda x: re.sub("\[\[User.*",'',str(x)))

    # remove IP addresses or user IDs
data['comment_text'] = data['comment_text'].map(lambda x: re.sub("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}",'',str(x)))

    # lower uppercase letters
data['comment_text'] = data['comment_text'].map(lambda x: str(x).lower())

    #remove http links in the text
data['comment_text'] = data['comment_text'].map(lambda x: re.sub("(http://.*?\s)|(http://.*)",'',str(x)))

    #remove all punctuation except for apostrophe (')
data['comment_text'] = data['comment_text'].map(lambda x: re.sub('["#$%&\()*+,-.!?/:;<=>@[\\]^_`{|}~]','',str(x)))


##

data['comment_text'].replace('', np.nan, inplace=True)

##

data.dropna(inplace=True)

##

COMMENT = 'comment_text'
LABELS = ['toxic','severe_toxic','obscene','threat','insult','identity_hate']
comments = data[[COMMENT]]
labels = data[LABELS]

assert(len(comments) == len(labels))

comments.to_csv('C:\\Users\\Theo Delemazure\\Documents\\GitHub\\ToxicCommentProject\\data\\clean\\comments.csv')
labels.to_csv('C:\\Users\\Theo Delemazure\\Documents\\GitHub\\ToxicCommentProject\\data\\clean\\labels.csv')

##
'''
n = len(comments)
n_train =int(2*n/3)
a = np.arange(n)
np.random.shuffle(a)
train_index,test_index = np.split(a,[n_train])
train_comments =  pd.DataFrame(comments.to_numpy()[train_index])
test_comments = pd.DataFrame(comments.to_numpy()[test_index])
train_labels = pd.DataFrame(labels.to_numpy()[train_index])
test_labels = pd.DataFrame(labels.to_numpy()[test_index])

##


test_comments.to_csv('C:\\Users\\Theo Delemazure\\Documents\\GitHub\\ToxicCommentProject\\data\\clean\\test_comments.csv')

train_comments.to_csv('C:\\Users\\Theo Delemazure\\Documents\\GitHub\\ToxicCommentProject\\data\\clean\\train_comments.csv')

test_labels.to_csv('C:\\Users\\Theo Delemazure\\Documents\\GitHub\\ToxicCommentProject\\data\\clean\\test_labels.csv')
train_labels.to_csv('C:\\Users\\Theo Delemazure\\Documents\\GitHub\\ToxicCommentProject\\data\\clean\\train_labels.csv')

'''