MachineLearning/set_extraction.py at main · francescapoli98/MachineLearning · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
# -*- coding: utf-8 -*-
"""set_extraction.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/12wuVmBjqfv5XRWVePY1KQLFbjTdfG99m

# Libraries
"""


import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import StandardScaler

"""# Data Loading

"""

cols = ["ID", "INPUT0", "INPUT1", "INPUT2", "INPUT3", "INPUT4", "INPUT5", "INPUT6", "INPUT7", "INPUT8", "TARGET_X", "TARGET_Y"]

dtf = pd.read_csv('data/ML-CUP22-TR.csv', skiprows=7, header=None, sep=",", names=cols)

dtf.head(3)

dtf.set_index("ID", inplace=True)

dtf.shape

dtf.info()

"""# Check for correlations

"""

dtf.corr()

fig, ax = plt.subplots(figsize=(10,10))
sns.heatmap(dtf.corr(), annot=True)
plt.show()

"""# Development Set (80%) and Test Set (20%) Extraction

"""

x = dtf.iloc[:,0:9]
y = dtf.iloc[:,9:11]

x.head()

y.head()

# data normalization

scaler = StandardScaler()

cols = x.columns
x = pd.DataFrame(scaler.fit_transform(x.values), columns=cols)  # now x values are scaled (not targets)

x.head()

y.head()

x_arr = x.to_numpy().astype(np.float64)
y_arr = y.to_numpy().astype(np.float64)

x_train, x_test, y_train, y_test = train_test_split(x_arr, y_arr, test_size=0.3, random_state=42, shuffle=True)


print("x_train ", x_train.shape)
print("y_train ", y_train.shape)
print("x_test ", x_test.shape)
print("y_test ", y_test.shape)