From a5831555d9f5b4d801e6db15262799600a4e16da Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Wed, 2 Oct 2019 06:15:50 +0900 Subject: [PATCH 01/37] # First commit --- sm/data_loader.py | 16 +++++++++++++ sm/model/ops.py | 27 ++++++++++++++++++++++ sm/train.py | 59 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+) create mode 100644 sm/data_loader.py create mode 100644 sm/model/ops.py create mode 100644 sm/train.py diff --git a/sm/data_loader.py b/sm/data_loader.py new file mode 100644 index 0000000..4877491 --- /dev/null +++ b/sm/data_loader.py @@ -0,0 +1,16 @@ +import torch +import torchvision +import torchvision.transforms as transforms + + +def data_loader(): + transform = transforms.Compose( + [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) + train_set = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform) + train_loader = torch.utils.data.DataLoader(train_set, batch_size=4, shuffle=True, num_workers=2) + test_set = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform) + test_loader = torch.utils.data.DataLoader(test_set, batch_size=4, shuffle=False, num_workers=2) + + classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') + + return train_loader, test_loader, classes diff --git a/sm/model/ops.py b/sm/model/ops.py new file mode 100644 index 0000000..9385603 --- /dev/null +++ b/sm/model/ops.py @@ -0,0 +1,27 @@ +import torch.nn as nn +import torch.nn.functional as F + +# args +IMAGE_WIDTH = 32 +IMAGE_HEIGHT = 32 +COLOR_CHANNELS = 3 +BATCH_SIZE = 32 + + +class MLP(nn.Module): + def __init__(self): + super(MLP, self).__init__() + self.activation = nn.ReLU() + self.fc_1 = nn.Linear(IMAGE_WIDTH * IMAGE_HEIGHT * COLOR_CHANNELS, 120) + self.dropout = nn.Dropout() + self.fc_2 = nn.Linear(120, 120) + self.dropout = nn.Dropout() + self.out = nn.Linear(120, 10) + + def forward(self, x): + x = x.view(-1, IMAGE_WIDTH * IMAGE_HEIGHT * COLOR_CHANNELS) + x = self.activation(self.fc_1(x)) + x = self.dropout(x) + x = self.activation(self.fc_2(x)) + x = self.dropout(x) + return F.log_softmax(self.out(x)) diff --git a/sm/train.py b/sm/train.py new file mode 100644 index 0000000..3cc0273 --- /dev/null +++ b/sm/train.py @@ -0,0 +1,59 @@ +import torch +import torch.nn as nn +import torch.optim as optim +from sm.data_loader import data_loader +from tqdm import tqdm +import argparse +from sm.model.ops import MLP + + +def train(args, model, device, train_loader, optimizer, epoch): + model.train() + for i, (data, target) in tqdm(enumerate(train_loader, 0)): + optimizer.zero_grad() + output = model(data) + criterion = nn.CrossEntropyLoss() + loss = criterion(output, target) + loss.backward() + optimizer.step() + if i % args.log_interval == 0: + print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + epoch, i * len(data), len(train_loader.dataset), + 100. * i / len(train_loader), loss.item())) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--batch-size', type=int, default=64, metavar='N', + help='input batch size for training (default: 64)') + parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', + help='input batch size for testing (default: 1000)') + parser.add_argument('--epochs', type=int, default=1, metavar='N', + help='number of epochs to train (default: 10)') + parser.add_argument('--lr', type=float, default=0.01, metavar='LR', + help='learning rate (default: 0.01)') + parser.add_argument('--momentum', type=float, default=0.5, metavar='M', + help='SGD momentum (default: 0.5)') + parser.add_argument('--seed', type=int, default=1, metavar='S', + help='random seed (default: 1)') + parser.add_argument('--log-interval', type=int, default=10, metavar='N', + help='how many batches to wait before logging training status') + + parser.add_argument('--save_model', action='store_true', default=False, + help='For Saving the current Model') + args = parser.parse_args() + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + train_loader = data_loader()[0] + + model = MLP() + optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) + + for epoch in range(1, args.epochs + 1): + train(args, model, device, train_loader, optimizer, epoch) + + if (args.save_model): + torch.save(model.state_dict(), "CIFAR-10_MLP.pt") + + +if __name__ == "__main__": + main() \ No newline at end of file From fcb85082df5194c94d98b7b042deb7288e31f2f2 Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Thu, 3 Oct 2019 03:39:52 +0900 Subject: [PATCH 02/37] CNN Model --- sm/model/ops.py | 20 ++++++++++++++++++++ sm/train.py | 4 ++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/sm/model/ops.py b/sm/model/ops.py index 9385603..88dfb78 100644 --- a/sm/model/ops.py +++ b/sm/model/ops.py @@ -25,3 +25,23 @@ def forward(self, x): x = self.activation(self.fc_2(x)) x = self.dropout(x) return F.log_softmax(self.out(x)) + + +class CNN(nn.Module): + def __init__(self): + super(CNN, self).__init__() + self.conv1 = nn.Conv2d(3, 6, 5) + self.pool = nn.MaxPool2d(2, 2) + self.conv2 = nn.Conv2d(6, 16, 5) + self.fc1 = nn.Linear(16 * 5 * 5, 120) + self.fc2 = nn.Linear(120, 84) + self.fc3 = nn.Linear(84, 10) + + def forward(self, x): + x = self.pool(F.relu(self.conv1(x))) + x = self.pool(F.relu(self.conv2(x))) + x = x.view(-1, 16 * 5 * 5) + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x \ No newline at end of file diff --git a/sm/train.py b/sm/train.py index 3cc0273..a8618f7 100644 --- a/sm/train.py +++ b/sm/train.py @@ -4,7 +4,7 @@ from sm.data_loader import data_loader from tqdm import tqdm import argparse -from sm.model.ops import MLP +from sm.model.ops import MLP, CNN def train(args, model, device, train_loader, optimizer, epoch): @@ -45,7 +45,7 @@ def main(): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") train_loader = data_loader()[0] - model = MLP() + model = CNN() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) for epoch in range(1, args.epochs + 1): From 3364dc625e5e527a99c06d3fb4629bdb7c9a25c4 Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Thu, 3 Oct 2019 06:10:35 +0900 Subject: [PATCH 03/37] add argparse --- sm/model/ops.py | 3 ++- sm/train.py | 11 ++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/sm/model/ops.py b/sm/model/ops.py index 88dfb78..d28b7e1 100644 --- a/sm/model/ops.py +++ b/sm/model/ops.py @@ -1,5 +1,6 @@ import torch.nn as nn import torch.nn.functional as F +from torchvision.models import * # args IMAGE_WIDTH = 32 @@ -44,4 +45,4 @@ def forward(self, x): x = F.relu(self.fc1(x)) x = F.relu(self.fc2(x)) x = self.fc3(x) - return x \ No newline at end of file + return x diff --git a/sm/train.py b/sm/train.py index a8618f7..b790cc3 100644 --- a/sm/train.py +++ b/sm/train.py @@ -1,10 +1,10 @@ import torch import torch.nn as nn import torch.optim as optim -from sm.data_loader import data_loader +from data_loader import data_loader from tqdm import tqdm import argparse -from sm.model.ops import MLP, CNN +from model.ops import MLP, CNN def train(args, model, device, train_loader, optimizer, epoch): @@ -24,6 +24,7 @@ def train(args, model, device, train_loader, optimizer, epoch): def main(): parser = argparse.ArgumentParser() + parser.add_argument('--model', type=str, default='MLP') parser.add_argument('--batch-size', type=int, default=64, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', @@ -45,7 +46,11 @@ def main(): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") train_loader = data_loader()[0] - model = CNN() + if args.model == 'CNN': + model = CNN() + if args.model == 'MLP': + model = MLP() + optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) for epoch in range(1, args.epochs + 1): From ea7a5e8a8831baa61a8d877598cb67dcfc7b7049 Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Thu, 3 Oct 2019 13:13:41 +0900 Subject: [PATCH 04/37] repository hierarchy reconstructed --- sm/data_loader.py | 16 ------------ sm/model/ops.py | 48 ----------------------------------- sm/train.py | 64 ----------------------------------------------- 3 files changed, 128 deletions(-) delete mode 100644 sm/data_loader.py delete mode 100644 sm/model/ops.py delete mode 100644 sm/train.py diff --git a/sm/data_loader.py b/sm/data_loader.py deleted file mode 100644 index 4877491..0000000 --- a/sm/data_loader.py +++ /dev/null @@ -1,16 +0,0 @@ -import torch -import torchvision -import torchvision.transforms as transforms - - -def data_loader(): - transform = transforms.Compose( - [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) - train_set = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform) - train_loader = torch.utils.data.DataLoader(train_set, batch_size=4, shuffle=True, num_workers=2) - test_set = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform) - test_loader = torch.utils.data.DataLoader(test_set, batch_size=4, shuffle=False, num_workers=2) - - classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') - - return train_loader, test_loader, classes diff --git a/sm/model/ops.py b/sm/model/ops.py deleted file mode 100644 index d28b7e1..0000000 --- a/sm/model/ops.py +++ /dev/null @@ -1,48 +0,0 @@ -import torch.nn as nn -import torch.nn.functional as F -from torchvision.models import * - -# args -IMAGE_WIDTH = 32 -IMAGE_HEIGHT = 32 -COLOR_CHANNELS = 3 -BATCH_SIZE = 32 - - -class MLP(nn.Module): - def __init__(self): - super(MLP, self).__init__() - self.activation = nn.ReLU() - self.fc_1 = nn.Linear(IMAGE_WIDTH * IMAGE_HEIGHT * COLOR_CHANNELS, 120) - self.dropout = nn.Dropout() - self.fc_2 = nn.Linear(120, 120) - self.dropout = nn.Dropout() - self.out = nn.Linear(120, 10) - - def forward(self, x): - x = x.view(-1, IMAGE_WIDTH * IMAGE_HEIGHT * COLOR_CHANNELS) - x = self.activation(self.fc_1(x)) - x = self.dropout(x) - x = self.activation(self.fc_2(x)) - x = self.dropout(x) - return F.log_softmax(self.out(x)) - - -class CNN(nn.Module): - def __init__(self): - super(CNN, self).__init__() - self.conv1 = nn.Conv2d(3, 6, 5) - self.pool = nn.MaxPool2d(2, 2) - self.conv2 = nn.Conv2d(6, 16, 5) - self.fc1 = nn.Linear(16 * 5 * 5, 120) - self.fc2 = nn.Linear(120, 84) - self.fc3 = nn.Linear(84, 10) - - def forward(self, x): - x = self.pool(F.relu(self.conv1(x))) - x = self.pool(F.relu(self.conv2(x))) - x = x.view(-1, 16 * 5 * 5) - x = F.relu(self.fc1(x)) - x = F.relu(self.fc2(x)) - x = self.fc3(x) - return x diff --git a/sm/train.py b/sm/train.py deleted file mode 100644 index b790cc3..0000000 --- a/sm/train.py +++ /dev/null @@ -1,64 +0,0 @@ -import torch -import torch.nn as nn -import torch.optim as optim -from data_loader import data_loader -from tqdm import tqdm -import argparse -from model.ops import MLP, CNN - - -def train(args, model, device, train_loader, optimizer, epoch): - model.train() - for i, (data, target) in tqdm(enumerate(train_loader, 0)): - optimizer.zero_grad() - output = model(data) - criterion = nn.CrossEntropyLoss() - loss = criterion(output, target) - loss.backward() - optimizer.step() - if i % args.log_interval == 0: - print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( - epoch, i * len(data), len(train_loader.dataset), - 100. * i / len(train_loader), loss.item())) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--model', type=str, default='MLP') - parser.add_argument('--batch-size', type=int, default=64, metavar='N', - help='input batch size for training (default: 64)') - parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', - help='input batch size for testing (default: 1000)') - parser.add_argument('--epochs', type=int, default=1, metavar='N', - help='number of epochs to train (default: 10)') - parser.add_argument('--lr', type=float, default=0.01, metavar='LR', - help='learning rate (default: 0.01)') - parser.add_argument('--momentum', type=float, default=0.5, metavar='M', - help='SGD momentum (default: 0.5)') - parser.add_argument('--seed', type=int, default=1, metavar='S', - help='random seed (default: 1)') - parser.add_argument('--log-interval', type=int, default=10, metavar='N', - help='how many batches to wait before logging training status') - - parser.add_argument('--save_model', action='store_true', default=False, - help='For Saving the current Model') - args = parser.parse_args() - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - train_loader = data_loader()[0] - - if args.model == 'CNN': - model = CNN() - if args.model == 'MLP': - model = MLP() - - optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) - - for epoch in range(1, args.epochs + 1): - train(args, model, device, train_loader, optimizer, epoch) - - if (args.save_model): - torch.save(model.state_dict(), "CIFAR-10_MLP.pt") - - -if __name__ == "__main__": - main() \ No newline at end of file From 8ecd21130b448173bf97a913e7c0556b65e42cf3 Mon Sep 17 00:00:00 2001 From: Martin Hwang Date: Sat, 5 Oct 2019 18:43:43 +0900 Subject: [PATCH 05/37] Re-construct hierarchy (#11) * Repository hierarchy reconstructed * Add binarylinear --- .gitignore | 1 + dataloader/__init__.py | 1 + {sm => dataloader}/data_loader.py | 0 models/__init__.py | 3 ++ models/binarized_mlp.py | 24 ++++++++++++++ models/conv.py | 21 +++++++++++++ models/mlp.py | 26 ++++++++++++++++ sm/model/ops.py | 48 ---------------------------- sm/train.py => train.py | 0 utils/BinaryLinear.py | 52 +++++++++++++++++++++++++++++++ utils/__init__.py | 1 + 11 files changed, 129 insertions(+), 48 deletions(-) create mode 100644 .gitignore create mode 100644 dataloader/__init__.py rename {sm => dataloader}/data_loader.py (100%) create mode 100644 models/__init__.py create mode 100644 models/binarized_mlp.py create mode 100644 models/conv.py create mode 100644 models/mlp.py delete mode 100644 sm/model/ops.py rename sm/train.py => train.py (100%) create mode 100644 utils/BinaryLinear.py create mode 100644 utils/__init__.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9f11b75 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea/ diff --git a/dataloader/__init__.py b/dataloader/__init__.py new file mode 100644 index 0000000..f4c9a23 --- /dev/null +++ b/dataloader/__init__.py @@ -0,0 +1 @@ +from dataloader import data_loader \ No newline at end of file diff --git a/sm/data_loader.py b/dataloader/data_loader.py similarity index 100% rename from sm/data_loader.py rename to dataloader/data_loader.py diff --git a/models/__init__.py b/models/__init__.py new file mode 100644 index 0000000..13bc530 --- /dev/null +++ b/models/__init__.py @@ -0,0 +1,3 @@ +from models import binarized_mlp +from models import mlp +from models import conv \ No newline at end of file diff --git a/models/binarized_mlp.py b/models/binarized_mlp.py new file mode 100644 index 0000000..f01bfaf --- /dev/null +++ b/models/binarized_mlp.py @@ -0,0 +1,24 @@ +import torch +import torch.nn as nn + +from utils.BinaryLinear import BinaryLinear + +class Net(nn.Module): + def __init__(self): + super(Net, self).__init__() + self.fc1 = BinaryLinear(28 * 28, 512, bias=False, mode="Stocastic") + self.batch1 = nn.BatchNorm1d(512) + self.fc2 = BinaryLinear(512, 512, bias=False, mode="Stocastic") + self.batch2 = nn.BatchNorm1d(512) + self.fc3 = BinaryLinear(512, 10, bias=False, mode="Stocastic") + self.dropout = nn.Dropout(0.2) + + def forward(self, x): + x = x.view(-1, 28 * 28) + x = torch.sigmoid(self.fc1(x)) + x = self.batch1(x) + x = torch.sigmoid(self.fc2(x)) + x = self.batch2(x) + x = self.dropout(x) + x = self.fc3(x) + return x diff --git a/models/conv.py b/models/conv.py new file mode 100644 index 0000000..9180dec --- /dev/null +++ b/models/conv.py @@ -0,0 +1,21 @@ +import torch.nn as nn +import torch.nn.functional as F + +class CNN(nn.Module): + def __init__(self): + super(CNN, self).__init__() + self.conv1 = nn.Conv2d(3, 6, 5) + self.pool = nn.MaxPool2d(2, 2) + self.conv2 = nn.Conv2d(6, 16, 5) + self.fc1 = nn.Linear(16 * 5 * 5, 120) + self.fc2 = nn.Linear(120, 84) + self.fc3 = nn.Linear(84, 10) + + def forward(self, x): + x = self.pool(F.relu(self.conv1(x))) + x = self.pool(F.relu(self.conv2(x))) + x = x.view(-1, 16 * 5 * 5) + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x diff --git a/models/mlp.py b/models/mlp.py new file mode 100644 index 0000000..e7c6be6 --- /dev/null +++ b/models/mlp.py @@ -0,0 +1,26 @@ +import torch.nn as nn +import torch.nn.functional as F + +class MLP(nn.Module): + def __init__(self, shape): + """ + + Args: + shape (tuple) + : structed as (image width, image height, image channels) + """ + super(MLP, self).__init__() + self.activation = nn.ReLU() + self.fc_1 = nn.Linear(shape[0] * shape[1] * shape[2], 120) + self.dropout = nn.Dropout() + self.fc_2 = nn.Linear(120, 120) + self.dropout = nn.Dropout() + self.out = nn.Linear(120, 10) + + def forward(self, x): + x = x.view(-1, x.shape[0] * x.shape[1] * x.shape[2]) + x = self.activation(self.fc_1(x)) + x = self.dropout(x) + x = self.activation(self.fc_2(x)) + x = self.dropout(x) + return F.log_softmax(self.out(x)) \ No newline at end of file diff --git a/sm/model/ops.py b/sm/model/ops.py deleted file mode 100644 index d28b7e1..0000000 --- a/sm/model/ops.py +++ /dev/null @@ -1,48 +0,0 @@ -import torch.nn as nn -import torch.nn.functional as F -from torchvision.models import * - -# args -IMAGE_WIDTH = 32 -IMAGE_HEIGHT = 32 -COLOR_CHANNELS = 3 -BATCH_SIZE = 32 - - -class MLP(nn.Module): - def __init__(self): - super(MLP, self).__init__() - self.activation = nn.ReLU() - self.fc_1 = nn.Linear(IMAGE_WIDTH * IMAGE_HEIGHT * COLOR_CHANNELS, 120) - self.dropout = nn.Dropout() - self.fc_2 = nn.Linear(120, 120) - self.dropout = nn.Dropout() - self.out = nn.Linear(120, 10) - - def forward(self, x): - x = x.view(-1, IMAGE_WIDTH * IMAGE_HEIGHT * COLOR_CHANNELS) - x = self.activation(self.fc_1(x)) - x = self.dropout(x) - x = self.activation(self.fc_2(x)) - x = self.dropout(x) - return F.log_softmax(self.out(x)) - - -class CNN(nn.Module): - def __init__(self): - super(CNN, self).__init__() - self.conv1 = nn.Conv2d(3, 6, 5) - self.pool = nn.MaxPool2d(2, 2) - self.conv2 = nn.Conv2d(6, 16, 5) - self.fc1 = nn.Linear(16 * 5 * 5, 120) - self.fc2 = nn.Linear(120, 84) - self.fc3 = nn.Linear(84, 10) - - def forward(self, x): - x = self.pool(F.relu(self.conv1(x))) - x = self.pool(F.relu(self.conv2(x))) - x = x.view(-1, 16 * 5 * 5) - x = F.relu(self.fc1(x)) - x = F.relu(self.fc2(x)) - x = self.fc3(x) - return x diff --git a/sm/train.py b/train.py similarity index 100% rename from sm/train.py rename to train.py diff --git a/utils/BinaryLinear.py b/utils/BinaryLinear.py new file mode 100644 index 0000000..79f7682 --- /dev/null +++ b/utils/BinaryLinear.py @@ -0,0 +1,52 @@ +import copy +import torch +import torch.nn.functional as F + + +class BinaryLinear(torch.nn.Linear): + + def __init__(self, in_features, out_features, bias=True, mode="Stocastic"): + super().__init__(in_features, out_features, bias) + self.mode = mode + self.bin_weight = self.weight_binarization(self.weight, self.mode) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + self.clipping_weight(self.weight) + self.bin_weight = self.weight_binarization(self.weight, self.mode) + return F.linear(input, self.bin_weight, self.bias) + + def weight_binarization(self, weight: torch.tensor, mode:str): + with torch.set_grad_enabled(False): + if mode == "Stocastic": + bin_weight = self.stocastic(weight) + elif mode == "Deterministic": + bin_weight = self.deterministic(weight) + else: + raise RuntimeError("{} is does not exist or not supported".format(mode)) + bin_weight.requires_grad = True + bin_weight.register_hook(self.cp_bin_grad_to_real_grad_hook) + return bin_weight + + @staticmethod + def deterministic(weight: torch.tensor) -> torch.tensor: + return weight.sign() + + @staticmethod + def stocastic(weight: torch.tensor) -> torch.tensor: + p = torch.sigmoid(weight) + uniform_matrix = torch.empty(p.shape).uniform_(0,1) + bin_weight = (p >= uniform_matrix).type(torch.float32) + bin_weight[bin_weight==0] = -1 + return bin_weight + + def cp_bin_grad_to_real_grad_hook(self, grad): + # it not using deepcopy + # you will be meet Error as + # `Can't detach views in-place. Use detach() instead` + self.weight.grad = copy.deepcopy(grad) + + def clipping_weight(self, weight:torch.tensor) -> torch.tensor: + with torch.set_grad_enabled(False): + weight = torch.clamp(weight, -1, 1) + weight.requires_grad = True + return weight diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..2bd7d76 --- /dev/null +++ b/utils/__init__.py @@ -0,0 +1 @@ +from utils.BinaryLinear import BinaryLinear \ No newline at end of file From 574e269be5f7ad914ba3780a789267a41de2e61b Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Wed, 9 Oct 2019 05:23:57 +0900 Subject: [PATCH 06/37] add ResNet50 --- models/resnet.py | 93 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 models/resnet.py diff --git a/models/resnet.py b/models/resnet.py new file mode 100644 index 0000000..073da1f --- /dev/null +++ b/models/resnet.py @@ -0,0 +1,93 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from torchvision.models import ResNet + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, in_planes, planes, stride=1): + super(BasicBlock, self).__init__() + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion * planes) + ) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.bn2(self.conv2(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, in_planes, planes, stride=1): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, self.expansion * planes, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(self.expansion * planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion * planes) + ) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = F.relu(self.bn2(self.conv2(out))) + out = self.bn3(self.conv3(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class ResNet(nn.Module): + def __init__(self, block, num_blocks, num_classes=10): + super(ResNet, self).__init__() + self.in_planes = 64 + self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) + self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) + self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) + self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) + self.linear = nn.Linear(512 * block.expansion, num_classes) + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1] * (num_blocks - 1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.layer1(out) + out = self.layer2(out) + out = self.layer3(out) + out = self.layer4(out) + out = F.avg_pool2d(out, 4) + out = out.view(out.size(0), -1) + out = self.linear(out) + return out + + +def ResNet50(): + return ResNet(Bottleneck, [3, 4, 6, 3]) From 943e85f87acb3903801106e7ee0064cade0f3787 Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Wed, 9 Oct 2019 05:24:47 +0900 Subject: [PATCH 07/37] Use GPU --- train.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/train.py b/train.py index b790cc3..a6f8f13 100644 --- a/train.py +++ b/train.py @@ -1,15 +1,19 @@ import torch import torch.nn as nn import torch.optim as optim -from data_loader import data_loader +from dataloader.data_loader import data_loader from tqdm import tqdm import argparse -from model.ops import MLP, CNN +from models.mlp import MLP +from models.conv import CNN +from models.resnet import ResNet50 def train(args, model, device, train_loader, optimizer, epoch): model.train() for i, (data, target) in tqdm(enumerate(train_loader, 0)): + data = data.to(device) + target = target.to(device) optimizer.zero_grad() output = model(data) criterion = nn.CrossEntropyLoss() @@ -24,7 +28,7 @@ def train(args, model, device, train_loader, optimizer, epoch): def main(): parser = argparse.ArgumentParser() - parser.add_argument('--model', type=str, default='MLP') + parser.add_argument('--model', type=str, default='ResNet50') parser.add_argument('--batch-size', type=int, default=64, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', @@ -42,22 +46,31 @@ def main(): parser.add_argument('--save_model', action='store_true', default=False, help='For Saving the current Model') + parser.add_argument('--cuda', action='store_true', + help='use CUDA') args = parser.parse_args() - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + if torch.cuda.is_available(): + if not args.cuda: + print("WARNING: You have a CUDA device, so you should probably run with --cuda") + + device = torch.device("cuda" if args.cuda else "cpu") train_loader = data_loader()[0] - if args.model == 'CNN': - model = CNN() - if args.model == 'MLP': - model = MLP() + # if args.model == 'CNN': + # model = CNN() + # if args.model == 'MLP': + # model = MLP() + model = ResNet50() + model.to(device) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) - for epoch in range(1, args.epochs + 1): + for epoch in tqdm(range(1, args.epochs + 1)): train(args, model, device, train_loader, optimizer, epoch) if (args.save_model): - torch.save(model.state_dict(), "CIFAR-10_MLP.pt") + torch.save(model.state_dict(), "CIFAR-10_ResNet50.pt") if __name__ == "__main__": From 28a7fc1ab802944da51cd436b93cb5a025adb06d Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Wed, 9 Oct 2019 05:25:50 +0900 Subject: [PATCH 08/37] test --- test.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 test.py diff --git a/test.py b/test.py new file mode 100644 index 0000000..6f72b6a --- /dev/null +++ b/test.py @@ -0,0 +1,44 @@ +import torch +from models.resnet import ResNet50 +from dataloader.data_loader import data_loader + + +if __name__ == "__main__": + classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') + + test_loader = data_loader()[1] + correct = 0 + total = 0 + model = ResNet50() + model.load_state_dict(torch.load('CIFAR-10_ResNet50.pt')) + + model.eval() + with torch.no_grad(): + for data in test_loader: + images, labels = data + outputs = model(images) + _, predicted = torch.max(outputs.data, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + print('Accuracy of the network on the 10000 test images: %d %%' % (100 * correct / total)) + + # Class correct + class_correct = list(0. for i in range(10)) + class_total = list(0. for i in range(10)) + with torch.no_grad(): + for data in test_loader: + images, labels = data + outputs = model(images) + _, predicted = torch.max(outputs, 1) + c = (predicted == labels).squeeze() + for i in range(4): + label = labels[i] + class_correct[label] += c[i].item() + class_total[label] += 1 + + for i in range(10): + print('Accuracy of %5s : %2d %%' % ( + classes[i], 100 * class_correct[i] / class_total[i])) + + + From f2d8c7415b9f2496d978380ffd3e95a547057dff Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Fri, 11 Oct 2019 23:17:53 +0900 Subject: [PATCH 09/37] minor fixed --- dataloader/data_loader.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dataloader/data_loader.py b/dataloader/data_loader.py index 4877491..e0488aa 100644 --- a/dataloader/data_loader.py +++ b/dataloader/data_loader.py @@ -5,7 +5,9 @@ def data_loader(): transform = transforms.Compose( - [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) + [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))], + + ) train_set = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform) train_loader = torch.utils.data.DataLoader(train_set, batch_size=4, shuffle=True, num_workers=2) test_set = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform) @@ -14,3 +16,4 @@ def data_loader(): classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') return train_loader, test_loader, classes + From 6de5887561606a40f1b503e6f9e4a9538fd3019d Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Fri, 11 Oct 2019 23:18:20 +0900 Subject: [PATCH 10/37] add train_acc --- train.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/train.py b/train.py index a6f8f13..168fb40 100644 --- a/train.py +++ b/train.py @@ -7,23 +7,33 @@ from models.mlp import MLP from models.conv import CNN from models.resnet import ResNet50 +from torchvision.models import ResNet def train(args, model, device, train_loader, optimizer, epoch): model.train() + # calculate accuracy of predictions in the current batch + correct, total = 0, 0 for i, (data, target) in tqdm(enumerate(train_loader, 0)): data = data.to(device) target = target.to(device) optimizer.zero_grad() - output = model(data) + outputs = model(data) criterion = nn.CrossEntropyLoss() - loss = criterion(output, target) + + _, predicted = (torch.max(outputs.data, 1)) + + total += target.size(0) + correct += (predicted == target).sum().item() + train_acc = 100. * correct / total + + loss = criterion(outputs, target) loss.backward() optimizer.step() if i % args.log_interval == 0: - print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + print('Train Epoch: {} [{}/{} ({:.0f}%)]\tAccuracy: {:.6f}\tLoss: {:.6f}'.format( epoch, i * len(data), len(train_loader.dataset), - 100. * i / len(train_loader), loss.item())) + 100. * i / len(train_loader), train_acc, loss.item())) def main(): @@ -33,7 +43,7 @@ def main(): help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', help='input batch size for testing (default: 1000)') - parser.add_argument('--epochs', type=int, default=1, metavar='N', + parser.add_argument('--epochs', type=int, default=10, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=0.01, metavar='LR', help='learning rate (default: 0.01)') @@ -74,4 +84,5 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() + From b3e85a5e6da972facfe3b0858831edec88cdfbab Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Fri, 11 Oct 2019 23:18:44 +0900 Subject: [PATCH 11/37] add test.py --- test.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/test.py b/test.py index 6f72b6a..b143147 100644 --- a/test.py +++ b/test.py @@ -1,26 +1,36 @@ import torch from models.resnet import ResNet50 from dataloader.data_loader import data_loader - +import argparse if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--cuda', action='store_true', + help='use CUDA') + args = parser.parse_args() classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') + device = torch.device("cuda" if args.cuda else "cpu") test_loader = data_loader()[1] correct = 0 total = 0 model = ResNet50() + model.to(device) model.load_state_dict(torch.load('CIFAR-10_ResNet50.pt')) model.eval() with torch.no_grad(): for data in test_loader: images, labels = data + images = images.to(device) + labels = labels.to(device) + model = model.to(device) outputs = model(images) _, predicted = torch.max(outputs.data, 1) total += labels.size(0) correct += (predicted == labels).sum().item() - print('Accuracy of the network on the 10000 test images: %d %%' % (100 * correct / total)) + train_acc = 100 * correct / total + print('Accuracy of the network on the 10000 test images: %d %%' % (train_acc)) # Class correct class_correct = list(0. for i in range(10)) @@ -28,6 +38,9 @@ with torch.no_grad(): for data in test_loader: images, labels = data + images = images.to(device) + labels = labels.to(device) + model = model.to(device) outputs = model(images) _, predicted = torch.max(outputs, 1) c = (predicted == labels).squeeze() @@ -39,6 +52,3 @@ for i in range(10): print('Accuracy of %5s : %2d %%' % ( classes[i], 100 * class_correct[i] / class_total[i])) - - - From 6433bd17f93c75b82309effd0dc9435c4ab99f1e Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Sat, 12 Oct 2019 21:33:36 +0900 Subject: [PATCH 12/37] add validation_set_loader --- dataloader/data_loader.py | 99 +++++++++++++++++++++++++++++++++++---- 1 file changed, 91 insertions(+), 8 deletions(-) diff --git a/dataloader/data_loader.py b/dataloader/data_loader.py index 4877491..5e06699 100644 --- a/dataloader/data_loader.py +++ b/dataloader/data_loader.py @@ -1,16 +1,99 @@ import torch import torchvision import torchvision.transforms as transforms +import numpy as np +from torch.utils.data.sampler import SubsetRandomSampler +# from utils import plot_images -def data_loader(): - transform = transforms.Compose( - [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) - train_set = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform) - train_loader = torch.utils.data.DataLoader(train_set, batch_size=4, shuffle=True, num_workers=2) +def train_valid_loader(batch_size, + augment, + num_workers, + valid_size=0.1, + shuffle=True, + show_sample=False, + pin_memory=True, + random_seed=42): + """ + train_data loader and validation_data loader + + :param batch_size: samples per batch to load. + :param augment: whether to apply the data augment. + :param valid_size: valid_size(type=float, in [0, 1]). + :param shuffle: whether to shuffle the train_validation indices. + :param show_sample: plot sample grid of the dataset. + :param num_workers: number of subprocesses to use when loading the dataset. + :param pin_memory: whether to copy tensors into CUDA pinned memory. Set it to True if using GPU. + :param random_seed: + :return: train_loader, valid_loader + """ + + # define transforms + valid_transform = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) + ]) + + if augment: + train_transform = transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) + ]) + train_set = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=train_transform) + valid_set = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=valid_transform) + + num_train = len(train_set) + indices = list(range(num_train)) + split = int(np.floor(valid_size * num_train)) + + if shuffle: + np.random.seed(random_seed) + np.random.shuffle(indices) + + train_idx, valid_idx = indices[split:], indices[:split] + train_sampler = SubsetRandomSampler(train_idx) + valid_sampler = SubsetRandomSampler(valid_idx) + + train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, sampler=train_sampler, + num_workers=num_workers, pin_memory=pin_memory) + valid_loader = torch.utils.data.DataLoader(valid_set, batch_size=batch_size, sampler=valid_sampler, + num_workers=num_workers, pin_memory=pin_memory) + + # 시각화 + if show_sample: + sample_loader = torch.utils.data.DataLoader( + train_set, batch_size=9, shuffle=shuffle, + num_workers=num_workers, pin_memory=pin_memory + ) + data_iter = iter(sample_loader) + images, labels = data_iter.next() + X = images.numpy().transpose([0, 2, 3, 1]) + plot_images(X, labels) + + return train_loader, valid_loader + + +def test_loader(batch_size, shuffle=False, num_workers=4, pin_memory=False): + """ + test_data loader + + :param batch_size: samples per batch to load. + :param shuffle: whether to shuffle the train_validation indices. + :param num_workers: number of subprocesses to use when loading the dataset. + :param pin_memory: whether to copy tensors into CUDA pinned memory. Set it to True if using GPU. + :return: test_loader + """ + transform = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) + ]) test_set = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform) - test_loader = torch.utils.data.DataLoader(test_set, batch_size=4, shuffle=False, num_workers=2) + test_loader = torch.utils.data.DataLoader( + dataset=test_set, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers, pin_memory=pin_memory, + ) + + return test_loader - classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') - return train_loader, test_loader, classes From afc72c9d1d2b67b2d7fada2f7fa3c0e275e96873 Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Sat, 12 Oct 2019 21:35:27 +0900 Subject: [PATCH 13/37] validate --- validate.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 validate.py diff --git a/validate.py b/validate.py new file mode 100644 index 0000000..55aa061 --- /dev/null +++ b/validate.py @@ -0,0 +1,50 @@ +# import +# built in package +import argparse + +# custom package +import torch + +# from +from dataloader.data_loader import train_valid_loader +from models.resnet import ResNet50 + + +if __name__ == "__main__": + classes = ('plane', 'car', 'bird', 'cat', + 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') + + test_loader = train_valid_loader(10, augment=True, num_workers=4)[1] + correct = 0 + total = 0 + model = ResNet50() + model.load_state_dict(torch.load("CIFAR-10_ResNet50.pt")) + + model.eval() + with torch.no_grad(): + for data in test_loader: + images, labels = data + outputs = model(images) + _, predicted = torch.max(outputs.data, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + + print('Accuracy of the network on the 10000 test images: %d %%' % ( + 100 * correct / total)) + + class_correct = list(0. for i in range(10)) + class_total = list(0. for i in range(10)) + with torch.no_grad(): + for data in test_loader: + images, labels = data + outputs = model(images) + _, predicted = torch.max(outputs, 1) + c = (predicted == labels).squeeze() + for i in range(4): + label = labels[i] + class_correct[label] += c[i].item() + class_total[label] += 1 + + for i in range(10): + print('Accuracy of %5s : %2d %%' % ( + classes[i], 100 * class_correct[i] / class_total[i])) From 5ce21d415857e1dedc28a6467b6855a63ba7f97c Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Tue, 15 Oct 2019 05:27:39 +0900 Subject: [PATCH 14/37] add confusion_matrix --- metrics.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 metrics.py diff --git a/metrics.py b/metrics.py new file mode 100644 index 0000000..1771ab0 --- /dev/null +++ b/metrics.py @@ -0,0 +1,45 @@ +import torch + + +def confusion_matrix(out, target): + tp = 0 + fp = 0 + tn = 0 + fn = 0 + + for i in range(len(target)): + # True Positive, 실제 True(1)인 정답을 True로 예측(정답) + if target[i] == out[i] == 1: + tp += 1 + # False Positive, 실제는 False지만 정답을 True(1)라고 예측(오답) + if out[i] == 1 and target[i] != out[i]: + fp += 1 + # True Negative, 실제는 False를 False라고 예측(정답) + if target[i] == out[i] == 0: + tn += 1 + # False Negative, 실제는 True인 정답을 False라고 예측(오답) + if target[i] == 0 and target[i] != out[i]: + fn += 1 + + return tp, fp, tn, fn + + +def accuracy(out, target): + with torch.no_grad(): + _, predicted = torch.max(out, 1) + acc = (predicted == target).sum() / target.size(0) + return acc + + +def precision(out, target): + with torch.no_grad(): + tp, fp = confusion_matrix(out, target)[0], confusion_matrix(out, target)[1] + return tp / (tp + fp) + + +def recall(out, target): + with torch.no_grad(): + tp, fn = confusion_matrix(out, target)[0], confusion_matrix(out, target)[3] + return tp / (tp + fn) + + From 00f6e5eccfbf164233a175c1eafbab163107c0fb Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Tue, 15 Oct 2019 05:28:01 +0900 Subject: [PATCH 15/37] add validation --- validate.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 validate.py diff --git a/validate.py b/validate.py new file mode 100644 index 0000000..4c6d38d --- /dev/null +++ b/validate.py @@ -0,0 +1,54 @@ +import torch +from models.resnet import ResNet50 +from dataloader.data_loader import train_valid_loader +import argparse + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--cuda', action='store_true', + help='use CUDA') + args = parser.parse_args() + classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') + device = torch.device("cuda" if args.cuda else "cpu") + + valid_loader = train_valid_loader()[1] + correct = 0 + total = 0 + model = ResNet50() + model.to(device) + model.load_state_dict(torch.load('CIFAR-10_ResNet50.pt')) + + model.eval() + with torch.no_grad(): + for data in valid_loader: + images, labels = data + images = images.to(device) + labels = labels.to(device) + model = model.to(device) + outputs = model(images) + _, predicted = torch.max(outputs.data, 1) + total += labels.size(0) + correct += (predicted == labels).sum().item() + train_acc = 100 * correct / total + print('Accuracy of the network on the 10000 test images: %d %%' % (train_acc)) + + # Class correct + class_correct = list(0. for i in range(10)) + class_total = list(0. for i in range(10)) + with torch.no_grad(): + for data in valid_loader: + images, labels = data + images = images.to(device) + labels = labels.to(device) + model = model.to(device) + outputs = model(images) + _, predicted = torch.max(outputs, 1) + c = (predicted == labels).squeeze() + for i in range(4): + label = labels[i] + class_correct[label] += c[i].item() + class_total[label] += 1 + + for i in range(10): + print('Accuracy of %5s : %2d %%' % ( + classes[i], 100 * class_correct[i] / class_total[i])) From 933f67c734c56797bb5cef0726e3729bf75788b2 Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Tue, 15 Oct 2019 05:29:24 +0900 Subject: [PATCH 16/37] minor change --- train.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/train.py b/train.py index 168fb40..1775a37 100644 --- a/train.py +++ b/train.py @@ -1,19 +1,20 @@ import torch import torch.nn as nn import torch.optim as optim -from dataloader.data_loader import data_loader +from dataloader.data_loader import train_valid_loader +from dataloader.data_loader import test_loader from tqdm import tqdm import argparse +import numpy as np from models.mlp import MLP from models.conv import CNN from models.resnet import ResNet50 -from torchvision.models import ResNet def train(args, model, device, train_loader, optimizer, epoch): model.train() # calculate accuracy of predictions in the current batch - correct, total = 0, 0 + correct, running_loss, total = 0, 0, 0 for i, (data, target) in tqdm(enumerate(train_loader, 0)): data = data.to(device) target = target.to(device) @@ -23,17 +24,19 @@ def train(args, model, device, train_loader, optimizer, epoch): _, predicted = (torch.max(outputs.data, 1)) - total += target.size(0) - correct += (predicted == target).sum().item() - train_acc = 100. * correct / total - loss = criterion(outputs, target) loss.backward() optimizer.step() + + total += target.size(0) + correct += (predicted == target).sum().item() + train_acc = 100. * correct / total + running_loss += loss.item() if i % args.log_interval == 0: - print('Train Epoch: {} [{}/{} ({:.0f}%)]\tAccuracy: {:.6f}\tLoss: {:.6f}'.format( + print('Train Epoch: {} [{}/{} ({:.0f}%)]\tAccuracy: {:.2f}\tLoss: {:.2f}'.format( epoch, i * len(data), len(train_loader.dataset), - 100. * i / len(train_loader), train_acc, loss.item())) + 100. * i / len(train_loader), train_acc, running_loss / 10)) + running_loss = 0.0 def main(): @@ -53,7 +56,6 @@ def main(): help='random seed (default: 1)') parser.add_argument('--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') - parser.add_argument('--save_model', action='store_true', default=False, help='For Saving the current Model') parser.add_argument('--cuda', action='store_true', @@ -65,7 +67,7 @@ def main(): print("WARNING: You have a CUDA device, so you should probably run with --cuda") device = torch.device("cuda" if args.cuda else "cpu") - train_loader = data_loader()[0] + train_loader = train_valid_loader()[0] # if args.model == 'CNN': # model = CNN() From 702eaa83d0a0412142e6b6d11ac09a1b575fc3fb Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Tue, 15 Oct 2019 05:29:45 +0900 Subject: [PATCH 17/37] minor change --- test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test.py b/test.py index b143147..0092f8f 100644 --- a/test.py +++ b/test.py @@ -1,6 +1,6 @@ import torch from models.resnet import ResNet50 -from dataloader.data_loader import data_loader +from dataloader.data_loader import test_loader import argparse if __name__ == "__main__": @@ -11,7 +11,7 @@ classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') device = torch.device("cuda" if args.cuda else "cpu") - test_loader = data_loader()[1] + test_loader = test_loader() correct = 0 total = 0 model = ResNet50() From 693c81d41417beb9238a4a03be3c8c04feba5da7 Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Tue, 15 Oct 2019 05:30:03 +0900 Subject: [PATCH 18/37] split data into train, validation, test --- dataloader/data_loader.py | 77 ++++++++++++++++++++++++++++++++++----- 1 file changed, 68 insertions(+), 9 deletions(-) diff --git a/dataloader/data_loader.py b/dataloader/data_loader.py index e0488aa..785964c 100644 --- a/dataloader/data_loader.py +++ b/dataloader/data_loader.py @@ -1,19 +1,78 @@ import torch import torchvision import torchvision.transforms as transforms +from torch.utils.data.sampler import SubsetRandomSampler +from torch.utils.data.sampler import Sampler -def data_loader(): - transform = transforms.Compose( - [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))], +class CustomSampler(Sampler): + """ + Samples elements sequentially from some offset. + Arguments: + num_samples: number of desired datapoints + start: offset where we shuld start selecting from + """ + def __init__(self, num_samples, start=0): + self.num_samples = num_samples + self.start = start + + def __iter__(self): + return iter(range(self.start, self.start + self.num_samples)) + + def __len__(self): + return self.num_samples + + +num_train = 49000 +num_valid = 1000 + +def train_valid_loader(valid_size=0.1, + shuffle=True, + pin_memory=True, + random_seed=42): + """ + train_data loader and validation_data loader + :param num_workers: number of subprocesses + :param valid_size: valid_size(type=fload, in [0, 1]). + :param augment: whether to apply the data augment + :param shuffle: whether to shuffle data + :param pin_memory: copy tensors into CUDA pinned memory + :param random_seed: + :return: train_loader valid_loader + """ + # define transforms + train_transform = transforms.Compose( + [transforms.Resize((32, 32)), + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] + ) + valid_transform = transforms.Compose( + [transforms.ToTensor(), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] + ) + train_set = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, + transform=train_transform) + valid_set = torchvision.datasets.CIFAR10(root='./data', train=True, download=False, + transform=valid_transform) + train_loader = torch.utils.data.DataLoader(train_set, batch_size=4, sampler=CustomSampler(num_train), + num_workers=0) + valid_loader = torch.utils.data.DataLoader(valid_set, batch_size=4, sampler=CustomSampler(num_valid, num_train), + num_workers=0) + + return train_loader, valid_loader + + +def test_loader(): + transform = transforms.Compose( + [transforms.ToTensor(), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] ) - train_set = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform) - train_loader = torch.utils.data.DataLoader(train_set, batch_size=4, shuffle=True, num_workers=2) - test_set = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform) - test_loader = torch.utils.data.DataLoader(test_set, batch_size=4, shuffle=False, num_workers=2) - classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') + test_data_set = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform) + test_data_loader = torch.utils.data.DataLoader(test_data_set, batch_size=4, shuffle=False, num_workers=0) - return train_loader, test_loader, classes + return test_data_loader From 4539c0722e5558fcd3db8c439c6103931c47b4c9 Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Fri, 25 Oct 2019 07:21:46 +0900 Subject: [PATCH 19/37] remove metrics.py --- metrics.py | 45 --------------------------------------------- 1 file changed, 45 deletions(-) delete mode 100644 metrics.py diff --git a/metrics.py b/metrics.py deleted file mode 100644 index 1771ab0..0000000 --- a/metrics.py +++ /dev/null @@ -1,45 +0,0 @@ -import torch - - -def confusion_matrix(out, target): - tp = 0 - fp = 0 - tn = 0 - fn = 0 - - for i in range(len(target)): - # True Positive, 실제 True(1)인 정답을 True로 예측(정답) - if target[i] == out[i] == 1: - tp += 1 - # False Positive, 실제는 False지만 정답을 True(1)라고 예측(오답) - if out[i] == 1 and target[i] != out[i]: - fp += 1 - # True Negative, 실제는 False를 False라고 예측(정답) - if target[i] == out[i] == 0: - tn += 1 - # False Negative, 실제는 True인 정답을 False라고 예측(오답) - if target[i] == 0 and target[i] != out[i]: - fn += 1 - - return tp, fp, tn, fn - - -def accuracy(out, target): - with torch.no_grad(): - _, predicted = torch.max(out, 1) - acc = (predicted == target).sum() / target.size(0) - return acc - - -def precision(out, target): - with torch.no_grad(): - tp, fp = confusion_matrix(out, target)[0], confusion_matrix(out, target)[1] - return tp / (tp + fp) - - -def recall(out, target): - with torch.no_grad(): - tp, fn = confusion_matrix(out, target)[0], confusion_matrix(out, target)[3] - return tp / (tp + fn) - - From 3f24d241547de914d0747f1116cf2627b29c3d5d Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Sat, 26 Oct 2019 23:42:41 +0900 Subject: [PATCH 20/37] add aug module with albumentations --- augmentations.py | 60 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 augmentations.py diff --git a/augmentations.py b/augmentations.py new file mode 100644 index 0000000..933314c --- /dev/null +++ b/augmentations.py @@ -0,0 +1,60 @@ +import numpy as np +import cv2 +from matplotlib import pyplot as plt +from albumentations import ( +HorizontalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90, + Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue, + IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, + IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose + +) + + +def aug(p=0.5): + return Compose([ + RandomRotate90(), + Flip(), + Transpose(), + OneOf([ + IAAAdditiveGaussianNoise(), + GaussNoise(), + ], p=0.2), + OneOf([ + MotionBlur(p=0.2), + MedianBlur(blur_limit=3, p=0.1), + Blur(blur_limit=3, p=0.1), + ], p=0.2), + ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=0.2), + OneOf([ + OpticalDistortion(p=0.3), + GridDistortion(p=0.1), + IAAPiecewiseAffine(p=0.3), + ], p=0.2), + OneOf([ + CLAHE(clip_limit=2), + IAASharpen(), + IAAEmboss(), + RandomBrightnessContrast(), + ], p=0.3), + HueSaturationValue(p=0.3), + ], p=p) + + +def show_img(img, figsize=(8, 8)): + fig, ax = plt.subplots(figsize=figsize) + ax.grid(False) + ax.set_yticklabels([]) + ax.set_xticklabels([]) + ax.imshow(img) + plt.imshow(img) + + +if __name__ == "__main__": + + + image = cv2.imread('dog.12473.jpg') + augmentation = aug(p=0.9) + data = {'image': image} + augmented = augmentation(**data) + image = augmented['image'] + show_img(image) \ No newline at end of file From 6832bd984a71e482741bd69ce5d6993dd58a7043 Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Sun, 27 Oct 2019 00:02:32 +0900 Subject: [PATCH 21/37] fix dataloader --- dataloader/data_loader.py | 93 +++++++++------------------------------ 1 file changed, 20 insertions(+), 73 deletions(-) diff --git a/dataloader/data_loader.py b/dataloader/data_loader.py index 785964c..ef8ee6b 100644 --- a/dataloader/data_loader.py +++ b/dataloader/data_loader.py @@ -1,78 +1,25 @@ import torch import torchvision import torchvision.transforms as transforms -from torch.utils.data.sampler import SubsetRandomSampler -from torch.utils.data.sampler import Sampler -class CustomSampler(Sampler): - """ - Samples elements sequentially from some offset. - Arguments: - num_samples: number of desired datapoints - start: offset where we shuld start selecting from - """ - def __init__(self, num_samples, start=0): - self.num_samples = num_samples - self.start = start - - def __iter__(self): - return iter(range(self.start, self.start + self.num_samples)) - - def __len__(self): - return self.num_samples - - -num_train = 49000 -num_valid = 1000 - - -def train_valid_loader(valid_size=0.1, - shuffle=True, - pin_memory=True, - random_seed=42): - """ - train_data loader and validation_data loader - :param num_workers: number of subprocesses - :param valid_size: valid_size(type=fload, in [0, 1]). - :param augment: whether to apply the data augment - :param shuffle: whether to shuffle data - :param pin_memory: copy tensors into CUDA pinned memory - :param random_seed: - :return: train_loader valid_loader - """ - # define transforms - train_transform = transforms.Compose( - [transforms.Resize((32, 32)), - transforms.RandomCrop(32, padding=4), - transforms.RandomHorizontalFlip(), - transforms.ToTensor(), - transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] - ) - valid_transform = transforms.Compose( - [transforms.ToTensor(), - transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] - ) - train_set = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, - transform=train_transform) - valid_set = torchvision.datasets.CIFAR10(root='./data', train=True, download=False, - transform=valid_transform) - train_loader = torch.utils.data.DataLoader(train_set, batch_size=4, sampler=CustomSampler(num_train), - num_workers=0) - valid_loader = torch.utils.data.DataLoader(valid_set, batch_size=4, sampler=CustomSampler(num_valid, num_train), - num_workers=0) - - return train_loader, valid_loader - - -def test_loader(): - transform = transforms.Compose( - [transforms.ToTensor(), - transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] - ) - - test_data_set = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform) - test_data_loader = torch.utils.data.DataLoader(test_data_set, batch_size=4, shuffle=False, num_workers=0) - - return test_data_loader - +def data_loader(dataset="CIFAR-10", batch_size = 16): + + if dataset == "CIFAR-10": + transform = transforms.Compose( + [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) + train_set = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform) + train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=2) + test_set = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform) + test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, shuffle=False, num_workers=2) + classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') + elif dataset == "MNIST": + transform = transforms.Compose( + [transforms.ToTensor()]) + train_set = torchvision.datasets.MNIST(root='./data', train=True, download=True, transform=transform) + train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=2) + test_set = torchvision.datasets.MNIST(root='./data', train=False, download=True, transform=transform) + test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, shuffle=False, num_workers=2) + classes = ("0", "1", "2", "3", "4", "5", "6", "7", "8", "9") + + return train_loader, test_loader, classes \ No newline at end of file From 83b87c9ee48770d59870bc3216b74b0e4c3807ba Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Sun, 27 Oct 2019 00:03:36 +0900 Subject: [PATCH 22/37] =?UTF-8?q?=20=EB=B3=80=EA=B2=BD=20=EC=82=AC?= =?UTF-8?q?=ED=95=AD=EC=97=90=20=EB=8C=80=ED=95=9C=20=EC=BB=A4=EB=B0=8B=20?= =?UTF-8?q?=EB=A9=94=EC=8B=9C=EC=A7=80=EB=A5=BC=20=EC=9E=85=EB=A0=A5?= =?UTF-8?q?=ED=95=98=EC=8B=AD=EC=8B=9C=EC=98=A4.=20'#'=20=EB=AC=B8?= =?UTF-8?q?=EC=9E=90=EB=A1=9C=20=EC=8B=9C=EC=9E=91=ED=95=98=EB=8A=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test.py | 54 ------------------------------------------------------ 1 file changed, 54 deletions(-) delete mode 100644 test.py diff --git a/test.py b/test.py deleted file mode 100644 index 0092f8f..0000000 --- a/test.py +++ /dev/null @@ -1,54 +0,0 @@ -import torch -from models.resnet import ResNet50 -from dataloader.data_loader import test_loader -import argparse - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('--cuda', action='store_true', - help='use CUDA') - args = parser.parse_args() - classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') - device = torch.device("cuda" if args.cuda else "cpu") - - test_loader = test_loader() - correct = 0 - total = 0 - model = ResNet50() - model.to(device) - model.load_state_dict(torch.load('CIFAR-10_ResNet50.pt')) - - model.eval() - with torch.no_grad(): - for data in test_loader: - images, labels = data - images = images.to(device) - labels = labels.to(device) - model = model.to(device) - outputs = model(images) - _, predicted = torch.max(outputs.data, 1) - total += labels.size(0) - correct += (predicted == labels).sum().item() - train_acc = 100 * correct / total - print('Accuracy of the network on the 10000 test images: %d %%' % (train_acc)) - - # Class correct - class_correct = list(0. for i in range(10)) - class_total = list(0. for i in range(10)) - with torch.no_grad(): - for data in test_loader: - images, labels = data - images = images.to(device) - labels = labels.to(device) - model = model.to(device) - outputs = model(images) - _, predicted = torch.max(outputs, 1) - c = (predicted == labels).squeeze() - for i in range(4): - label = labels[i] - class_correct[label] += c[i].item() - class_total[label] += 1 - - for i in range(10): - print('Accuracy of %5s : %2d %%' % ( - classes[i], 100 * class_correct[i] / class_total[i])) From 0ebaacd9eaf0aca3c1897e08daa3d975bfe31d18 Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Sun, 27 Oct 2019 00:05:23 +0900 Subject: [PATCH 23/37] delete train --- train.py | 90 -------------------------------------------------------- 1 file changed, 90 deletions(-) delete mode 100644 train.py diff --git a/train.py b/train.py deleted file mode 100644 index 1775a37..0000000 --- a/train.py +++ /dev/null @@ -1,90 +0,0 @@ -import torch -import torch.nn as nn -import torch.optim as optim -from dataloader.data_loader import train_valid_loader -from dataloader.data_loader import test_loader -from tqdm import tqdm -import argparse -import numpy as np -from models.mlp import MLP -from models.conv import CNN -from models.resnet import ResNet50 - - -def train(args, model, device, train_loader, optimizer, epoch): - model.train() - # calculate accuracy of predictions in the current batch - correct, running_loss, total = 0, 0, 0 - for i, (data, target) in tqdm(enumerate(train_loader, 0)): - data = data.to(device) - target = target.to(device) - optimizer.zero_grad() - outputs = model(data) - criterion = nn.CrossEntropyLoss() - - _, predicted = (torch.max(outputs.data, 1)) - - loss = criterion(outputs, target) - loss.backward() - optimizer.step() - - total += target.size(0) - correct += (predicted == target).sum().item() - train_acc = 100. * correct / total - running_loss += loss.item() - if i % args.log_interval == 0: - print('Train Epoch: {} [{}/{} ({:.0f}%)]\tAccuracy: {:.2f}\tLoss: {:.2f}'.format( - epoch, i * len(data), len(train_loader.dataset), - 100. * i / len(train_loader), train_acc, running_loss / 10)) - running_loss = 0.0 - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--model', type=str, default='ResNet50') - parser.add_argument('--batch-size', type=int, default=64, metavar='N', - help='input batch size for training (default: 64)') - parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', - help='input batch size for testing (default: 1000)') - parser.add_argument('--epochs', type=int, default=10, metavar='N', - help='number of epochs to train (default: 10)') - parser.add_argument('--lr', type=float, default=0.01, metavar='LR', - help='learning rate (default: 0.01)') - parser.add_argument('--momentum', type=float, default=0.5, metavar='M', - help='SGD momentum (default: 0.5)') - parser.add_argument('--seed', type=int, default=1, metavar='S', - help='random seed (default: 1)') - parser.add_argument('--log-interval', type=int, default=10, metavar='N', - help='how many batches to wait before logging training status') - parser.add_argument('--save_model', action='store_true', default=False, - help='For Saving the current Model') - parser.add_argument('--cuda', action='store_true', - help='use CUDA') - args = parser.parse_args() - - if torch.cuda.is_available(): - if not args.cuda: - print("WARNING: You have a CUDA device, so you should probably run with --cuda") - - device = torch.device("cuda" if args.cuda else "cpu") - train_loader = train_valid_loader()[0] - - # if args.model == 'CNN': - # model = CNN() - # if args.model == 'MLP': - # model = MLP() - model = ResNet50() - model.to(device) - - optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) - - for epoch in tqdm(range(1, args.epochs + 1)): - train(args, model, device, train_loader, optimizer, epoch) - - if (args.save_model): - torch.save(model.state_dict(), "CIFAR-10_ResNet50.pt") - - -if __name__ == "__main__": - main() - From fb7e9acb98e6f7e6f5ee2448cda1184dad7376df Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Sun, 27 Oct 2019 00:05:48 +0900 Subject: [PATCH 24/37] delete validate --- validate.py | 54 ----------------------------------------------------- 1 file changed, 54 deletions(-) delete mode 100644 validate.py diff --git a/validate.py b/validate.py deleted file mode 100644 index 4c6d38d..0000000 --- a/validate.py +++ /dev/null @@ -1,54 +0,0 @@ -import torch -from models.resnet import ResNet50 -from dataloader.data_loader import train_valid_loader -import argparse - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('--cuda', action='store_true', - help='use CUDA') - args = parser.parse_args() - classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') - device = torch.device("cuda" if args.cuda else "cpu") - - valid_loader = train_valid_loader()[1] - correct = 0 - total = 0 - model = ResNet50() - model.to(device) - model.load_state_dict(torch.load('CIFAR-10_ResNet50.pt')) - - model.eval() - with torch.no_grad(): - for data in valid_loader: - images, labels = data - images = images.to(device) - labels = labels.to(device) - model = model.to(device) - outputs = model(images) - _, predicted = torch.max(outputs.data, 1) - total += labels.size(0) - correct += (predicted == labels).sum().item() - train_acc = 100 * correct / total - print('Accuracy of the network on the 10000 test images: %d %%' % (train_acc)) - - # Class correct - class_correct = list(0. for i in range(10)) - class_total = list(0. for i in range(10)) - with torch.no_grad(): - for data in valid_loader: - images, labels = data - images = images.to(device) - labels = labels.to(device) - model = model.to(device) - outputs = model(images) - _, predicted = torch.max(outputs, 1) - c = (predicted == labels).squeeze() - for i in range(4): - label = labels[i] - class_correct[label] += c[i].item() - class_total[label] += 1 - - for i in range(10): - print('Accuracy of %5s : %2d %%' % ( - classes[i], 100 * class_correct[i] / class_total[i])) From b7df5c0f217ad648eb0048a15ecc075607db8e00 Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Sun, 27 Oct 2019 00:14:30 +0900 Subject: [PATCH 25/37] delete resnet --- models/resnet.py | 93 ------------------------------------------------ 1 file changed, 93 deletions(-) delete mode 100644 models/resnet.py diff --git a/models/resnet.py b/models/resnet.py deleted file mode 100644 index 073da1f..0000000 --- a/models/resnet.py +++ /dev/null @@ -1,93 +0,0 @@ -import torch -import torch.nn as nn -import torch.nn.functional as F -from torchvision.models import ResNet - - -class BasicBlock(nn.Module): - expansion = 1 - - def __init__(self, in_planes, planes, stride=1): - super(BasicBlock, self).__init__() - self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) - self.bn1 = nn.BatchNorm2d(planes) - self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) - self.bn2 = nn.BatchNorm2d(planes) - - self.shortcut = nn.Sequential() - if stride != 1 or in_planes != self.expansion * planes: - self.shortcut = nn.Sequential( - nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False), - nn.BatchNorm2d(self.expansion * planes) - ) - - def forward(self, x): - out = F.relu(self.bn1(self.conv1(x))) - out = self.bn2(self.conv2(out)) - out += self.shortcut(x) - out = F.relu(out) - return out - - -class Bottleneck(nn.Module): - expansion = 4 - - def __init__(self, in_planes, planes, stride=1): - super(Bottleneck, self).__init__() - self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) - self.bn1 = nn.BatchNorm2d(planes) - self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) - self.bn2 = nn.BatchNorm2d(planes) - self.conv3 = nn.Conv2d(planes, self.expansion * planes, kernel_size=1, bias=False) - self.bn3 = nn.BatchNorm2d(self.expansion * planes) - - self.shortcut = nn.Sequential() - if stride != 1 or in_planes != self.expansion * planes: - self.shortcut = nn.Sequential( - nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False), - nn.BatchNorm2d(self.expansion * planes) - ) - - def forward(self, x): - out = F.relu(self.bn1(self.conv1(x))) - out = F.relu(self.bn2(self.conv2(out))) - out = self.bn3(self.conv3(out)) - out += self.shortcut(x) - out = F.relu(out) - return out - - -class ResNet(nn.Module): - def __init__(self, block, num_blocks, num_classes=10): - super(ResNet, self).__init__() - self.in_planes = 64 - self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) - self.bn1 = nn.BatchNorm2d(64) - self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) - self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) - self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) - self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) - self.linear = nn.Linear(512 * block.expansion, num_classes) - - def _make_layer(self, block, planes, num_blocks, stride): - strides = [stride] + [1] * (num_blocks - 1) - layers = [] - for stride in strides: - layers.append(block(self.in_planes, planes, stride)) - self.in_planes = planes * block.expansion - return nn.Sequential(*layers) - - def forward(self, x): - out = F.relu(self.bn1(self.conv1(x))) - out = self.layer1(out) - out = self.layer2(out) - out = self.layer3(out) - out = self.layer4(out) - out = F.avg_pool2d(out, 4) - out = out.view(out.size(0), -1) - out = self.linear(out) - return out - - -def ResNet50(): - return ResNet(Bottleneck, [3, 4, 6, 3]) From 53433a5c021fa3db8fc4be7406f31d2f939ee2ab Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Sun, 27 Oct 2019 00:14:59 +0900 Subject: [PATCH 26/37] add BinaryLinear --- utils/BinaryLinear.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/BinaryLinear.py b/utils/BinaryLinear.py index 79f7682..26eb431 100644 --- a/utils/BinaryLinear.py +++ b/utils/BinaryLinear.py @@ -32,7 +32,7 @@ def deterministic(weight: torch.tensor) -> torch.tensor: return weight.sign() @staticmethod - def stocastic(weight: torch.tensor) -> torch.tensor: + def stochastic(weight: torch.tensor) -> torch.tensor: p = torch.sigmoid(weight) uniform_matrix = torch.empty(p.shape).uniform_(0,1) bin_weight = (p >= uniform_matrix).type(torch.float32) From 46330c87e9f8d7ef75821733865e59a244e54847 Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Sun, 27 Oct 2019 00:15:23 +0900 Subject: [PATCH 27/37] add binarized_conv --- utils/binarized_conv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/binarized_conv.py b/utils/binarized_conv.py index fa26c8c..8e14f0d 100644 --- a/utils/binarized_conv.py +++ b/utils/binarized_conv.py @@ -54,7 +54,7 @@ def deterministic(self, weight: torch.tensor) -> torch.tensor: bin_weight = weight.sign() return bin_weight - def stocastic(self, weight: torch.tensor) -> torch.tensor: + def stochastic(self, weight: torch.tensor) -> torch.tensor: with torch.no_grad(): p = torch.sigmoid(weight) uniform_matrix = torch.empty(p.shape).uniform_(0, 1) From 4e0eed0908e2a286be7cd995e28fb29fcc382ac8 Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Sun, 27 Oct 2019 00:16:17 +0900 Subject: [PATCH 28/37] add binarized_linear --- utils/binarized_linear.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/binarized_linear.py b/utils/binarized_linear.py index a148160..25a1f59 100644 --- a/utils/binarized_linear.py +++ b/utils/binarized_linear.py @@ -34,7 +34,7 @@ def deterministic(self, weight: torch.tensor) -> torch.tensor: bin_weight = weight.sign() return bin_weight - def stocastic(self, weight: torch.tensor) -> torch.tensor: + def stochastic(self, weight: torch.tensor) -> torch.tensor: with torch.no_grad(): p = torch.sigmoid(weight) uniform_matrix = torch.empty(p.shape).uniform_(0,1) From d4fa9781fd33db8a600180d24082b658f195f055 Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Sun, 27 Oct 2019 00:28:06 +0900 Subject: [PATCH 29/37] add aug module with albumentation --- models/binarized_conv.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/models/binarized_conv.py b/models/binarized_conv.py index b825586..6bb2dd5 100644 --- a/models/binarized_conv.py +++ b/models/binarized_conv.py @@ -3,7 +3,7 @@ import torch import torch.nn as nn import torchvision.transforms as transforms - +from augmentations import aug from torchsummary import summary from torch.nn import functional as F from torch.utils.data import DataLoader @@ -77,15 +77,15 @@ def configure_optimizers(self): @pl.data_loader def train_dataloader(self): - return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=128) + return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=aug()), batch_size=128) @pl.data_loader def val_dataloader(self): - return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32) + return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=aug()), batch_size=32) @pl.data_loader def test_dataloader(self): - return DataLoader(MNIST(os.getcwd(), train=False, download=True, transform=transforms.ToTensor()), batch_size=32) + return DataLoader(MNIST(os.getcwd(), train=False, download=True, transform=aug()), batch_size=32) if __name__ == "__main__": From 3a240047d041e94d74d2def70c4fa1b7852c8f0e Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Sun, 27 Oct 2019 00:28:56 +0900 Subject: [PATCH 30/37] add aug module with albumentations --- models/binarized_mlp.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/models/binarized_mlp.py b/models/binarized_mlp.py index 4e3c95a..053d343 100644 --- a/models/binarized_mlp.py +++ b/models/binarized_mlp.py @@ -4,6 +4,7 @@ import torch.nn as nn import torchvision.transforms as transforms +from augmentations import aug from torchsummary import summary from torch.nn import functional as F from torch.utils.data import DataLoader @@ -63,11 +64,11 @@ def configure_optimizers(self): @pl.data_loader def train_dataloader(self): - return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=128) + return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=aug()), batch_size=128) @pl.data_loader def val_dataloader(self): - return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32) + return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=aug()), batch_size=32) @pl.data_loader def test_dataloader(self): From 142794d7da6ba66c610aab74b8f3a1f9fb7febd9 Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Tue, 29 Oct 2019 23:36:38 +0900 Subject: [PATCH 31/37] change configurations --- augmentations.py | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/augmentations.py b/augmentations.py index 94e1842..1d48d99 100644 --- a/augmentations.py +++ b/augmentations.py @@ -1,19 +1,18 @@ -import random -import numpy as np +import json import cv2 from matplotlib import pyplot as plt from albumentations import ( -HorizontalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90, + HorizontalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90, Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue, IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose ) -def aug(p=0.5, p1=random.uniform(0, 1)): +def aug(config:{}, p=0.5): return Compose([ HorizontalFlip(True), - RandomRotate90(), + RandomRotate90(True), Flip(), Transpose(), OneOf([ @@ -21,23 +20,24 @@ def aug(p=0.5, p1=random.uniform(0, 1)): GaussNoise(), ], p=0.2), OneOf([ - MotionBlur(p=0.2), - MedianBlur(blur_limit=3, p=0.1), - Blur(blur_limit=3, p=0.1), + MotionBlur(p=config["MotionBlur"]), + MedianBlur(blur_limit=config["blur_limit"], p=0.1), + Blur(blur_limit=config["blur_limit"], p=0.1), ], p=0.2), - ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=0.2), + ShiftScaleRotate(shift_limit=config["shift_limit"], scale_limit=config['scale_limit'], + rotate_limit=config["rotate_limit"], p=0.2), OneOf([ OpticalDistortion(p=0.3), GridDistortion(p=0.1), IAAPiecewiseAffine(p=0.3), ], p=0.2), OneOf([ - CLAHE(clip_limit=2), + CLAHE(clip_limit=config["clip_limit"]), IAASharpen(), IAAEmboss(), RandomBrightnessContrast(), ], p=0.3), - HueSaturationValue(p=0.3), + HueSaturationValue(p=config["HueSaturationValue"]), ], p=p) @@ -50,10 +50,23 @@ def show_img(img, figsize=(8, 8)): plt.imshow(img) -if __name__ == "__main__": +# with open('aug_config.json') as json_file: +# json_data = json.load(json_file) +# print(json_data['MotionBlur']) +if __name__ == "__main__": image = cv2.imread('dog.12473.jpg') - augmentation = aug(p=0.5) + config = { + "MotionBlur": 0.2, + "blur_limit": 3, + "MedianBlur": 0.1, + "shift_limit": 0.0625, + "scale_limit": 0.2, + "rotate_limit": 45, + "clip_limit": 2, + "HueSaturationValue": 0.3 + } + augmentation = aug(config) data = {'image': image} augmented = augmentation(**data) image = augmented['image'] From 5035938b664796eeb856ccd027eb4a9574de7d82 Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Thu, 31 Oct 2019 08:56:33 +0900 Subject: [PATCH 32/37] resnet with pytorch_lightning --- models/resnet.py | 120 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 models/resnet.py diff --git a/models/resnet.py b/models/resnet.py new file mode 100644 index 0000000..7991f74 --- /dev/null +++ b/models/resnet.py @@ -0,0 +1,120 @@ +import os +import torch +import torch.nn as nn +import pytorch_lightning as pl +import torch.nn.functional as F +import torchvision.transforms as transforms +from torch.utils.data import DataLoader +from pytorch_lightning import Trainer +from torchvision.datasets import CIFAR10, MNIST + + +def conv3x3(in_channels, out_channels, stride=1): + return nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, + padding=1, bias=False) + +class ResidualBlock(nn.Module): + def __init__(self, in_channels, out_channels, stride=1, downsample=None): + super(ResidualBlock, self).__init__() + self.conv1 = conv3x3(in_channels, out_channels, stride) + self.bn1 = nn.BatchNorm2d(out_channels) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(out_channels, out_channels) + self.bn2 = nn.BatchNorm2d(out_channels) + self.downsample = downsample + + def forward(self, x): + residual = x + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + out = self.conv2(out) + out = self.bn2(out) + if self.downsample: + residual = self.downsample(x) + out += residual + out = self.relu(out) + return out + + +class ResNet(pl.LightningModule): + def __init__(self, block, layers, num_classes=10): + super(ResNet, self).__init__() + self.in_channels = 16 + self.conv = conv3x3(3, 16) + self.bn = nn.BatchNorm2d(16) + self.relu = nn.ReLU(inplace=True) + self.layer1 = self.make_layer(block, 16, layers[0]) + self.layer2 = self.make_layer(block, 32, layers[1], 2) + self.layer3 = self.make_layer(block, 64, layers[2], 2) + self.avg_pool = nn.AvgPool2d(8) + self.fc = nn.Linear(64, num_classes) + + def make_layer(self, block, out_channels, blocks, stride=1): + downsample = None + if (stride != 1) or (self.in_channels != out_channels): + downsample = nn.Sequential( + conv3x3(self.in_channels, out_channels, stride=stride), + nn.BatchNorm2d(out_channels)) + layers = [] + layers.append(block(self.in_channels, out_channels, stride, downsample)) + self.in_channels = out_channels + for i in range(1, blocks): + layers.append(block(out_channels, out_channels)) + return nn.Sequential(*layers) + + def forward(self, x): + out = self.conv(x) + out = self.bn(out) + out = self.relu(out) + out = self.layer1(out) + out = self.layer2(out) + out = self.layer3(out) + out = self.avg_pool(out) + out = out.view(out.size(0), -1) + out = self.fc(out) + return out + + def training_step(self, batch, batch_nb): + x, y = batch + y_hat = self.forward(x) + return {'loss': F.cross_entropy(y_hat, y)} + + def validation_step(self, batch, batch_nb): + x, y = batch + y_hat = self.forward(x) + return {'val_loss': F.cross_entropy(y_hat, y)} + + def validation_end(self, outputs): + avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean() + return {'avg_val_loss': avg_loss} + + def test_step(self, batch, batch_nb): + x, y = batch + y_hat = self.forward(x) + return {'test_loss': F.cross_entropy(y_hat, y)} + + def test_end(self, outputs): + avg_loss = torch.stack([x['test_loss'] for x in outputs]).mean() + return {'avg_test_loss': avg_loss} + + def configure_optimizers(self): + return torch.optim.Adam(self.parameters(), lr=0.02) + + @pl.data_loader + def train_dataloader(self): + return DataLoader(CIFAR10(os.getcwd(), train=True, transform=transforms.ToTensor(), download=True), batch_size=128) + + @pl.data_loader + def val_dataloader(self): + return DataLoader(CIFAR10(os.getcwd(), train=True, transform=transforms.ToTensor(), download=True), batch_size=32) + + @pl.data_loader + def test_dataloader(self): + return DataLoader(CIFAR10(os.getcwd(), train=False, download=True), batch_size=32) + +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + +model = ResNet(ResidualBlock, [3, 4, 6]).to(device) +trainer = Trainer() +trainer.fit(model) \ No newline at end of file From 6235784e39bdee4a813a84c76e9d07ca1f86e20d Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Tue, 19 Nov 2019 23:11:44 +0900 Subject: [PATCH 33/37] ResNet with pytorch lightning --- models/resnet.py | 89 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 30 deletions(-) diff --git a/models/resnet.py b/models/resnet.py index 7991f74..5a713db 100644 --- a/models/resnet.py +++ b/models/resnet.py @@ -13,6 +13,7 @@ def conv3x3(in_channels, out_channels, stride=1): return nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False) + class ResidualBlock(nn.Module): def __init__(self, in_channels, out_channels, stride=1, downsample=None): super(ResidualBlock, self).__init__() @@ -37,42 +38,64 @@ def forward(self, x): return out +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, in_planes, planes, stride=1): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(self.expansion*planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion*planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion*planes) + ) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = F.relu(self.bn2(self.conv2(out))) + out = self.bn3(self.conv3(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + class ResNet(pl.LightningModule): - def __init__(self, block, layers, num_classes=10): + def __init__(self, block, num_blocks, num_classes=10): super(ResNet, self).__init__() - self.in_channels = 16 - self.conv = conv3x3(3, 16) - self.bn = nn.BatchNorm2d(16) - self.relu = nn.ReLU(inplace=True) - self.layer1 = self.make_layer(block, 16, layers[0]) - self.layer2 = self.make_layer(block, 32, layers[1], 2) - self.layer3 = self.make_layer(block, 64, layers[2], 2) - self.avg_pool = nn.AvgPool2d(8) - self.fc = nn.Linear(64, num_classes) - - def make_layer(self, block, out_channels, blocks, stride=1): - downsample = None - if (stride != 1) or (self.in_channels != out_channels): - downsample = nn.Sequential( - conv3x3(self.in_channels, out_channels, stride=stride), - nn.BatchNorm2d(out_channels)) + self.in_planes = 64 + + self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) + self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) + self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) + self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) + self.linear = nn.Linear(512 * block.expansion, num_classes) + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1] * (num_blocks - 1) layers = [] - layers.append(block(self.in_channels, out_channels, stride, downsample)) - self.in_channels = out_channels - for i in range(1, blocks): - layers.append(block(out_channels, out_channels)) + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion return nn.Sequential(*layers) def forward(self, x): - out = self.conv(x) - out = self.bn(out) - out = self.relu(out) + out = F.relu(self.bn1(self.conv1(x))) out = self.layer1(out) out = self.layer2(out) out = self.layer3(out) - out = self.avg_pool(out) + out = self.layer4(out) + out = F.avg_pool2d(out, 4) out = out.view(out.size(0), -1) - out = self.fc(out) + out = self.linear(out) return out def training_step(self, batch, batch_nb): @@ -113,8 +136,14 @@ def val_dataloader(self): def test_dataloader(self): return DataLoader(CIFAR10(os.getcwd(), train=False, download=True), batch_size=32) -device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') -model = ResNet(ResidualBlock, [3, 4, 6]).to(device) -trainer = Trainer() -trainer.fit(model) \ No newline at end of file +def ResNet50(): + return ResNet(Bottleneck, [3,4,6,3]) + + +if __name__ == "__main__": + device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + + model = ResNet50() + trainer = Trainer() + trainer.fit(model) \ No newline at end of file From f4ff94deb69997de54d636f47961e37bdb59c728 Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Wed, 20 Nov 2019 00:49:10 +0900 Subject: [PATCH 34/37] Revert "change configurations" This reverts commit 142794d7da6ba66c610aab74b8f3a1f9fb7febd9. --- augmentations.py | 39 +++++++++++++-------------------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/augmentations.py b/augmentations.py index 1d48d99..94e1842 100644 --- a/augmentations.py +++ b/augmentations.py @@ -1,18 +1,19 @@ -import json +import random +import numpy as np import cv2 from matplotlib import pyplot as plt from albumentations import ( - HorizontalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90, +HorizontalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90, Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue, IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose ) -def aug(config:{}, p=0.5): +def aug(p=0.5, p1=random.uniform(0, 1)): return Compose([ HorizontalFlip(True), - RandomRotate90(True), + RandomRotate90(), Flip(), Transpose(), OneOf([ @@ -20,24 +21,23 @@ def aug(config:{}, p=0.5): GaussNoise(), ], p=0.2), OneOf([ - MotionBlur(p=config["MotionBlur"]), - MedianBlur(blur_limit=config["blur_limit"], p=0.1), - Blur(blur_limit=config["blur_limit"], p=0.1), + MotionBlur(p=0.2), + MedianBlur(blur_limit=3, p=0.1), + Blur(blur_limit=3, p=0.1), ], p=0.2), - ShiftScaleRotate(shift_limit=config["shift_limit"], scale_limit=config['scale_limit'], - rotate_limit=config["rotate_limit"], p=0.2), + ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=0.2), OneOf([ OpticalDistortion(p=0.3), GridDistortion(p=0.1), IAAPiecewiseAffine(p=0.3), ], p=0.2), OneOf([ - CLAHE(clip_limit=config["clip_limit"]), + CLAHE(clip_limit=2), IAASharpen(), IAAEmboss(), RandomBrightnessContrast(), ], p=0.3), - HueSaturationValue(p=config["HueSaturationValue"]), + HueSaturationValue(p=0.3), ], p=p) @@ -50,23 +50,10 @@ def show_img(img, figsize=(8, 8)): plt.imshow(img) -# with open('aug_config.json') as json_file: -# json_data = json.load(json_file) -# print(json_data['MotionBlur']) - if __name__ == "__main__": + image = cv2.imread('dog.12473.jpg') - config = { - "MotionBlur": 0.2, - "blur_limit": 3, - "MedianBlur": 0.1, - "shift_limit": 0.0625, - "scale_limit": 0.2, - "rotate_limit": 45, - "clip_limit": 2, - "HueSaturationValue": 0.3 - } - augmentation = aug(config) + augmentation = aug(p=0.5) data = {'image': image} augmented = augmentation(**data) image = augmented['image'] From 6550c5a1c2201c5af152a3d6b767bb0ada841ff0 Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Wed, 20 Nov 2019 01:02:19 +0900 Subject: [PATCH 35/37] Delete augmetations --- augmentations.py | 60 ------------------------------------------------ 1 file changed, 60 deletions(-) delete mode 100644 augmentations.py diff --git a/augmentations.py b/augmentations.py deleted file mode 100644 index 94e1842..0000000 --- a/augmentations.py +++ /dev/null @@ -1,60 +0,0 @@ -import random -import numpy as np -import cv2 -from matplotlib import pyplot as plt -from albumentations import ( -HorizontalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90, - Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue, - IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, - IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose -) - - -def aug(p=0.5, p1=random.uniform(0, 1)): - return Compose([ - HorizontalFlip(True), - RandomRotate90(), - Flip(), - Transpose(), - OneOf([ - IAAAdditiveGaussianNoise(), - GaussNoise(), - ], p=0.2), - OneOf([ - MotionBlur(p=0.2), - MedianBlur(blur_limit=3, p=0.1), - Blur(blur_limit=3, p=0.1), - ], p=0.2), - ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=0.2), - OneOf([ - OpticalDistortion(p=0.3), - GridDistortion(p=0.1), - IAAPiecewiseAffine(p=0.3), - ], p=0.2), - OneOf([ - CLAHE(clip_limit=2), - IAASharpen(), - IAAEmboss(), - RandomBrightnessContrast(), - ], p=0.3), - HueSaturationValue(p=0.3), - ], p=p) - - -def show_img(img, figsize=(8, 8)): - fig, ax = plt.subplots(figsize=figsize) - ax.grid(False) - ax.set_yticklabels([]) - ax.set_xticklabels([]) - ax.imshow(img) - plt.imshow(img) - - -if __name__ == "__main__": - - image = cv2.imread('dog.12473.jpg') - augmentation = aug(p=0.5) - data = {'image': image} - augmented = augmentation(**data) - image = augmented['image'] - show_img(image) \ No newline at end of file From c47bf11cdca137f2e24b2bf2ab7485faff8d9386 Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Wed, 20 Nov 2019 01:13:35 +0900 Subject: [PATCH 36/37] fixed typo --- layers/BinaryLinear.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/layers/BinaryLinear.py b/layers/BinaryLinear.py index f6bf18b..ea2c026 100644 --- a/layers/BinaryLinear.py +++ b/layers/BinaryLinear.py @@ -5,7 +5,7 @@ class BinaryLinear(torch.nn.Linear): - def __init__(self, in_features, out_features, bias=True, mode="Stocastic"): + def __init__(self, in_features, out_features, bias=True, mode="Stochastic"): super().__init__(in_features, out_features, bias) self.mode = mode self.bin_weight = self.weight_binarization(self.weight, self.mode) @@ -17,8 +17,8 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: def weight_binarization(self, weight: torch.tensor, mode:str): with torch.set_grad_enabled(False): - if mode == "Stocastic": - bin_weight = self.stocastic(weight) + if mode == "Stochastic": + bin_weight = self.stochastic(weight) elif mode == "Deterministic": bin_weight = self.deterministic(weight) else: @@ -32,11 +32,7 @@ def deterministic(weight: torch.tensor) -> torch.tensor: return weight.sign() @staticmethod -<<<<<<< HEAD def stochastic(weight: torch.tensor) -> torch.tensor: -======= - def stocastic(weight: torch.tensor) -> torch.tensor: ->>>>>>> upstream/develop p = torch.sigmoid(weight) uniform_matrix = torch.empty(p.shape).uniform_(0,1) bin_weight = (p >= uniform_matrix).type(torch.float32) @@ -53,4 +49,4 @@ def clipping_weight(self, weight:torch.tensor) -> torch.tensor: with torch.set_grad_enabled(False): weight = torch.clamp(weight, -1, 1) weight.requires_grad = True - return weight + return weight \ No newline at end of file From 05e54fb060cec694cca86357b3b0e96a30f57132 Mon Sep 17 00:00:00 2001 From: Ronalmoo Date: Wed, 20 Nov 2019 01:14:06 +0900 Subject: [PATCH 37/37] fixed typo --- models/binarized_conv.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/models/binarized_conv.py b/models/binarized_conv.py index 5f71206..6463d1a 100644 --- a/models/binarized_conv.py +++ b/models/binarized_conv.py @@ -3,7 +3,7 @@ import torch import torch.nn as nn import torchvision.transforms as transforms -from augmentations import aug + from torchsummary import summary from torch.nn import functional as F from torch.utils.data import DataLoader @@ -77,15 +77,15 @@ def configure_optimizers(self): @pl.data_loader def train_dataloader(self): - return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=aug()), batch_size=128) + return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=128) @pl.data_loader def val_dataloader(self): - return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=aug()), batch_size=32) + return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32) @pl.data_loader def test_dataloader(self): - return DataLoader(MNIST(os.getcwd(), train=False, download=True, transform=aug()), batch_size=32) + return DataLoader(MNIST(os.getcwd(), train=False, download=True, transform=transforms.ToTensor()), batch_size=32) if __name__ == "__main__": @@ -114,4 +114,4 @@ def test_dataloader(self): trainer = Trainer(checkpoint_callback=checkpoint_callback, max_nb_epochs=1, train_percent_check=0.1) trainer.fit(model) - trainer.test(model) + trainer.test(model) \ No newline at end of file