Skip to content

Format Code #41

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
WGierke wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Format Code #41

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@
*.ipynb*
training/.flag*
prediction*.csv
.idea
26 changes: 13 additions & 13 deletions config_submit.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
config = {'datapath':'/work/DataBowl3/stage2/stage2/',
'preprocess_result_path':'./prep_result/',
'outputfile':'prediction.csv',
'detector_model':'net_detector',
'detector_param':'./model/detector.ckpt',
'classifier_model':'net_classifier',
'classifier_param':'./model/classifier.ckpt',
'n_gpu':8,
'n_worker_preprocessing':None,
'use_exsiting_preprocessing':False,
'skip_preprocessing':False,
'skip_detect':False}
config = {'datapath': '/work/DataBowl3/stage2/stage2/',
'preprocess_result_path': './prep_result/',
'outputfile': 'prediction.csv',

'detector_model': 'net_detector',
'detector_param': './model/detector.ckpt',
'classifier_model': 'net_classifier',
'classifier_param': './model/classifier.ckpt',
'n_gpu': 8,
'n_worker_preprocessing': None,
'use_exsiting_preprocessing': False,
'skip_preprocessing': False,
'skip_detect': False}
227 changes: 115 additions & 112 deletions data_classifier.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,51 +1,49 @@
import numpy as np
import torch
from torch.utils.data import Dataset
import os
import time
import collections
import random
from layers import iou
from scipy.ndimage import zoom
import warnings

import numpy as np
import torch
from scipy.ndimage import zoom
from scipy.ndimage.interpolation import rotate
from layers import nms,iou
import pandas
from torch.utils.data import Dataset

from layers import nms, iou


class DataBowl3Classifier(Dataset):
def __init__(self, split, config, phase = 'train'):
assert(phase == 'train' or phase == 'val' or phase == 'test')
def __init__(self, split, config, phase='train'):
assert (phase == 'train' or phase == 'val' or phase == 'test')

self.random_sample = config['random_sample']
self.T = config['T']
self.topk = config['topk']
self.crop_size = config['crop_size']
self.stride = config['stride']
self.augtype = config['augtype']
self.augtype = config['augtype']
self.filling_value = config['filling_value']
#self.labels = np.array(pandas.read_csv(config['labelfile']))

# self.labels = np.array(pandas.read_csv(config['labelfile']))

datadir = config['datadir']
bboxpath = config['bboxpath']
bboxpath = config['bboxpath']
self.phase = phase
self.candidate_box = []
self.pbb_label = []

idcs = split
self.filenames = [os.path.join(datadir, '%s_clean.npy' % idx.split('-')[0]) for idx in idcs]
if self.phase!='test':
self.yset = 1-np.array([f.split('-')[1][2] for f in idcs]).astype('int')


if self.phase != 'test':
self.yset = 1 - np.array([f.split('-')[1][2] for f in idcs]).astype('int')

for idx in idcs:
pbb = np.load(os.path.join(bboxpath,idx+'_pbb.npy'))
pbb = pbb[pbb[:,0]>config['conf_th']]
pbb = np.load(os.path.join(bboxpath, idx + '_pbb.npy'))
pbb = pbb[pbb[:, 0] > config['conf_th']]
pbb = nms(pbb, config['nms_th'])
lbb = np.load(os.path.join(bboxpath,idx+'_lbb.npy'))

lbb = np.load(os.path.join(bboxpath, idx + '_lbb.npy'))
pbb_label = []

for p in pbb:
isnod = False
for l in lbb:
Expand All @@ -54,166 +52,171 @@ def __init__(self, split, config, phase = 'train'):
isnod = True
break
pbb_label.append(isnod)
# if idx.startswith()
# if idx.startswith()
self.candidate_box.append(pbb)
self.pbb_label.append(np.array(pbb_label))
self.crop = simpleCrop(config,phase)

self.crop = simpleCrop(config, phase)

def __getitem__(self, idx,split=None):
def __getitem__(self, idx, split=None):
t = time.time()
np.random.seed(int(str(t%1)[2:7]))#seed according to time
np.random.seed(int(str(t % 1)[2:7])) # seed according to time

pbb = self.candidate_box[idx]
pbb_label = self.pbb_label[idx]
conf_list = pbb[:,0]
conf_list = pbb[:, 0]
T = self.T
topk = self.topk
img = np.load(self.filenames[idx])
if self.random_sample and self.phase=='train':
chosenid = sample(conf_list,topk,T=T)
#chosenid = conf_list.argsort()[::-1][:topk]
if self.random_sample and self.phase == 'train':
chosenid = sample(conf_list, topk, T=T)
# chosenid = conf_list.argsort()[::-1][:topk]
else:
chosenid = conf_list.argsort()[::-1][:topk]
croplist = np.zeros([topk,1,self.crop_size[0],self.crop_size[1],self.crop_size[2]]).astype('float32')
coordlist = np.zeros([topk,3,self.crop_size[0]/self.stride,self.crop_size[1]/self.stride,self.crop_size[2]/self.stride]).astype('float32')
padmask = np.concatenate([np.ones(len(chosenid)),np.zeros(self.topk-len(chosenid))])
croplist = np.zeros([topk, 1, self.crop_size[0], self.crop_size[1], self.crop_size[2]]).astype('float32')
coordlist = np.zeros([topk, 3, self.crop_size[0] / self.stride, self.crop_size[1] / self.stride,
self.crop_size[2] / self.stride]).astype('float32')
padmask = np.concatenate([np.ones(len(chosenid)), np.zeros(self.topk - len(chosenid))])
isnodlist = np.zeros([topk])


for i,id in enumerate(chosenid):
target = pbb[id,1:]
for i, id in enumerate(chosenid):
target = pbb[id, 1:]
isnod = pbb_label[id]
crop,coord = self.crop(img,target)
if self.phase=='train':
crop,coord = augment(crop,coord,
ifflip=self.augtype['flip'],ifrotate=self.augtype['rotate'],
ifswap = self.augtype['swap'],filling_value = self.filling_value)
crop, coord = self.crop(img, target)
if self.phase == 'train':
crop, coord = augment(crop, coord,
ifflip=self.augtype['flip'], ifrotate=self.augtype['rotate'],
ifswap=self.augtype['swap'], filling_value=self.filling_value)
crop = crop.astype(np.float32)
croplist[i] = crop
coordlist[i] = coord
isnodlist[i] = isnod
if self.phase!='test':

if self.phase != 'test':
y = np.array([self.yset[idx]])
return torch.from_numpy(croplist).float(), torch.from_numpy(coordlist).float(), torch.from_numpy(isnodlist).int(), torch.from_numpy(y)
return torch.from_numpy(croplist).float(), torch.from_numpy(coordlist).float(), torch.from_numpy(
isnodlist).int(), torch.from_numpy(y)
else:
return torch.from_numpy(croplist).float(), torch.from_numpy(coordlist).float()

def __len__(self):
if self.phase != 'test':
return len(self.candidate_box)
else:
return len(self.candidate_box)



class simpleCrop():
def __init__(self,config,phase):
def __init__(self, config, phase):
self.crop_size = config['crop_size']
self.scaleLim = config['scaleLim']
self.radiusLim = config['radiusLim']
self.jitter_range = config['jitter_range']
self.isScale = config['augtype']['scale'] and phase=='train'
self.isScale = config['augtype']['scale'] and phase == 'train'
self.stride = config['stride']
self.filling_value = config['filling_value']
self.phase = phase
def __call__(self,imgs,target):

def __call__(self, imgs, target):
if self.isScale:
radiusLim = self.radiusLim
scaleLim = self.scaleLim
scaleRange = [np.min([np.max([(radiusLim[0]/target[3]),scaleLim[0]]),1])
,np.max([np.min([(radiusLim[1]/target[3]),scaleLim[1]]),1])]
scale = np.random.rand()*(scaleRange[1]-scaleRange[0])+scaleRange[0]
crop_size = (np.array(self.crop_size).astype('float')/scale).astype('int')
scaleRange = [np.min([np.max([(radiusLim[0] / target[3]), scaleLim[0]]), 1])
, np.max([np.min([(radiusLim[1] / target[3]), scaleLim[1]]), 1])]
scale = np.random.rand() * (scaleRange[1] - scaleRange[0]) + scaleRange[0]
crop_size = (np.array(self.crop_size).astype('float') / scale).astype('int')
else:
crop_size = np.array(self.crop_size).astype('int')
if self.phase=='train':
jitter_range = target[3]*self.jitter_range
jitter = (np.random.rand(3)-0.5)*jitter_range
if self.phase == 'train':
jitter_range = target[3] * self.jitter_range
jitter = (np.random.rand(3) - 0.5) * jitter_range
else:
jitter = 0
start = (target[:3]- crop_size/2 + jitter).astype('int')
pad = [[0,0]]
start = (target[:3] - crop_size / 2 + jitter).astype('int')
pad = [[0, 0]]
for i in range(3):
if start[i]<0:
if start[i] < 0:
leftpad = -start[i]
start[i] = 0
else:
leftpad = 0
if start[i]+crop_size[i]>imgs.shape[i+1]:
rightpad = start[i]+crop_size[i]-imgs.shape[i+1]
if start[i] + crop_size[i] > imgs.shape[i + 1]:
rightpad = start[i] + crop_size[i] - imgs.shape[i + 1]
else:
rightpad = 0
pad.append([leftpad,rightpad])
imgs = np.pad(imgs,pad,'constant',constant_values =self.filling_value)
crop = imgs[:,start[0]:start[0]+crop_size[0],start[1]:start[1]+crop_size[1],start[2]:start[2]+crop_size[2]]

normstart = np.array(start).astype('float32')/np.array(imgs.shape[1:])-0.5
normsize = np.array(crop_size).astype('float32')/np.array(imgs.shape[1:])
xx,yy,zz = np.meshgrid(np.linspace(normstart[0],normstart[0]+normsize[0],self.crop_size[0]/self.stride),
np.linspace(normstart[1],normstart[1]+normsize[1],self.crop_size[1]/self.stride),
np.linspace(normstart[2],normstart[2]+normsize[2],self.crop_size[2]/self.stride),indexing ='ij')
coord = np.concatenate([xx[np.newaxis,...], yy[np.newaxis,...],zz[np.newaxis,:]],0).astype('float32')
pad.append([leftpad, rightpad])
imgs = np.pad(imgs, pad, 'constant', constant_values=self.filling_value)
crop = imgs[:, start[0]:start[0] + crop_size[0], start[1]:start[1] + crop_size[1],
start[2]:start[2] + crop_size[2]]

normstart = np.array(start).astype('float32') / np.array(imgs.shape[1:]) - 0.5
normsize = np.array(crop_size).astype('float32') / np.array(imgs.shape[1:])
xx, yy, zz = np.meshgrid(np.linspace(normstart[0], normstart[0] + normsize[0], self.crop_size[0] / self.stride),
np.linspace(normstart[1], normstart[1] + normsize[1], self.crop_size[1] / self.stride),
np.linspace(normstart[2], normstart[2] + normsize[2], self.crop_size[2] / self.stride),
indexing='ij')
coord = np.concatenate([xx[np.newaxis, ...], yy[np.newaxis, ...], zz[np.newaxis, :]], 0).astype('float32')

if self.isScale:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
crop = zoom(crop,[1,scale,scale,scale],order=1)
newpad = self.crop_size[0]-crop.shape[1:][0]
if newpad<0:
crop = crop[:,:-newpad,:-newpad,:-newpad]
elif newpad>0:
pad2 = [[0,0],[0,newpad],[0,newpad],[0,newpad]]
crop = np.pad(crop,pad2,'constant',constant_values =self.filling_value)

return crop,coord

def sample(conf,N,T=1):
if len(conf)>N:
crop = zoom(crop, [1, scale, scale, scale], order=1)
newpad = self.crop_size[0] - crop.shape[1:][0]
if newpad < 0:
crop = crop[:, :-newpad, :-newpad, :-newpad]
elif newpad > 0:
pad2 = [[0, 0], [0, newpad], [0, newpad], [0, newpad]]
crop = np.pad(crop, pad2, 'constant', constant_values=self.filling_value)

return crop, coord


def sample(conf, N, T=1):
if len(conf) > N:
target = range(len(conf))
chosen_list = []
for i in range(N):
chosenidx = sampleone(target,conf,T)
chosenidx = sampleone(target, conf, T)
chosen_list.append(target[chosenidx])
target.pop(chosenidx)
conf = np.delete(conf, chosenidx)


return chosen_list
else:
return np.arange(len(conf))

def sampleone(target,conf,T):
assert len(conf)>1
p = softmax(conf/T)
p = np.max([np.ones_like(p)*0.00001,p],axis=0)
p = p/np.sum(p)
return np.random.choice(np.arange(len(target)),size=1,replace = False, p=p)[0]

def sampleone(target, conf, T):
assert len(conf) > 1
p = softmax(conf / T)
p = np.max([np.ones_like(p) * 0.00001, p], axis=0)
p = p / np.sum(p)
return np.random.choice(np.arange(len(target)), size=1, replace=False, p=p)[0]


def softmax(x):
maxx = np.max(x)
return np.exp(x-maxx)/np.sum(np.exp(x-maxx))
return np.exp(x - maxx) / np.sum(np.exp(x - maxx))


def augment(sample, coord, ifflip = True, ifrotate=True, ifswap = True,filling_value=0):
def augment(sample, coord, ifflip=True, ifrotate=True, ifswap=True, filling_value=0):
# angle1 = np.random.rand()*180
if ifrotate:
validrot = False
counter = 0
angle1 = np.random.rand()*180
angle1 = np.random.rand() * 180
size = np.array(sample.shape[2:4]).astype('float')
rotmat = np.array([[np.cos(angle1/180*np.pi),-np.sin(angle1/180*np.pi)],[np.sin(angle1/180*np.pi),np.cos(angle1/180*np.pi)]])
sample = rotate(sample,angle1,axes=(2,3),reshape=False,cval=filling_value)

rotmat = np.array([[np.cos(angle1 / 180 * np.pi), -np.sin(angle1 / 180 * np.pi)],
[np.sin(angle1 / 180 * np.pi), np.cos(angle1 / 180 * np.pi)]])
sample = rotate(sample, angle1, axes=(2, 3), reshape=False, cval=filling_value)

if ifswap:
if sample.shape[1]==sample.shape[2] and sample.shape[1]==sample.shape[3]:
if sample.shape[1] == sample.shape[2] and sample.shape[1] == sample.shape[3]:
axisorder = np.random.permutation(3)
sample = np.transpose(sample,np.concatenate([[0],axisorder+1]))
coord = np.transpose(coord,np.concatenate([[0],axisorder+1]))
sample = np.transpose(sample, np.concatenate([[0], axisorder + 1]))
coord = np.transpose(coord, np.concatenate([[0], axisorder + 1]))

if ifflip:
flipid = np.array([np.random.randint(2),np.random.randint(2),np.random.randint(2)])*2-1
sample = np.ascontiguousarray(sample[:,::flipid[0],::flipid[1],::flipid[2]])
coord = np.ascontiguousarray(coord[:,::flipid[0],::flipid[1],::flipid[2]])
return sample, coord
flipid = np.array([np.random.randint(2), np.random.randint(2), np.random.randint(2)]) * 2 - 1
sample = np.ascontiguousarray(sample[:, ::flipid[0], ::flipid[1], ::flipid[2]])
coord = np.ascontiguousarray(coord[:, ::flipid[0], ::flipid[1], ::flipid[2]])
return sample, coord
Loading