diff --git a/Link to the dataset.txt b/Link to the dataset.txt new file mode 100644 index 0000000..d27ba8e --- /dev/null +++ b/Link to the dataset.txt @@ -0,0 +1 @@ +https://drive.google.com/open?id=16Jac8MeodgoiqA_YJ1_e8ICJMU_SFay6 \ No newline at end of file diff --git a/Predict.py b/Predict.py new file mode 100644 index 0000000..1359dff --- /dev/null +++ b/Predict.py @@ -0,0 +1,92 @@ + +# coding: utf-8 + +# In[101]: + + +import argparse +import numpy as np +import os +import pandas as pd +import tensorflow as tf +from keras.models import load_model +from keras.preprocessing.image import ImageDataGenerator + + +# In[102]: + + +# creating function for calculating the results +def predict_proba(model_path, in_folder, out_file): + + print('Start image processing...') + +# loading the model +model = load_model('second_try.h5') +batch_size=64 + + +# In[103]: + + +# define ImageDataGenerator +test_datagen = ImageDataGenerator(rescale=1. / 255) +test_generator = test_datagen.flow_from_directory( + directory='final_dataset_2/test', + target_size=(150, 150), + shuffle=False, + class_mode=None, + batch_size=batch_size) + + +# In[104]: + + +# getting filenames +filenames = [filename.split('\\')[-1] for filename in + test_generator.filenames] + + +# In[105]: + + +# getting predictions +nb_samples = len(filenames) +test_generator.reset() +preds = model.predict_generator(test_generator, + steps=nb_samples / batch_size, verbose=1) + + +# In[94]: + + +# creating dataframe with filenames and iphone probabilities +def results(filenames, preds, out_file): + output_data = pd.DataFrame() + output_data['image_name'] = filenames + output_data['iphone_probability'] = [pred[0] for pred in preds] + + # saving CSV with outputs + output_data.to_csv(out_file, index=False) + + +# In[99]: + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Iphone detector') + parser.add_argument('--model', type=str, default='second_try.h5', help='path to model') + parser.add_argument('--input', type=str, default='.final_dataset_2/test', help='path to folder with pictures') + parser.add_argument('--output', type=str, default='predictions.csv', help='path to file with model output') + + args = parser.parse_args(args=[]) + print("model= {0} input_data= {1} output_data= {2}".format(args.model, args.input, args.output)) + + +# In[107]: + + +predict_proba(args.model, args.input, args.output) +print() +print('Predictions file has been successfully created!') + diff --git a/README.md b/README.md deleted file mode 100644 index 5d8ac1a..0000000 --- a/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# iphoneOrNot - -You should to create model to detect iphone (all versions) on a picture. -If picture contains two or more iphones you should return only one probability for all picture. Picture is a typical for internet shop. -Solution should contains all ML stages (you can skip collect data stage) and -pretrained model in git or link to another data storage. -Also you should to provide example running inference. - - -## Data: -Collect data is a part of task. - - -## Restrictions: -* inference work on CPU -* neural net frameworks - keras, pytorch -* python3 -* work without internet - - -## Performance measure: -Area under precision and recall on hidden data. -Who will commit solution which better then random and don't copy pasted get 20 scores. -Another scores will depend on you rating based on hidden data. - - - -## Hard deadline: -June 1st - - -## Interface to detection - -python predict.py --model path_to_model --input path_to_input_data --output path_to_results - -input data - folder with images -output data - csv file with two columns: image_name,iphone_probability \ No newline at end of file diff --git a/iphoneOrNot_model.ipynb b/iphoneOrNot_model.ipynb new file mode 100644 index 0000000..5083f1a --- /dev/null +++ b/iphoneOrNot_model.ipynb @@ -0,0 +1,301 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "import zipfile\n", + "\n", + "zf = zipfile.ZipFile('final_dataset_2.zip')\n", + "zf.extractall('http://34.66.5.18:8888/files/')\n", + "zf.close()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "from keras.preprocessing.image import ImageDataGenerator\n", + "from keras.preprocessing.image import img_to_array, load_img\n", + "from keras.models import Sequential\n", + "from keras.layers import Conv2D, MaxPooling2D\n", + "from keras.layers import Activation, Dropout, Flatten, Dense\n", + "from keras import backend as K\n", + "\n", + "from PIL import Image" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# setting the dimensions of the images\n", + "img_width, img_height = 150, 150" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# data loading\n", + "train_data_dir = 'http://34.66.5.18:8888/files/final_dataset_2/train'\n", + "validation_data_dir = 'http://34.66.5.18:8888/files/final_dataset_2/test'\n", + "batch_size = 64" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "if K.image_data_format() == 'channels_first':\n", + " input_shape = (3, img_width, img_height)\n", + "else:\n", + " input_shape = (img_width, img_height, 3)\n", + "\n", + "model = Sequential()\n", + "model.add(Conv2D(32, (3, 3), input_shape=input_shape))\n", + "model.add(Activation('relu'))\n", + "model.add(MaxPooling2D(pool_size=(2, 2)))\n", + "\n", + "model.add(Conv2D(32, (3, 3)))\n", + "model.add(Activation('relu'))\n", + "model.add(MaxPooling2D(pool_size=(2, 2)))\n", + "\n", + "model.add(Conv2D(64, (3, 3)))\n", + "model.add(Activation('relu'))\n", + "model.add(MaxPooling2D(pool_size=(2, 2)))\n", + "\n", + "model.add(Flatten())\n", + "model.add(Dense(64))\n", + "model.add(Activation('relu'))\n", + "model.add(Dropout(0.5))\n", + "model.add(Dense(1))\n", + "model.add(Activation('sigmoid'))" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "conv2d_1 (Conv2D) (None, 148, 148, 32) 896 \n", + "_________________________________________________________________\n", + "activation_1 (Activation) (None, 148, 148, 32) 0 \n", + "_________________________________________________________________\n", + "max_pooling2d_1 (MaxPooling2 (None, 74, 74, 32) 0 \n", + "_________________________________________________________________\n", + "conv2d_2 (Conv2D) (None, 72, 72, 32) 9248 \n", + "_________________________________________________________________\n", + "activation_2 (Activation) (None, 72, 72, 32) 0 \n", + "_________________________________________________________________\n", + "max_pooling2d_2 (MaxPooling2 (None, 36, 36, 32) 0 \n", + "_________________________________________________________________\n", + "conv2d_3 (Conv2D) (None, 34, 34, 64) 18496 \n", + "_________________________________________________________________\n", + "activation_3 (Activation) (None, 34, 34, 64) 0 \n", + "_________________________________________________________________\n", + "max_pooling2d_3 (MaxPooling2 (None, 17, 17, 64) 0 \n", + "_________________________________________________________________\n", + "flatten_1 (Flatten) (None, 18496) 0 \n", + "_________________________________________________________________\n", + "dense_1 (Dense) (None, 64) 1183808 \n", + "_________________________________________________________________\n", + "activation_4 (Activation) (None, 64) 0 \n", + "_________________________________________________________________\n", + "dropout_1 (Dropout) (None, 64) 0 \n", + "_________________________________________________________________\n", + "dense_2 (Dense) (None, 1) 65 \n", + "_________________________________________________________________\n", + "activation_5 (Activation) (None, 1) 0 \n", + "=================================================================\n", + "Total params: 1,212,513\n", + "Trainable params: 1,212,513\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n" + ] + } + ], + "source": [ + "model.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "model.compile(loss='binary_crossentropy',\n", + " optimizer='adam',\n", + " metrics=['accuracy'])" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "# data augmentation\n", + "train_datagen = ImageDataGenerator(\n", + " rescale=1./255,\n", + " shear_range=0.2,\n", + " zoom_range=0.2,\n", + " horizontal_flip=True)\n", + "\n", + "test_datagen = ImageDataGenerator(rescale=1./255)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found 29760 images belonging to 2 classes.\n", + "Found 7439 images belonging to 2 classes.\n" + ] + } + ], + "source": [ + "# data preparation\n", + "train_generator = train_datagen.flow_from_directory(\n", + " train_data_dir,\n", + " target_size=(img_width, img_height),\n", + " batch_size=batch_size,\n", + " class_mode='binary')\n", + "\n", + "validation_generator = test_datagen.flow_from_directory(\n", + " validation_data_dir,\n", + " target_size=(img_width, img_height),\n", + " batch_size=batch_size,\n", + " class_mode='binary',\n", + " shuffle=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/20\n", + "465/465 [==============================] - 217s 466ms/step - loss: 0.4617 - acc: 0.7844 - val_loss: 0.5004 - val_acc: 0.7311\n", + "Epoch 2/20\n", + "465/465 [==============================] - 210s 452ms/step - loss: 0.3712 - acc: 0.8415 - val_loss: 0.6034 - val_acc: 0.6834\n", + "Epoch 3/20\n", + "465/465 [==============================] - 212s 456ms/step - loss: 0.3108 - acc: 0.8777 - val_loss: 0.8289 - val_acc: 0.6182\n", + "Epoch 4/20\n", + "465/465 [==============================] - 210s 451ms/step - loss: 0.2679 - acc: 0.8996 - val_loss: 0.4968 - val_acc: 0.8032\n", + "Epoch 5/20\n", + "465/465 [==============================] - 209s 449ms/step - loss: 0.2331 - acc: 0.9143 - val_loss: 0.7307 - val_acc: 0.7065\n", + "Epoch 6/20\n", + "465/465 [==============================] - 211s 453ms/step - loss: 0.2070 - acc: 0.9263 - val_loss: 0.8482 - val_acc: 0.6924\n", + "Epoch 7/20\n", + "465/465 [==============================] - 210s 452ms/step - loss: 0.1847 - acc: 0.9362 - val_loss: 0.5117 - val_acc: 0.7856\n", + "Epoch 8/20\n", + "465/465 [==============================] - 209s 449ms/step - loss: 0.1644 - acc: 0.9433 - val_loss: 0.4291 - val_acc: 0.8395\n", + "Epoch 9/20\n", + "465/465 [==============================] - 209s 450ms/step - loss: 0.1461 - acc: 0.9508 - val_loss: 0.2944 - val_acc: 0.8938\n", + "Epoch 10/20\n", + "465/465 [==============================] - 210s 451ms/step - loss: 0.1364 - acc: 0.9534 - val_loss: 0.4258 - val_acc: 0.8528\n", + "Epoch 11/20\n", + "465/465 [==============================] - 209s 450ms/step - loss: 0.1278 - acc: 0.9583 - val_loss: 0.3789 - val_acc: 0.8420\n", + "Epoch 12/20\n", + "465/465 [==============================] - 209s 450ms/step - loss: 0.1194 - acc: 0.9610 - val_loss: 0.3668 - val_acc: 0.8415\n", + "Epoch 13/20\n", + "465/465 [==============================] - 211s 453ms/step - loss: 0.1186 - acc: 0.9590 - val_loss: 0.3316 - val_acc: 0.8704\n", + "Epoch 14/20\n", + "465/465 [==============================] - 209s 449ms/step - loss: 0.1131 - acc: 0.9620 - val_loss: 0.4434 - val_acc: 0.8252\n", + "Epoch 15/20\n", + "465/465 [==============================] - 210s 451ms/step - loss: 0.1034 - acc: 0.9670 - val_loss: 0.3758 - val_acc: 0.8609\n", + "Epoch 16/20\n", + "465/465 [==============================] - 211s 453ms/step - loss: 0.1077 - acc: 0.9644 - val_loss: 0.2141 - val_acc: 0.9273\n", + "Epoch 17/20\n", + "465/465 [==============================] - 210s 451ms/step - loss: 0.0991 - acc: 0.9675 - val_loss: 0.2799 - val_acc: 0.8927\n", + "Epoch 18/20\n", + "465/465 [==============================] - 211s 454ms/step - loss: 0.0953 - acc: 0.9685 - val_loss: 0.6787 - val_acc: 0.7381\n", + "Epoch 19/20\n", + "465/465 [==============================] - 211s 454ms/step - loss: 0.0932 - acc: 0.9689 - val_loss: 0.3931 - val_acc: 0.8324\n", + "Epoch 20/20\n", + "465/465 [==============================] - 212s 456ms/step - loss: 0.0909 - acc: 0.9693 - val_loss: 0.2353 - val_acc: 0.9103\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# training\n", + "nb_epochs = 20\n", + "\n", + "model.fit_generator(\n", + " train_generator,\n", + " steps_per_epoch=train_generator.samples // batch_size,\n", + " validation_data=validation_generator, \n", + " validation_steps=validation_generator.samples // batch_size,\n", + " epochs=nb_epochs)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "model.save('second_try.h5')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/predict.py b/predict.py deleted file mode 100644 index 96bdb3d..0000000 --- a/predict.py +++ /dev/null @@ -1,15 +0,0 @@ -import argparse - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Iphone detector') - parser.add_argument('--model', type=str, help='path to model') - parser.add_argument('--input', type=str, help='path to folder with pictures') - parser.add_argument('--output', type=str, help='path to file with model output') - - args = parser.parse_args() - - print("model= {0} input_data= {1} output_data= {2}".format(args.model, args.input, args.output)) - - - diff --git a/second_try.h5 b/second_try.h5 new file mode 100644 index 0000000..17de489 Binary files /dev/null and b/second_try.h5 differ