diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..135767f --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk diff --git a/bevstratov/iphone_model.ipynb b/bevstratov/iphone_model.ipynb new file mode 100644 index 0000000..bb68b21 --- /dev/null +++ b/bevstratov/iphone_model.ipynb @@ -0,0 +1,1207 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "iphone-model.ipynb", + "version": "0.3.2", + "provenance": [], + "toc_visible": true + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.3" + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "B63VqmmTz4nA", + "colab_type": "text" + }, + "source": [ + "# Final Project: iPhone or Not\n", + "*Boris Evstratov*" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0wSBJLXlkMCK", + "colab_type": "text" + }, + "source": [ + "### 0. Importing packages" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "hgXp4iXrkMCM", + "colab_type": "code", + "colab": {} + }, + "source": [ + "from keras import models, layers\n", + "from keras.layers import Input, Add, Dense, Activation, ZeroPadding2D, BatchNormalization, Flatten, Conv2D, AveragePooling2D, MaxPooling2D, GlobalMaxPooling2D\n", + "from keras.models import Model, load_model, Sequential\n", + "from keras.preprocessing.image import ImageDataGenerator\n", + "from keras.callbacks import ModelCheckpoint\n", + "from keras.initializers import glorot_uniform\n", + "from keras.utils import to_categorical\n", + "from sklearn.metrics import average_precision_score, precision_recall_curve\n", + "import tensorflow as tf\n", + "from tensorflow.contrib import lite\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import keras.backend as K\n", + "K.set_image_data_format('channels_last')\n", + "K.set_learning_phase(1)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GedFXX-nkMCW", + "colab_type": "text" + }, + "source": [ + "### 1. Creating a Residual Network\n", + "*source*: [Hitchhiker’s Guide to Residual Networks (ResNet) in Keras](https://towardsdatascience.com/hitchhikers-guide-to-residual-networks-resnet-in-keras-385ec01ec8ff)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "V7koQNSikMCX", + "colab_type": "text" + }, + "source": [ + "#### 1.1 Define the identity block" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "HTSaaDhXkMCZ", + "colab_type": "code", + "colab": {} + }, + "source": [ + "def identity_block(X, f, filters, stage, block):\n", + " \n", + " # Defining name basis\n", + " conv_name_base = 'res' + str(stage) + block + '_branch'\n", + " bn_name_base = 'bn' + str(stage) + block + '_branch'\n", + " \n", + " # Retrieve Filters\n", + " F1, F2, F3 = filters\n", + " \n", + " # Save the input value\n", + " X_shortcut = X\n", + " \n", + " # First component of main path\n", + " X = Conv2D(filters = F1, kernel_size = (1, 1), strides = (1,1), padding = 'valid', name = conv_name_base + '2a', kernel_initializer = glorot_uniform(seed=0))(X)\n", + " X = BatchNormalization(axis = 3, name = bn_name_base + '2a')(X)\n", + " X = Activation('relu')(X)\n", + " \n", + " # Second component of main path\n", + " X = Conv2D(filters = F2, kernel_size = (f, f), strides = (1, 1), padding = 'same', name = conv_name_base + '2b', kernel_initializer = glorot_uniform(seed=0))(X)\n", + " X = BatchNormalization(axis = 3, name = bn_name_base + '2b')(X)\n", + " X = Activation('relu')(X)\n", + "\n", + " # Third component of main path \n", + " X = Conv2D(filters = F3, kernel_size = (1, 1), strides = (1, 1), padding = 'valid', name = conv_name_base + '2c', kernel_initializer = glorot_uniform(seed=0))(X)\n", + " X = BatchNormalization(axis = 3, name = bn_name_base + '2c')(X)\n", + "\n", + " # Final step: Add shortcut value to main path, and pass it through a RELU activation\n", + " X = Add()([X, X_shortcut])\n", + " X = Activation('relu')(X)\n", + " \n", + " return X" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KpudnWBAkMCb", + "colab_type": "text" + }, + "source": [ + "#### 1.2 Define the convolution block" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "5jlbKIE9kMCc", + "colab_type": "code", + "colab": {} + }, + "source": [ + "def convolutional_block(X, f, filters, stage, block, s=2):\n", + "\n", + " # Defining name basis\n", + " conv_name_base = 'res' + str(stage) + block + '_branch'\n", + " bn_name_base = 'bn' + str(stage) + block + '_branch'\n", + "\n", + " # Retrieve Filters\n", + " F1, F2, F3 = filters\n", + "\n", + " # Save the input value\n", + " X_shortcut = X\n", + "\n", + " ##### MAIN PATH #####\n", + " # First component of main path \n", + " X = Conv2D(filters=F1, kernel_size=(1, 1), strides=(s, s), padding='valid', name=conv_name_base + '2a', kernel_initializer=glorot_uniform(seed=0))(X)\n", + " X = BatchNormalization(axis=3, name=bn_name_base + '2a')(X)\n", + " X = Activation('relu')(X)\n", + "\n", + " # Second component of main path\n", + " X = Conv2D(filters=F2, kernel_size=(f, f), strides=(1, 1), padding='same', name=conv_name_base + '2b', kernel_initializer=glorot_uniform(seed=0))(X)\n", + " X = BatchNormalization(axis=3, name=bn_name_base + '2b')(X)\n", + " X = Activation('relu')(X)\n", + "\n", + " # Third component of main path\n", + " X = Conv2D(filters=F3, kernel_size=(1, 1), strides=(1, 1), padding='valid', name=conv_name_base + '2c', kernel_initializer=glorot_uniform(seed=0))(X)\n", + " X = BatchNormalization(axis=3, name=bn_name_base + '2c')(X)\n", + "\n", + " ##### SHORTCUT PATH #### \n", + " X_shortcut = Conv2D(filters=F3, kernel_size=(1, 1), strides=(s, s), padding='valid', name=conv_name_base + '1', kernel_initializer=glorot_uniform(seed=0))(X_shortcut)\n", + " X_shortcut = BatchNormalization(axis=3, name=bn_name_base + '1')(X_shortcut)\n", + "\n", + " # Final step: Add shortcut value to main path, and pass it through a RELU activation\n", + " X = Add()([X, X_shortcut])\n", + " X = Activation('relu')(X)\n", + "\n", + " return X" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mBP1hQ0SkMCf", + "colab_type": "text" + }, + "source": [ + "#### 1.3 Biuld the model\n", + "Combining both blocks into a 50-layer residual network" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "qGbDH4HakMCg", + "colab_type": "code", + "colab": {} + }, + "source": [ + "def ResNet50(input_shape = (64, 64, 3), classes = 6):\n", + " \n", + " # Define the input as a tensor with shape input_shape\n", + " X_input = Input(input_shape)\n", + " \n", + " # Zero-Padding\n", + " X = ZeroPadding2D((3, 3))(X_input)\n", + " \n", + " # Stage 1\n", + " X = Conv2D(64, (7, 7), strides = (2, 2), name = 'conv1', kernel_initializer = glorot_uniform(seed=0))(X)\n", + " X = BatchNormalization(axis = 3, name = 'bn_conv1')(X)\n", + " X = Activation('relu')(X)\n", + " X = MaxPooling2D((3, 3), strides=(2, 2))(X)\n", + "\n", + " # Stage 2\n", + " X = convolutional_block(X, f = 3, filters = [64, 64, 256], stage = 2, block='a', s = 1)\n", + " X = identity_block(X, 3, [64, 64, 256], stage=2, block='b')\n", + " X = identity_block(X, 3, [64, 64, 256], stage=2, block='c')\n", + "\n", + " # Stage 3\n", + " X = convolutional_block(X, f=3, filters=[128, 128, 512], stage=3, block='a', s=2)\n", + " X = identity_block(X, 3, [128, 128, 512], stage=3, block='b')\n", + " X = identity_block(X, 3, [128, 128, 512], stage=3, block='c')\n", + " X = identity_block(X, 3, [128, 128, 512], stage=3, block='d')\n", + "\n", + " # Stage 4\n", + " X = convolutional_block(X, f=3, filters=[256, 256, 1024], stage=4, block='a', s=2)\n", + " X = identity_block(X, 3, [256, 256, 1024], stage=4, block='b')\n", + " X = identity_block(X, 3, [256, 256, 1024], stage=4, block='c')\n", + " X = identity_block(X, 3, [256, 256, 1024], stage=4, block='d')\n", + " X = identity_block(X, 3, [256, 256, 1024], stage=4, block='e')\n", + " X = identity_block(X, 3, [256, 256, 1024], stage=4, block='f')\n", + "\n", + " # Stage 5\n", + " X = convolutional_block(X, f=3, filters=[512, 512, 2048], stage=5, block='a', s=2)\n", + " X = identity_block(X, 3, [512, 512, 2048], stage=5, block='b')\n", + " X = identity_block(X, 3, [512, 512, 2048], stage=5, block='c')\n", + "\n", + " # AVGPOOL\n", + " X = AveragePooling2D(pool_size=(2,2), padding='same')(X)\n", + "\n", + " # Output layer\n", + " X = Flatten()(X)\n", + " X = Dense(classes, activation='softmax', name='fc' + str(classes), kernel_initializer = glorot_uniform(seed=0))(X)\n", + " \n", + " \n", + " # Create model\n", + " model = Model(inputs = X_input, outputs = X, name='ResNet50')\n", + "\n", + " return model" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fNmfrSDJkMCi", + "colab_type": "text" + }, + "source": [ + "### 2. Data management" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OgCZICUfkMCj", + "colab_type": "text" + }, + "source": [ + "#### 2.1 Data import" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "NnEDmOwckMCj", + "colab_type": "code", + "colab": {} + }, + "source": [ + "# Unzipping uploaded dataset\n", + "from zipfile import ZipFile\n", + "zip = ZipFile('data.zip')\n", + "zip.extractall()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "qz_T6OQmkMCl", + "colab_type": "code", + "colab": {} + }, + "source": [ + "# Specifying data paths and necessary parameters\n", + "dataset_path = 'data'\n", + "train_path = dataset_path + '/train'\n", + "test_path = dataset_path + '/test'\n", + "image_res = 224\n", + "batch_size = 32\n", + "epochs_nb = 10" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qblS4mg6kMCm", + "colab_type": "text" + }, + "source": [ + "#### 2.2 Data preprocessing" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "XOs6jH5ykMCm", + "colab_type": "code", + "colab": {} + }, + "source": [ + "# Setting ImageDataGenerators\n", + "train_datagen = ImageDataGenerator(brightness_range = [0.3, 1.5], \n", + " channel_shift_range = 30,\n", + " rescale = 1./255,\n", + " rotation_range = 90,\n", + " shear_range = 0.3,\n", + " zoom_range = 0.2,\n", + " vertical_flip = True,\n", + " horizontal_flip = True)\n", + "\n", + "test_datagen = ImageDataGenerator(rescale = 1./255)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "G-dcX5FFkMCo", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 51 + }, + "outputId": "1680dfbb-af7d-440f-9ded-8afbc6b4bb79" + }, + "source": [ + "# Generating Flows from directories\n", + "train_set = train_datagen.flow_from_directory(train_path,\n", + " target_size = (image_res, image_res),\n", + " batch_size = batch_size,\n", + " class_mode = 'binary')\n", + "\n", + "test_set = test_datagen.flow_from_directory(test_path,\n", + " target_size = (image_res, image_res),\n", + " batch_size = batch_size,\n", + " class_mode = 'binary',\n", + " shuffle=False)" + ], + "execution_count": 8, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Found 24457 images belonging to 2 classes.\n", + "Found 6884 images belonging to 2 classes.\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "j2B34T4TkMCq", + "colab_type": "text" + }, + "source": [ + "### 3. Model training" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "oHdUxkGjkMCq", + "colab_type": "code", + "colab": {} + }, + "source": [ + "model = ResNet50(input_shape = (image_res, image_res, 3), classes = 2)\n", + "model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "ve8UynGFkMCr", + "colab_type": "code", + "colab": {} + }, + "source": [ + "# Define checkpoints for CNN autosaving\n", + "filepath= \"cnn-cv-iphone-resnet-50.hdf5\"\n", + "checkpoint = ModelCheckpoint(filepath, monitor='acc', verbose=1, save_best_only=True, mode='max', save_weights_only=False)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "-LUgXHfwkMCs", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1734 + }, + "outputId": "e9931c63-dde5-4130-cf07-fb5701ab431f" + }, + "source": [ + "# Training the model\n", + "model.fit_generator(train_set,\n", + " steps_per_epoch = train_set.samples / batch_size,\n", + " epochs = epochs_nb,\n", + " validation_data = test_set,\n", + " validation_steps = test_set.samples / batch_size,\n", + " verbose = 1,\n", + " callbacks=[checkpoint])" + ], + "execution_count": 13, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 1/25\n", + "765/764 [==============================] - 428s 559ms/step - loss: 11.3266 - acc: 0.2895 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00001: acc improved from -inf to 0.28949, saving model to cnn-cv-iphone-resnet-50.hdf5\n", + "Epoch 2/25\n", + "765/764 [==============================] - 425s 555ms/step - loss: 11.2094 - acc: 0.2969 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00002: acc improved from 0.28949 to 0.29685, saving model to cnn-cv-iphone-resnet-50.hdf5\n", + "Epoch 3/25\n", + "765/764 [==============================] - 423s 553ms/step - loss: 11.3027 - acc: 0.2910 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00003: acc did not improve from 0.29685\n", + "Epoch 4/25\n", + "765/764 [==============================] - 422s 552ms/step - loss: 11.2417 - acc: 0.2949 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00004: acc did not improve from 0.29685\n", + "Epoch 5/25\n", + "765/764 [==============================] - 419s 547ms/step - loss: 11.3286 - acc: 0.2894 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00005: acc did not improve from 0.29685\n", + "Epoch 6/25\n", + "765/764 [==============================] - 421s 551ms/step - loss: 11.3067 - acc: 0.2908 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00006: acc did not improve from 0.29685\n", + "Epoch 7/25\n", + "765/764 [==============================] - 422s 552ms/step - loss: 11.2798 - acc: 0.2925 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00007: acc did not improve from 0.29685\n", + "Epoch 8/25\n", + "765/764 [==============================] - 422s 552ms/step - loss: 11.2501 - acc: 0.2943 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00008: acc did not improve from 0.29685\n", + "Epoch 9/25\n", + "765/764 [==============================] - 417s 546ms/step - loss: 11.3153 - acc: 0.2902 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00009: acc did not improve from 0.29685\n", + "Epoch 10/25\n", + "765/764 [==============================] - 412s 538ms/step - loss: 11.3054 - acc: 0.2909 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00010: acc did not improve from 0.29685\n", + "Epoch 11/25\n", + "765/764 [==============================] - 413s 539ms/step - loss: 11.2775 - acc: 0.2926 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00011: acc did not improve from 0.29685\n", + "Epoch 12/25\n", + "765/764 [==============================] - 413s 540ms/step - loss: 11.2758 - acc: 0.2927 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00012: acc did not improve from 0.29685\n", + "Epoch 13/25\n", + "765/764 [==============================] - 414s 541ms/step - loss: 11.2690 - acc: 0.2931 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00013: acc did not improve from 0.29685\n", + "Epoch 14/25\n", + "765/764 [==============================] - 414s 541ms/step - loss: 11.3048 - acc: 0.2909 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00014: acc did not improve from 0.29685\n", + "Epoch 15/25\n", + "765/764 [==============================] - 414s 541ms/step - loss: 11.3019 - acc: 0.2911 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00015: acc did not improve from 0.29685\n", + "Epoch 16/25\n", + "765/764 [==============================] - 414s 542ms/step - loss: 11.2520 - acc: 0.2942 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00016: acc did not improve from 0.29685\n", + "Epoch 17/25\n", + "765/764 [==============================] - 413s 540ms/step - loss: 11.2995 - acc: 0.2912 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00017: acc did not improve from 0.29685\n", + "Epoch 18/25\n", + "765/764 [==============================] - 415s 542ms/step - loss: 11.2935 - acc: 0.2916 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00018: acc did not improve from 0.29685\n", + "Epoch 19/25\n", + "765/764 [==============================] - 412s 539ms/step - loss: 11.3026 - acc: 0.2910 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00019: acc did not improve from 0.29685\n", + "Epoch 20/25\n", + "765/764 [==============================] - 412s 538ms/step - loss: 11.2973 - acc: 0.2914 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00020: acc did not improve from 0.29685\n", + "Epoch 21/25\n", + "765/764 [==============================] - 413s 539ms/step - loss: 11.2449 - acc: 0.2947 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00021: acc did not improve from 0.29685\n", + "Epoch 22/25\n", + "765/764 [==============================] - 413s 539ms/step - loss: 11.2791 - acc: 0.2925 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00022: acc did not improve from 0.29685\n", + "Epoch 23/25\n", + "765/764 [==============================] - 412s 538ms/step - loss: 11.2638 - acc: 0.2935 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00023: acc did not improve from 0.29685\n", + "Epoch 24/25\n", + "765/764 [==============================] - 413s 540ms/step - loss: 11.3240 - acc: 0.2897 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00024: acc did not improve from 0.29685\n", + "Epoch 25/25\n", + "765/764 [==============================] - 413s 540ms/step - loss: 11.2605 - acc: 0.2937 - val_loss: 11.7368 - val_acc: 0.2638\n", + "\n", + "Epoch 00025: acc did not improve from 0.29685\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 13 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "jmV1ZkILkMCu", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 63 + }, + "outputId": "cfc44478-67de-4607-bc70-ba6c717112b6" + }, + "source": [ + "! ls" + ], + "execution_count": 16, + "outputs": [ + { + "output_type": "stream", + "text": [ + "cnn-cv-iphone-resnet-50.hdf5 data data.zip __MACOSX\tsample_data\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Bb06-bzln03x", + "colab_type": "text" + }, + "source": [ + "### 4. Model check" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "oD2n078IrNx0", + "colab_type": "text" + }, + "source": [ + "#### 4.1 Predictions" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "__PCXx7LrTCH", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "outputId": "2e33ca59-5e80-48fd-f5dc-a295cd472000" + }, + "source": [ + "test_set.reset()\n", + "model_pred = model.predict_generator(test_set, \n", + " steps= test_set.samples / batch_size, \n", + " verbose=1)" + ], + "execution_count": 18, + "outputs": [ + { + "output_type": "stream", + "text": [ + "216/215 [==============================] - 28s 131ms/step\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wqeZhJhcn4HC", + "colab_type": "text" + }, + "source": [ + "#### 4.2 Precision, Recall, AUC" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "f6hZOBiMq4PJ", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "outputId": "a72b3db8-ea0a-48a8-ab36-25f7e7b21ff3" + }, + "source": [ + "avg_precision = average_precision_score(test_set.classes, [i[0] for i in model_pred])\n", + "print('Average precision-recall score',avg_precision)" + ], + "execution_count": 22, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Average precision-recall score: 0.26380011621150495\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "A5Oe3sNbsLxo", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 312 + }, + "outputId": "442ccef1-f4cb-4c57-a583-fbf83304733e" + }, + "source": [ + "precision, recall, _ = precision_recall_curve(test_set.classes, [i[0] for i in model_pred])\n", + "plt.fill_between(recall, precision, alpha=0.2, color='r')\n", + "plt.xlabel('Recall')\n", + "plt.ylabel('Precision')\n", + "plt.ylim([0.0, 1.0])\n", + "plt.xlim([0.0, 1.0])\n", + "plt.title('2-class Precision-Recall curve')" + ], + "execution_count": 28, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "Text(0.5, 1.0, '2-class Precision-Recall curve')" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 28 + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xm0nXV97/H3JwmZJ8hMhpNABkwF\nFVOU67pCr5QCV8FlHUhFxVJirbRap2VXvRqx1mu96rVXrMZCERARWK0rVShOISgkkANBIAmBJBAy\noCQQApmn7/3j92yy2ZzznH1OzrOn83mttVf28Oxnf/dzTvb3PL/n+X22IgIzM7PO9Kt3AWZm1tjc\nKMzMLJcbhZmZ5XKjMDOzXG4UZmaWy43CzMxyuVFYt0m6VNJv6l1Hb5O0StLZXSwzTdIuSf1rVFbh\nJD0p6Zzs+kJJN9S7JmssbhR9hKRBkq6WtFHSi5IelHR+veuqRvZBtjf7gP69pGslDe/t14mIP4iI\nO7tY5qmIGB4Rh3v79bMP6YPZ+3xe0j2Szuzt1zHrLjeKvmMAsAk4CxgFfBa4WdL0OtbUHW+LiOHA\n6cA8Uv0vo6TZf6d/lL3PscAS4JY619PrJA2odw3WPc3+n8qqFBG7I2JhRDwZEUci4ifAE8DrO3uO\npKmS/l3SNknPSvpWJ8t9U9ImSS9Iul/Sfy977AxJ7dljv5f09ez+wZJuyNb7vKQVkiZU8T62ALcD\nr87Wc6ekL0m6G9gDnCRpVLb39LSkLZL+oXyoSNLlktZke1arJZ2e3V8+BNNZ3dMlRenDTtKJkhZL\nek7SOkmXl73OQkk3S7oue61VkuZ19R6z93kI+AEwWdK4snW+NdsbLO1xnFb2WIc/L0knS/pVdt92\nST+QNLqaOipJuih7/RckrZd0XuW2K3vvN1Rss8skPQX8StLtkq6oWPdvJb0ju36KpJ9n23WtpHf3\npF7rHW4UfVT2oTwbWNXJ4/2BnwAbgenAZOCmTla3AngtcAJwI3CLpMHZY98EvhkRI4GTgZuz+z9A\n2rOZCowB/hLYW0XdU4ELgJVld78PWACMyOq9FjgEzAReB5wL/EX2/HcBC4H3AyOBC4FnO3ipzuqu\ndBOwGTgReCfwj5L+R9njF2bLjAYWAx022w7e58CsxmeBHdl9rwOuAT5E2mbfBRYrDSvm/bwEfDmr\n8VWkbb6wmjoqajoDuA74VPZ+3gw82Y1VnJW9/p8APwTml617LtAG/FTSMODnpN+l8cDFwLezZawe\nIsKXPnYBjgN+AXw3Z5kzgW3AgA4euxT4Tc5zdwCvya7fBXwBGFuxzJ8D9wCnVVHvk8Au4HnSB+G3\ngSHZY3cCV5YtOwHYX3o8u28+sCS7fgfw0ZzXOaeLuqcDQRrKmwocBkaUPf5l4Nrs+kLgF2WPzQX2\n5rzPhcCB7H0eJjWJs8se/xfgixXPWUv6AO7059XB67wdWNnJ+14I3NDJ874LfKOrbVe5nrJtdlLZ\n4yOA3UBbdvtLwDXZ9fcAv+7gtT9f7/87ffXiPYo+JhvDv570gXRF2f23ZwdRd0l6L+lDcGOkIZCu\n1vnJbChnp6TnSXsKY7OHLyPtuTyaDS+9Nbv/etKH9k2Stkr6J0nH5bzM2yNidES0RcRfRUT53sem\nsuttpEb4dDY88zzpQ2Z89vhUYH1X7ymn7nInAs9FxItl920k/TVf8ruy63uAwZIGSHpv2fa+vWyZ\nmyNiNKnhPcLLhwbbgE+U3lf23qZmdXT685I0QdJN2TDcC8ANHP35dEe1264zL/2csm32U9LeAqRm\n/oPsehvwhor3+V5g4jG8th0DH1TqQyQJuJr0IXRBRBwsPRYR51cseyYwTdKAvGahdDzi08BbgFUR\ncUTSDtJwBxHxODA/a1DvAG6VNCYidpP+Yv+C0gH120h/HV/dg7dWHoG8ibRHMbaTujeRhpLyV9hJ\n3RWLbQVOkDSirFlMA7ZUsf4fcPSDsaPHt0taALRLujEins5q/1JEfKly+S5+Xv9I2kanRsRzkt5O\nlUNgFfK23W5gaNntjj7UK6Oqfwh8XtJdwGDSwfvS6yyNiD/uQY1WAO9R9C3/QhojflvFX+QduQ94\nGvjfkoYpHXx+UwfLjSAdD9gGDJD0OdLYPwCSLpE0LiKOkIZUAI5I+iNJp2Zj6y8AB4Ejx/TugOwD\n9WfA1ySNlNQvO5h7VrbIvwKflPR6JTMltVWup7O6K15rE2n47MvZ9jmNtCfSK/MQImItaa/r09ld\n3wP+UtIbstqHSfqfkkaQ//MaQRq62ylpMukYQ09cDXxQ0luy7TpZ0inZYw8CF0s6TumA/TurWN9t\npL2HK0lne5W270+A2ZLel63vOEl/KOlVPazbjpEbRR+RfRh+iHTQ+XcVw0yvEGmewNtIB4SfIh2w\nfU8Hi94B/BfwGGnYZR8vHwo6D1glaRfpAPHFWZOaCNxKahJrgKWk4aje8H5gILCadLzkVmBS9r5u\nIY2H3wi8CPyYdBC+Umd1V5pPGoPfCvwHaRz9F730PgC+CiyQND4i2oHLSXsDO4B1pONFXf28vkA6\nrXgnabjn33tSSETcB3wQ+Ea2rqWkD3qA/0Xa29iRvd6NVaxvf1bLOeXLZ3tn55KGpbaShu++Agzq\nSd127BThLy4yM7POeY/CzMxyFdYoJF0j6RlJj3TyuCT9s9IkpYeUTXoyM7PGUuQexbWkcd7OnA/M\nyi4LSAdazcyswRTWKCLiLuC5nEUuAq6LZDkwWtKkouoxM7Oeqec8ism8/OyYzdl9T1cumJ1PvgBg\n2NChrz9lzhzo58MrZmbVuv/++7dHxLiul3ylpphwFxGLgEUA8+bMifYvfhEmTYKTT4ZRo+pcnZlZ\n45O0safPreef5VtIkQAlU6hiRisAY8fCzp1w991w332wYwf4NF8zs0LUs1EsBt6fnf30RmBnNqu2\nOiNHwoQJsHcvLF8Oy5bB9u1uGGZmvaywoSdJPwTOBsZK2gx8nhTWRkR8hzR9/wLS7NI9pBmf3Td8\neLrs3g0rVsCIETBrFowb5+MYZma9oLBGERHzu3g8gI/02gsOG5Yue/fCAw/A0KEwezaMHw/9W+br\njc3Maq4pDmZ3y5Ah6bJvH6xcma7PmgUTJ8KA1nu7ZmZFa91PzsGD0+XAAXjkEVizJjWMyZPhuLyv\nPTAzs3Kt2yhKBg5MxysOHYK1a+Gxx9JptVOmwCCHUZqZdaX1G0XJgAHptNpDh2D9eli3DqZPh2nT\n0vCUmZl1qO80ipIBA2DMGDh8GDZuhCeeSM2irS0dDDczs5fpe42ipH//1DCOHIGtW1PTmDwZZsxI\np9iamRnQlxtFSb9+cPzxaaLetm2wZUuayOd4EDMzwI3iKAlGj04NY+dOuOeetMcxc2ZqJFK9KzQz\nqws3ikpSigcZORJ27YJ77017FrNnp8bhhmFmfYwbRZ5SPMiePY4HMbM+y42iGkOHpsu+fY4HMbM+\nx42iO0qzvffvhwcfTBP2Zs92PIiZtTSPn/TEoEFpb2LIkBQPsmQJPPlkigsxM2sx/jP4WDgexMz6\nADeK3uB4EDNrYW4Uvak8HuSpp2DDhhQN4ngQM2tibhRF6N8fTjjB8SBm1hLcKIrkeBAzawFuFLXg\neBAza2JuFLXkeBAza0JuFPXieBAzaxJuFPVWGQ8yeDCccorjQcysYbhRNIqO4kFmzYJJkxwPYmZ1\n5TGORlMeD7JqleNBzKzu/Kdqo6qMB3n8cTjpJMeDmFnNuVE0OseDmFmduVE0i47iQaZNS03D8SBm\nViA3imZTHg/y9NOpaZx4YhqWcjyImRXAjaJZlceDbN+eMqXGj0/xIKNH17s6M2shbhTNrjwe5IUX\nYNkyx4OYWa9yo2gVjgcxs4K4UbSi8niQ++5LzcPxIGbWQ24UrayjeJA5c1LUueNBzKxKbhR9QXk8\nyG9/63gQM+sWj0P0JaV4kKFDHQ9iZlUrtFFIOk/SWknrJH2mg8enSVoiaaWkhyRdUGQ9ljnuuHS8\nYuTIFA+ydGma8b1/f70rM7MGVFijkNQfuAo4H5gLzJc0t2KxzwI3R8TrgIuBbxdVj3WgFA8ycmSa\n6b1kCTz6KOzdW+/KzKyBFDlAfQawLiI2AEi6CbgIWF22TAAjs+ujgK0F1mOdKY8H2bQJnnjC8SBm\n9pIiG8VkYFPZ7c3AGyqWWQj8TNJfA8OAczpakaQFwAKAaRMm9HqhlnE8iJl1oN4Hs+cD10bEFOAC\n4HpJr6gpIhZFxLyImDdu1KiaF9nnlOJBxo1L8SC/+Q3cfz88/3y9KzOzOihyj2ILMLXs9pTsvnKX\nAecBRMQySYOBscAzBdZl1SrFg0CKB7nnnnRMw/EgZn1KkXsUK4BZkmZIGkg6WL24YpmngLcASHoV\nMBjYVmBN1lMjR6aJenv3wvLlKVNq+/aUMWVmLa2wPYqIOCTpCuAOoD9wTUSsknQl0B4Ri4FPAN+T\n9LekA9uXRviTp6FVxoMMH55mezsexKxlqdk+l+fNmRPt113nGcWNYt++NCzleBCzhibp/oiY15Pn\n+tPWjo3jQcxanscKrHc4HsSsZflPPutdpXiQQ4dSPMhjj6V5GFOmpD0PM2s6bhRWjFI8yKFDKR5k\n3bo007utDYYMqXd1ZtYNbhRWLMeDmDU9Nwqrjc7iQWbMSHM0zKxhuVFYbZXiQSLShL0tW9IptSef\nfHQWuJk1FDcKq4/yeJAXX3Q8iFkDc6Ow+hsxIl127UrxIKNGpcl7Y8a4YZg1ADcKaxzl8SArVqSD\n3bNnp/kZjgcxqxs3Cms8Q4emy759sHKl40HM6syNwhpXZ/EgEyemiX1mVhPen7fGVxkPcuedjgcx\nqyHvUVjz6CgeZMYMmDrV8SBmBXKjsOZTigc5fDjN9F6/Ps30njYt7XWYWa9yo7Dm1b9/OoX2yJGX\nx4O0taWzp8ysV7hRWPPr18/xIGYFcqOw1lEeD/Lss44HMeslbhTWeqQ0u3vUKMeDmPUCNwprbeXx\nIPfem4aiZs9OjcMNw6wqbhTWN5THg7S3Ox7ErBvcKKxv6SgeZPbsNNvb8SBmHXKjsL6pPB7koYfS\nBD7Hg5h1yPvc1rdVxoMsWZK+49vxIGYv8R6FGbw8HuTxx2HdOseDmGXcKMzKlceDbNjgeBAz3CjM\nOta/f2oYjgcxc6Mwy1WKB4lwPIj1WW4UZtWQXhkPMn58mu3teBBrcW4UZt3RUTzICSekuRiOB7EW\n5UZh1lOdxYOMGePZ3tZS3CjMjpXjQazFuVGY9RbHg1iLcqMw622V8SCPPpriQSZNcjyINaVC94sl\nnSdpraR1kj7TyTLvlrRa0ipJNxZZj1lNleJBhg2D1asdD2JNq7A9Ckn9gauAPwY2AyskLY6I1WXL\nzAL+DnhTROyQNL6oeszqxvEg1uSKHHo6A1gXERsAJN0EXASsLlvmcuCqiNgBEBHPFFiPWX11FA/S\n1pYujgexBlbk0NNkYFPZ7c3ZfeVmA7Ml3S1puaTzOlqRpAWS2iW1b9u5s6ByzWqkFA8yZgxs3gxL\nl6bk2l276l2ZWYfqfTB7ADALOBuYAtwl6dSIeL58oYhYBCwCmDdnTtS6SLNClMeD/O53KR5k0iQ4\n6STHg1hDqbpRSJoMtJU/JyLuynnKFmBq2e0p2X3lNgP3RsRB4AlJj5Eax4pq6zJrelKKAYmA555L\nmVLjxjkexBpGVY1C0leA95COLxzO7g4gr1GsAGZJmkFqEBcDf1axzI+B+cC/SRpLGoraUHX1Zq2k\nFA8CKR5k2bIUCzJrVtrzcDyI1Um1exRvB+ZExP5qVxwRhyRdAdwB9AeuiYhVkq4E2iNicfbYuZJK\nDehTEfFs996CWQsqjwe57z7Hg1hdKaLrIX9JtwPvioi6H22bN2dOtF93XTqDxKyv2LMnNQ3Hg1gP\nSbo/Iub15LnVftruAR6U9Evgpb2KiPibnryomXVTZ/EgEyb4jyYrXLW/YYuzi5nVU3k8yMMPp3iQ\nmTPTlyk5HsQKUlWjiIjvSxpIOtgMsDY7U8nM6mHQoHRm1MGDsGYNrF2bGsaUKTBwYL2rsxZT7VlP\nZwPfB54EBEyV9IEuTo81s6I5HsRqoNqhp68B50bEWgBJs4EfAq8vqjAz6wbHg1iBqm0Ux5WaBEBE\nPCbJA6JmjaYUD3LkSPpe7yeegGnTYPr09OVKZj1QbaNol/SvwA3Z7fcC7cWUZGbHrF+/NFkvAn7/\ne9i0yfEg1mPVNooPAx8BSqfD/hr4diEVmVnv6Swe5OSTUyMxq0K1Zz3tB76eXcys2VTGgyxf7ngQ\nq1puo5B0c0S8W9LDpGynl4mI0wqrzMyKUYoH2b3b8SBWla72KD6a/fvWogsxsxobNixd9u6F9vaj\n8SDjxqWD4maZ3EYREU9nV7cDeyPiSHZq7CnA7UUXZ2Y1MGRIupTHg8yaBRMnOh7EgOq/4e4uYHD2\nnRQ/A94HXFtUUWZWB4MHp7DBQYPgkUfSN+9t3Jhmf1ufVm2jUETsAd4BfDsi3gX8QXFlmVndlOJB\nhg1L8SBLlqRJfAcO1Lsyq5Nq9ysl6UzS/InLsvs8iGnWyhwPYplqG8XHgL8D/iP78qGTgCXFlWVm\nDaMyHmTdujTT2/EgfUa18yiWAkvLbm/g6OQ7M+sLOooHmTo17WU4HqSldTWP4v9GxMck/Scdz6O4\nsLDKzKwxlceDPPMMbN7seJAW19UexfXZv/+n6ELMrMk4HqTP6Goexf3Z1XayeRQAkvoDgwquzcya\ngeNBWl61p8f+Eig/ajUE+EXvl2NmTW3EiDQX48ABuPdeuOce2LYtHdewplXtWU+DI2JX6UZE7JLk\n0x3MrGOV8SBDh8KcOY4HaVLVNordkk6PiAcAJL0e2FtcWWbWEhwP0hK6M4/iFklbSd+ZPRF4T2FV\nmVlrGTw4XQ4cgIcfhkcfTQ3jxBPTxD5raNXOo1gh6RRgTnbX2ohwAIyZdc/AgekYxsGDKR5k7VqY\nORMmT07RIdaQqmoU2fGIjwNtEXG5pFmS5kTET4otz8xakuNBmkq1Zz39G3AAODO7vQX4h0IqMrO+\noxQPMnp0mum9ZAmsXg179tS7MitT7TGKkyPiPZLmA0TEHsknR5tZL+nfP33D3pEjsHVrijefMsXx\nIA2i2kZxQNIQshgPSScD+wurysz6po7iQSZOTPEgpUl9VnPVNorPA/8FTJX0A+BNwKVFFWVmfVx5\nPMiOHWniXikeZPRoz/ausS4bRTbE9CjpS4veSDo99qMRsb3g2sysryuPB9m1C5YtS7EgjgepqS4b\nRUSEpNsi4lTgpzWoyczslYYPT5fdu1M8yKhRMHt2OrbRr9rzcqwnqt26D0j6w0IrMTOrxrBhMGFC\nGpZqb4df/zol1x4+XO/KWla1xyjeAFwi6UlgN2n4KSLitKIKMzPLVR4P8uCDjgcpULVb808KrcLM\nrKccD1K43KEnSYMlfQz4FHAesCUiNpYuXa1c0nmS1kpaJ+kzOcv9qaSQNK/b78DMDI7GgwwfnuJB\nlixJ3/G932fyH6uujlF8H5gHPAycD3yt2hVnX250Vfa8ucB8SXM7WG4E8FHg3mrXbWbWqVI8yKhR\nKRrkzjtTTMi+ffWurGl1NfQ0NzvbCUlXA/d1Y91nAOsiYkP2/JuAi4DVFct9EfgKaa/FzKx3DBiQ\nzog6fDjFg6xfD9OmwfTp6fsxrGpd7VG8lBAbEYe6ue7JwKay25uz+14i6XRgakTknnYraYGkdknt\n23bu7GYZZtanleJBxoxJ8SBLl6ZjGbt2df1cA7reo3iNpBey6wKGZLdLZz2N7OkLS+oHfJ0qZnhH\nxCJgEcC8OXOip69pZn2Y40F6LLdRRMSxfGfhFmBq2e0p2X0lI4BXA3dm+YITgcWSLoyI9mN4XTOz\nzlXGg9x9dzoI7niQThV5svEKYJakGaQGcTHwZ6UHI2InMLZ0W9KdwCfdJMysJkrxIKNGOR6kC4XN\ne8+OaVwB3AGsAW6OiFWSrpR0YVGva2bWbcOHp9neBw6keJC774Zt21LsuRW6R0FE3AbcVnHf5zpZ\n9uwiazEz69KwYemyd2+KBxk6NOVJjR+fDor3UZ7nbmZWyfEgL9P33rGZWbUcDwK4UZiZda0UD3Lw\nYIoHWbsWZs6EyZNh0KB6V1c4Nwozs2qV4kEOHUrxII8/nmZ6T5uWhqpalBuFmVl3lceDbNyYIkKm\nTYO2tnQwvMW4UZiZ9VQpHuTIkRQPsnEjTJmS9jJGjKh3db3GjcLM7FhVxoNs2gSTJrVMPIgbhZlZ\nbymPB3n++TRxb9y4dOC7ieNB3CjMzHqbBCNHpkspHuT449PkvSaMB3GjMDMr0vDh6bJ7N9x3Xzp2\nMWtW2tPoV1iKUq9yozAzq4XyeJAHHmiqeBA3CjOzWmrCeJDGrMrMrNU1UTyIG4WZWT2V4kEOHUrN\nYu3a9CVKU6Y0TDyIG4WZWSMYMADGjk0NY/36FBHSIPEgbhRmZo2kAeNB3CjMzBpRA8WDuFGYmTWy\njuJBJk5MxzFqFA/iRmFm1gzK40F27qxpPIgbhZlZM+koHmT06DR5b8yYQhqGG4WZWbMqjwdZsaKw\neBA3CjOzZldwPIgbhZlZq8iLBzkGbhRmZq2mPB7kkUdgzRoGQo9zQdwozMxa1cCB6XjFvn0Mhh7n\ngTRHGLqZmfXcMR6ncKMwM7NcbhRmZpbLjcLMzHK5UZiZWS43CjMzy+VGYWZmudwozMwslxuFmZnl\nKrRRSDpP0lpJ6yR9poPHPy5ptaSHJP1SUluR9ZiZWfcV1igk9QeuAs4H5gLzJc2tWGwlMC8iTgNu\nBf6pqHrMzKxnityjOANYFxEbIuIAcBNwUfkCEbEkIvZkN5cDUwqsx8zMeqDIRjEZ2FR2e3N2X2cu\nA27v6AFJCyS1S2rftnNnL5ZoZmZdaYiD2ZIuAeYBX+3o8YhYFBHzImLeuBp9mbiZmSVFxoxvAaaW\n3Z6S3fcyks4B/h44KyL2F1iPmZn1QJF7FCuAWZJmSBoIXAwsLl9A0uuA7wIXRsQzBdZiZmY9VFij\niIhDwBXAHcAa4OaIWCXpSkkXZot9FRgO3CLpQUmLO1mdmZnVSaHfcBcRtwG3Vdz3ubLr5xT5+mZm\nduwa4mC2mZk1LjcKMzPL5UZhZma53CjMzCyXG4WZmeVyozAzs1xuFGZmlsuNwszMcrlRmJlZLjcK\nMzPL5UZhZma53CjMzCyXG4WZmeVyozAzs1xuFGZmlsuNwszMcrlRmJlZLjcKMzPL5UZhZma53CjM\nzCyXG4WZmeVyozAzs1xuFGZmlsuNwszMcrlRmJlZLjcKMzPL5UZhZma53CjMzCyXG4WZmeVyozAz\ns1xuFGZmlsuNwszMcrlRmJlZLjcKMzPL5UZhZma5Cm0Uks6TtFbSOkmf6eDxQZJ+lD1+r6TpRdZj\nZmbdV1ijkNQfuAo4H5gLzJc0t2Kxy4AdETET+AbwlaLqMTOznhlQ4LrPANZFxAYASTcBFwGry5a5\nCFiYXb8V+JYkRUTkrnn/fjh0qNcLNjNrScf4eVlko5gMbCq7vRl4Q2fLRMQhSTuBMcD28oUkLQAW\nAAgOjDjrrPVFFd1MDsLxx8GOetfRCLwtjvK2OMrb4qjd0NbT5xbZKHpNRCwCFgFIan8xYl6dS2oI\nktr3eVsA3hblvC2O8rY4SlJ7T59b5MHsLcDUsttTsvs6XEbSAGAU8GyBNZmZWTcV2ShWALMkzZA0\nELgYWFyxzGLgA9n1dwK/6vL4hJmZ1VRhQ0/ZMYcrgDuA/sA1EbFK0pVAe0QsBq4Grpe0DniO1Ey6\nsqiompuQt8VR3hZHeVsc5W1xVI+3hfwHvJmZ5fHMbDMzy+VGYWZmuRq2UTj+46gqtsXHJa2W9JCk\nX0rq8fnSja6rbVG23J9KCkkte2pkNdtC0ruz341Vkm6sdY21UsX/kWmSlkhamf0/uaAedRZN0jWS\nnpH0SCePS9I/Z9vpIUmnV7XiiGi4C+ng93rgJGAg8FtgbsUyfwV8J7t+MfCjetddx23xR8DQ7PqH\n+/K2yJYbAdwFLAfm1bvuOv5ezAJWAsdnt8fXu+46botFwIez63OBJ+tdd0Hb4s3A6cAjnTx+AXA7\nIOCNwL3VrLdR9yheiv+IiANAKf6j3EXA97PrtwJvkaQa1lgrXW6LiFgSEXuym8tJc1ZaUTW/FwBf\nJOWG7atlcTVWzba4HLgqInYARMQzNa6xVqrZFgGMzK6PArbWsL6aiYi7SGeQduYi4LpIlgOjJU3q\nar2N2ig6iv+Y3NkyEXEIKMV/tJpqtkW5y0h/MbSiLrdFtis9NSJ+WsvC6qCa34vZwGxJd0taLum8\nmlVXW9Vsi4XAJZI2A7cBf12b0hpOdz9PgCaJ8LDqSLoEmAecVe9a6kFSP+DrwKV1LqVRDCANP51N\n2su8S9KpEfF8Xauqj/nAtRHxNUlnkuZvvToijtS7sGbQqHsUjv84qpptgaRzgL8HLoyI/TWqrda6\n2hYjgFcDd0p6kjQGu7hFD2hX83uxGVgcEQcj4gngMVLjaDXVbIvLgJsBImIZMBgYW5PqGktVnyeV\nGrVROP7jqC63haTXAd8lNYlWHYeGLrZFROyMiLERMT0ippOO11wYET0OQ2tg1fwf+TFpbwJJY0lD\nURtqWWSNVLMtngLeAiDpVaRGsa2mVTaGxcD7s7Of3gjsjIinu3pSQw49RXHxH02nym3xVWA4cEt2\nPP+piLiwbkUXpMpt0SdUuS3uAM6VtBo4DHwqIlpur7vKbfEJ4HuS/pZ0YPvSVvzDUtIPSX8cjM2O\nx3weOA4gIr5DOj5zAbAO2AN8sKr1tuC2MjOzXtSoQ09mZtYg3CjMzCyXG4WZmeVyozAzs1xuFGZm\nlsuNwqyCpMOSHpT0iKT/lDS6l9d/qaRvZdcXSvpkb67frLe5UZi90t6IeG1EvJo0R+cj9S7IrJ7c\nKMzyLaMsNE3SpyStyLL8v1B2//uz+34r6frsvrdl35WyUtIvJE2oQ/1mx6whZ2abNQJJ/UmxD1dn\nt88lZSWdQcrzXyzpzaSMsc8C/y0itks6IVvFb4A3RkRI+gvg06QZwmZNxY3C7JWGSHqQtCexBvh5\ndv+52WVldns4qXG8BrglIrbVRyTUAAAA10lEQVQDRETp+wCmAD/K8v4HAk/Upnyz3uWhJ7NX2hsR\nrwXaSHsOpWMUAr6cHb94bUTMjIirc9bz/4BvRcSpwIdIQXRmTceNwqwT2bcG/g3wiSzK/g7gzyUN\nB5A0WdJ44FfAuySNye4vDT2N4miE8wcwa1IeejLLERErJT0EzI+I67OI6mVZSu8u4JIsqfRLwFJJ\nh0lDU5eSvlXtFkk7SM1kRj3eg9mxcnqsmZnl8tCTmZnlcqMwM7NcbhRmZpbLjcLMzHK5UZiZWS43\nCjMzy+VGYWZmuf4/b54y4kzOo6MAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3kxKq7mVs0zT", + "colab_type": "text" + }, + "source": [ + "### 5. Conclusion\n", + "We can see that from-scratch implementation of ResNet-50 showed poor performance and gained low values of important metrics.\n", + "Thus, I would like to take one more approach with pre-trained ResNet-50 CNN from Keras library." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IWYGprmPtNPT", + "colab_type": "text" + }, + "source": [ + "### 6. Keras ResNet CNN" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "uayHhll6tL_y", + "colab_type": "code", + "colab": {} + }, + "source": [ + "# Import ResNet50 CNN \n", + "from keras.applications import ResNet50 as RN50" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "3JTdlpA1tdLU", + "colab_type": "code", + "colab": {} + }, + "source": [ + "# Specify parameters for CNN\n", + "img_size = 224\n", + "batch_size = 32" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "4mtJ-f-PtnR6", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 51 + }, + "outputId": "488d0f21-a9a5-4413-d663-d4bf2da803f0" + }, + "source": [ + "# Generating Flows from directories for the new ResNet\n", + "train_set = train_datagen.flow_from_directory(train_path,\n", + " target_size = (img_size, img_size),\n", + " batch_size = batch_size,\n", + " class_mode = 'binary')\n", + "\n", + "test_set = test_datagen.flow_from_directory(test_path,\n", + " target_size = (img_size, img_size),\n", + " batch_size = batch_size,\n", + " class_mode = 'binary',\n", + " shuffle=False)" + ], + "execution_count": 45, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Found 24457 images belonging to 2 classes.\n", + "Found 6884 images belonging to 2 classes.\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "dcxP65_nt3GQ", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 131 + }, + "outputId": "2a131859-3814-4a2e-c8a7-cb37b9290e07" + }, + "source": [ + "# Use ResNet50 with pretrained weights from ImageNet dataset\n", + "model_RN50 = RN50(weights='imagenet', include_top = False, input_shape = (img_size,img_size,3))" + ], + "execution_count": 48, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.6/dist-packages/keras_applications/resnet50.py:265: UserWarning: The output shape of `ResNet50(include_top=False)` has been changed since Keras 2.2.0.\n", + " warnings.warn('The output shape of `ResNet50(include_top=False)` '\n" + ], + "name": "stderr" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "p8R_XTbEvK-g", + "colab_type": "code", + "colab": {} + }, + "source": [ + "# Tweaking up the preloaded model a little bit\n", + "model_new = models.Sequential()\n", + "model_new.add(model_RN50)\n", + "model_new.add(layers.GlobalAveragePooling2D())\n", + "\n", + "model_new.add(layers.Dense(256,activation='relu'))\n", + "model_new.add(layers.Dense(256,activation='relu'))\n", + "model_new.add(layers.Dense(1,activation='sigmoid'))\n", + "\n", + "for layer in model_new.layers[:1]:\n", + " layer.trainable = False\n", + "\n", + "model_new.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "cpMEDY5Ft_LF", + "colab_type": "code", + "colab": {} + }, + "source": [ + "# Define checkpoints for CNN autosaving\n", + "filepath= \"cnn-cv-iphone-resnet-50-pretrained.hdf5\"\n", + "checkpoint = ModelCheckpoint(filepath, monitor='acc', verbose=1, save_best_only=True, mode='max', save_weights_only=False)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "ajUjQGqDuoTi", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1394 + }, + "outputId": "c4a4b23c-79ca-40ca-d145-0e86d0120578" + }, + "source": [ + "# Training the model\n", + "model_new.fit_generator(train_set,\n", + " steps_per_epoch = train_set.samples / batch_size,\n", + " epochs = epochs_nb,\n", + " validation_data = test_set,\n", + " validation_steps = test_set.samples / batch_size,\n", + " verbose = 1,\n", + " callbacks=[checkpoint])" + ], + "execution_count": 52, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Epoch 1/20\n", + "765/764 [==============================] - 365s 477ms/step - loss: 0.2179 - acc: 0.9143 - val_loss: 0.9495 - val_acc: 0.6737\n", + "\n", + "Epoch 00001: acc improved from -inf to 0.91434, saving model to cnn-cv-iphone-resnet-50-pretrained.hdf5\n", + "Epoch 2/20\n", + "765/764 [==============================] - 352s 460ms/step - loss: 0.1549 - acc: 0.9411 - val_loss: 0.9598 - val_acc: 0.6549\n", + "\n", + "Epoch 00002: acc improved from 0.91434 to 0.94108, saving model to cnn-cv-iphone-resnet-50-pretrained.hdf5\n", + "Epoch 3/20\n", + "765/764 [==============================] - 351s 459ms/step - loss: 0.1371 - acc: 0.9492 - val_loss: 1.0934 - val_acc: 0.6053\n", + "\n", + "Epoch 00003: acc improved from 0.94108 to 0.94922, saving model to cnn-cv-iphone-resnet-50-pretrained.hdf5\n", + "Epoch 4/20\n", + "765/764 [==============================] - 352s 461ms/step - loss: 0.1328 - acc: 0.9518 - val_loss: 0.9370 - val_acc: 0.6880\n", + "\n", + "Epoch 00004: acc improved from 0.94922 to 0.95175, saving model to cnn-cv-iphone-resnet-50-pretrained.hdf5\n", + "Epoch 5/20\n", + "765/764 [==============================] - 352s 461ms/step - loss: 0.1243 - acc: 0.9548 - val_loss: 0.9352 - val_acc: 0.6737\n", + "\n", + "Epoch 00005: acc improved from 0.95175 to 0.95486, saving model to cnn-cv-iphone-resnet-50-pretrained.hdf5\n", + "Epoch 6/20\n", + "765/764 [==============================] - 352s 460ms/step - loss: 0.1151 - acc: 0.9590 - val_loss: 0.9080 - val_acc: 0.6902\n", + "\n", + "Epoch 00006: acc improved from 0.95486 to 0.95899, saving model to cnn-cv-iphone-resnet-50-pretrained.hdf5\n", + "Epoch 7/20\n", + "765/764 [==============================] - 352s 460ms/step - loss: 0.1131 - acc: 0.9588 - val_loss: 0.8982 - val_acc: 0.6756\n", + "\n", + "Epoch 00007: acc did not improve from 0.95899\n", + "Epoch 8/20\n", + "765/764 [==============================] - 352s 461ms/step - loss: 0.1092 - acc: 0.9596 - val_loss: 0.9210 - val_acc: 0.6884\n", + "\n", + "Epoch 00008: acc improved from 0.95899 to 0.96001, saving model to cnn-cv-iphone-resnet-50-pretrained.hdf5\n", + "Epoch 9/20\n", + "765/764 [==============================] - 352s 460ms/step - loss: 0.1081 - acc: 0.9612 - val_loss: 0.9733 - val_acc: 0.6954\n", + "\n", + "Epoch 00009: acc improved from 0.96001 to 0.96120, saving model to cnn-cv-iphone-resnet-50-pretrained.hdf5\n", + "Epoch 10/20\n", + "765/764 [==============================] - 351s 459ms/step - loss: 0.1046 - acc: 0.9614 - val_loss: 0.9315 - val_acc: 0.6941\n", + "\n", + "Epoch 00010: acc improved from 0.96120 to 0.96132, saving model to cnn-cv-iphone-resnet-50-pretrained.hdf5\n", + "Epoch 11/20\n", + "765/764 [==============================] - 351s 459ms/step - loss: 0.1022 - acc: 0.9616 - val_loss: 0.8705 - val_acc: 0.7101\n", + "\n", + "Epoch 00011: acc improved from 0.96132 to 0.96152, saving model to cnn-cv-iphone-resnet-50-pretrained.hdf5\n", + "Epoch 12/20\n", + "765/764 [==============================] - 353s 462ms/step - loss: 0.1011 - acc: 0.9630 - val_loss: 0.9520 - val_acc: 0.7061\n", + "\n", + "Epoch 00012: acc improved from 0.96152 to 0.96296, saving model to cnn-cv-iphone-resnet-50-pretrained.hdf5\n", + "Epoch 13/20\n", + "765/764 [==============================] - 353s 462ms/step - loss: 0.0984 - acc: 0.9647 - val_loss: 0.9786 - val_acc: 0.7034\n", + "\n", + "Epoch 00013: acc improved from 0.96296 to 0.96480, saving model to cnn-cv-iphone-resnet-50-pretrained.hdf5\n", + "Epoch 14/20\n", + "765/764 [==============================] - 350s 458ms/step - loss: 0.0914 - acc: 0.9666 - val_loss: 0.9192 - val_acc: 0.7000\n", + "\n", + "Epoch 00014: acc improved from 0.96480 to 0.96655, saving model to cnn-cv-iphone-resnet-50-pretrained.hdf5\n", + "Epoch 15/20\n", + "765/764 [==============================] - 351s 458ms/step - loss: 0.0885 - acc: 0.9686 - val_loss: 1.0647 - val_acc: 0.6791\n", + "\n", + "Epoch 00015: acc improved from 0.96655 to 0.96856, saving model to cnn-cv-iphone-resnet-50-pretrained.hdf5\n", + "Epoch 16/20\n", + "765/764 [==============================] - 353s 461ms/step - loss: 0.0929 - acc: 0.9657 - val_loss: 0.8945 - val_acc: 0.7074\n", + "\n", + "Epoch 00016: acc did not improve from 0.96856\n", + "Epoch 17/20\n", + "765/764 [==============================] - 352s 460ms/step - loss: 0.0935 - acc: 0.9665 - val_loss: 0.8774 - val_acc: 0.7130\n", + "\n", + "Epoch 00017: acc did not improve from 0.96856\n", + "Epoch 18/20\n", + "765/764 [==============================] - 352s 461ms/step - loss: 0.0899 - acc: 0.9678 - val_loss: 0.8978 - val_acc: 0.7025\n", + "\n", + "Epoch 00018: acc did not improve from 0.96856\n", + "Epoch 19/20\n", + "765/764 [==============================] - 351s 459ms/step - loss: 0.0862 - acc: 0.9681 - val_loss: 0.8408 - val_acc: 0.7286\n", + "\n", + "Epoch 00019: acc did not improve from 0.96856\n", + "Epoch 20/20\n", + "765/764 [==============================] - 353s 461ms/step - loss: 0.0834 - acc: 0.9693 - val_loss: 0.9162 - val_acc: 0.7371\n", + "\n", + "Epoch 00020: acc improved from 0.96856 to 0.96933, saving model to cnn-cv-iphone-resnet-50-pretrained.hdf5\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 52 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "KpeDFym8Ts0H", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "outputId": "e41a2590-cc3b-457d-ee88-2c735768efd2" + }, + "source": [ + "# Predict\n", + "test_set.reset()\n", + "\n", + "model_new_pred = model_new.predict_generator(test_set, \n", + " steps= test_set.samples / batch_size, \n", + " verbose=1)" + ], + "execution_count": 58, + "outputs": [ + { + "output_type": "stream", + "text": [ + "216/215 [==============================] - 32s 148ms/step\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "ookDHeL-RsZN", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "outputId": "c80f78a6-1706-4c8e-c3ab-6150cf20f8b3" + }, + "source": [ + "# Check the Model's metrics\n", + "avg_precision = average_precision_score(test_set.classes, [i[0] for i in model_new_pred])\n", + "print('Average precision-recall score',avg_precision)" + ], + "execution_count": 64, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Average precision-recall score 0.45536740225106964\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "FMtxpdZnT9gI", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 312 + }, + "outputId": "9edcb118-ca23-48b3-c99c-7ab78957465e" + }, + "source": [ + "precision, recall, _ = precision_recall_curve(test_set.classes, [i[0] for i in model_new_pred])\n", + "plt.fill_between(recall, precision, alpha=0.2, color='r')\n", + "plt.xlabel('Recall')\n", + "plt.ylabel('Precision')\n", + "plt.ylim([0.0, 1.0])\n", + "plt.xlim([0.0, 1.0])\n", + "plt.title('2-class Precision-Recall curve')" + ], + "execution_count": 65, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "Text(0.5, 1.0, '2-class Precision-Recall curve')" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 65 + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XuUnXV97/H3Zy65ziQBkhByIQHC\nxSgiEFHrOpUeKQVOBVcvCtUqLYXeaO3x0mVXexTtsdZ6ao89aistVsULAqvtioqlVVFaK0qQSyEY\nDSEhCQkhZBJIMpO5fc8f32e7dyYzz+wks2fPTD6vtfaavffz7Gf/9jPJ/szv+igiMDMzG0lLswtg\nZmYTm4PCzMxKOSjMzKyUg8LMzEo5KMzMrJSDwszMSjko7IhJulbSfzS7HGNN0mOSLh5ln1Ml7ZPU\nOk7FajhJmyRdUty/SdLnml0mm1gcFMcJSdMl3SJps6QXJD0k6fJml6sexRdZd/EF/YykT0vqGOv3\niYgXR8S3RtnnqYjoiIiBsX7/4ku6r/iceyT9p6RXjfX7mB0pB8Xxow3YArwGmAv8CXC7pBVNLNOR\neF1EdAAXAKvJ8h9CabL/m/5S8TnnA/cAdzS5PGNOUluzy2BHZrL/p7I6RcT+iLgpIjZFxGBEfAV4\nErhwpNdIWibpHyU9K+k5SR8bYb+PStoi6XlJD0j6bzXbLpK0ttj2jKSPFM/PkPS54rh7JN0v6eQ6\nPsc24GvAS4rjfEvSByR9BzgAnC5pblF72i5pm6T/XdtUJOl6SY8XNat1ki4onq9tghmp3CskReXL\nTtJiSWsk7Za0QdL1Ne9zk6TbJX22eK/HJK0e7TMWn7Mf+DywRNKCmmP+fFEbrNQ4Xlqzbdjfl6Qz\nJH2zeG6XpM9LmldPOYaSdFXx/s9LekLSZUPPXc1n/9yQc3adpKeAb0r6mqQbhxz7YUm/UNw/R9K/\nFed1vaQ3HE15bWw4KI5TxZfyWcBjI2xvBb4CbAZWAEuA20Y43P3Ay4ATgS8Ad0iaUWz7KPDRiJgD\nnAHcXjz/VrJmsww4CfgtoLuOci8DrgAerHn6V4EbgM6ivJ8G+oGVwPnApcBvFK//ZeAm4C3AHOBK\n4Llh3mqkcg91G7AVWAz8EvBnkv57zfYri33mAWuAYcN2mM85rSjjc0BX8dz5wKeA3yTP2SeBNcpm\nxbLfl4APFmV8EXnOb6qnHEPKdBHwWeBdxef5aWDTERziNcX7/xzwReCammOvApYDX5U0G/g38t/S\nQuBq4BPFPtYMEeHbcXYD2oGvA58s2edVwLNA2zDbrgX+o+S1XcB5xf17gfcB84fs8+vAfwIvraO8\nm4B9wB7yi/ATwMxi27eA99fsezJwsLK9eO4a4J7i/t3A20re55JRyr0CCLIpbxkwAHTWbP8g8Oni\n/k3A12u2rQK6Sz7nTUBv8TkHyJC4uGb73wB/OuQ168kv4BF/X8O8z+uBB0f43DcBnxvhdZ8E/mq0\nczf0ODXn7PSa7Z3AfmB58fgDwKeK+28E/n2Y935vs//vHK831yiOM0Ub/q3kF9KNNc9/rehE3Sfp\nTeSX4ObIJpDRjvnOoilnr6Q9ZE1hfrH5OrLm8sOieenni+dvJb+0b5P0tKS/kNRe8javj4h5EbE8\nIn4nImprH1tq7i8ng3B70Tyzh/ySWVhsXwY8MdpnKil3rcXA7oh4oea5zeRf8xU7au4fAGZIapP0\npprz/bWafW6PiHlk4D3KoU2Dy4F3VD5X8dmWFeUY8fcl6WRJtxXNcM8Dn6P6+zkS9Z67kfzk91Sc\ns6+StQXIMP98cX858Iohn/NNwKJjeG87Bu5UOo5IEnAL+SV0RUT0VbZFxOVD9n0VcKqktrKwUPZH\n/CHwWuCxiBiU1EU2dxARPwauKQLqF4A7JZ0UEfvJv9jfp+xQv4v86/iWo/hotUsgbyFrFPNHKPcW\nsimp/IAjlHvIbk8DJ0rqrAmLU4FtdRz/81S/GIfbvkvSDcBaSV+IiO1F2T8QER8Yuv8ov68/I8/R\nuRGxW9LrqbMJbIiyc7cfmFXzeLgv9aFLVX8ReK+ke4EZZOd95X2+HRE/exRltAZwjeL48jdkG/Hr\nhvxFPpzvA9uBP5c0W9n5/Oph9usk+wOeBdokvYds+wdA0pslLYiIQbJJBWBQ0s9IOrdoW38e6AMG\nj+nTAcUX6r8CfylpjqSWojP3NcUufw+8U9KFSislLR96nJHKPeS9tpDNZx8szs9LyZrImMxDiIj1\nZK3rD4un/g74LUmvKMo+W9L/kNRJ+e+rk2y62ytpCdnHcDRuAX5N0muL87pE0jnFtoeAqyW1Kzvs\nf6mO491F1h7eT472qpzfrwBnSfrV4njtkl4u6UVHWW47Rg6K40TxZfibZKfzjiHNTIeJnCfwOrJD\n+Cmyw/aNw+x6N/AvwI/IZpceDm0Kugx4TNI+soP46iKkFgF3kiHxOPBtsjlqLLwFmAasI/tL7gRO\nKT7XHWR7+BeAF4B/Jjvhhxqp3ENdQ7bBPw38E9mO/vUx+hwAHwZukLQwItYC15O1gS5gA9lfNNrv\n633ksOK9ZHPPPx5NQSLi+8CvAX9VHOvb5Bc9wP8iaxtdxft9oY7jHSzKcknt/kXt7FKyWeppsvnu\nQ8D0oym3HTtF+MJFZmY2MtcozMysVMOCQtKnJO2U9OgI2yXpr5WTlB5RMenJzMwmlkbWKD5NtvOO\n5HLgzOJ2A9nRamZmE0zDgiIi7gV2l+xyFfDZSPcB8ySd0qjymJnZ0WnmPIolHDo6Zmvx3PahOxbj\nyW8AmD1r1oXnnH02tLh7xcysXg888MCuiFgw+p6HmxQT7iLiZuBmgNVnnx1rv/xlWLJklFeZmVmF\npM1H+9pm/lm+jVwSoGIpdcxoNTOz8dXMoFgDvKUY/fRKYG8xq3Z0nvthZjZuGtb0JOmLwMXAfElb\ngfeSi7UREX9LTt+/gpxdeoCc8WlmZhNMw4IiIq4ZZXsAv3sUB3aNwsxsHHnokJmZlXJQmJlZqckZ\nFG56MjMbNw4KMzMr5aAwM7NSDgozMyvloDAzs1IOCjMzKzU5g8LMzMbN5AwK1yjMzMaNg8LMzEo5\nKMzMrJSDwszMSjkozMyslIPCzMxKTc6gMDOzcTM5g8I1CjOzcTM5g8LMzMbN5AwK1yjMzMaNg8LM\nzEo5KMzMrJSDwszMSk3OoDAzs3EzOYPCNQozs3HjoDAzs1IOCjMzK+WgMDOzUg4KMzMr5aAwM7NS\nkzMozMxs3DgozMys1OQMCjc9mZmNGweFmZmVclCYmVkpB4WZmZVqaFBIukzSekkbJL17mO2nSrpH\n0oOSHpF0RV0HdlCYmY2bhgWFpFbg48DlwCrgGkmrhuz2J8DtEXE+cDXwiboO7qAwMxs3jaxRXARs\niIiNEdEL3AZcNWSfAOYU9+cCT9d1ZAeFmdm4aWvgsZcAW2oebwVeMWSfm4B/lfR7wGzgkuEOJOkG\n4AaAUxcuHPOCmpnZyJrdmX0N8OmIWApcAdwq6bAyRcTNEbE6IlYvmDt33AtpZnY8a2RQbAOW1Txe\nWjxX6zrgdoCI+C4wA5jfwDKZmdkRamRQ3A+cKek0SdPIzuo1Q/Z5CngtgKQXkUHxbAPLZGZmR6hh\nQRER/cCNwN3A4+TopsckvV/SlcVu7wCul/Qw8EXg2gj3VJuZTSSN7MwmIu4C7hry3Htq7q8DXt3I\nMpiZ2bFpdme2mZlNcA4KMzMr5aAwM7NSDgozMyvloDAzs1IOCjMzK+WgMDOzUg4KMzMr5aAwM7NS\nDgozMyvloDAzs1IOCjMzK+WgMDOzUg4KMzMr5aAwM7NSDgozMyvloDAzs1IOCjMzK+WgMDOzUg4K\nMzMrNTWCIqLZJTAzm7Imf1Bs3w4PPNDsUpiZTVmTMygGB6v3X3gBBgaaVxYzsylucgbFwEAGBEBv\nL/T3N7c8ZmZT2OQMiohqOPT0NLcsZmZT3OQNCgkOHICDB5tdGjOzKW1yB8W6dW52MjNrsMkZFBUD\nA3kbHIQdO5pdGjOzKWlyBkWlRjEwAH19WavYvLnZpTIzm5Imb1D09mZNore3etu3r9klMzObciZv\nUGzenDWKSkgcOADr1x++7+7dDhAzs2MwOYOioq8vQ6KnJ0c/RcDDD1eHzHZ3ww9/CI8/no+91IeZ\n2RGbnEFR+cLv7a3WKvr68v6BA3kf4JFHskbR3Z0Bcv/9zSuzmdkkNTmDAqr9FCecAEuWZId2pRmq\ntTX36emBvXszPA4cyMAA2LOneeU2M5tkGhoUki6TtF7SBknvHmGfN0haJ+kxSV+o68AR2ZHd1wct\nLdDRkUFROwFvcDDvr1iRwbB7d9Y4tm/P+Re160WZmdmIGhYUklqBjwOXA6uAayStGrLPmcAfAa+O\niBcDf1DXwYcGhZSBsG9fPrduXQZCdzfMmJH779uXtYtt2zI4NmwY089rZjZVNbJGcRGwISI2RkQv\ncBtw1ZB9rgc+HhFdABGxs64jR2RADA5CZ2c+19aWTU2V2+bNWcMAWL48m6cOHMjA2Lcvg2TtWq8V\nZWY2ikYGxRJgS83jrcVztc4CzpL0HUn3SbpsuANJukHSWklrn927t9qZvWwZTJ+e99va4Jxzsrmp\npyebmubMyW3Tp8PMmXl/zx448UR4+um8/9BDGRpD9fVlrWPtWjdTmdlxrdmd2W3AmcDFwDXA30ma\nN3SniLg5IlZHxOoFc+fmkwMD2eQ01PLluW3/fli06NBtK1ZAe3u1A3znzlz6o9L5XdHTA9//Pjzx\nRAaOm6nM7DjWVu+OkpYAy2tfExH3lrxkG7Cs5vHS4rlaW4HvRUQf8KSkH5HBUT6OtbLMeMswOdfe\nXu2rGGrGjGp4zJ4NK1fCk0/Cpk0wa1bWPFpb4Qc/gK1bc/+enqyt9PXB4sUZMmZmx5G6gkLSh4A3\nAuuAyuXkAigLivuBMyWdRgbE1cCvDNnnn8maxD9Imk82RW0ctUARWWtobz98W2trfrkvGdrKNYzW\n1upFkB5+OJ9ra8umqBUr8n5/f4ZJBOzalU1Y55xTbdYyM5vi6q1RvB44OyLqvvhDRPRLuhG4G2gF\nPhURj0l6P7A2ItYU2y6VVAmgd0XEc3UcPL/gK/0TQ51zTr3FhKVLcyTUzJnZvzFjBsyfnyEB+XPl\nylwepK0tb4sXOyjM7LhRb1BsBNqBI7pKUETcBdw15Ln31NwP4O3F7UgOnD+Ha3o6UrNnw1lnZV/F\n/PlZG+noOHQfKcMnIgNj69assQzXR2JmNsXUGxQHgIckfYOasIiI329IqUYzOJi3sfyirvRdVEZH\nDUfKkVbPPQePPgovfvHYhJWZ2QRWb1CsKW4TQ2XCXTO+pGfPhqeeykD5zneyNnLyyeNfDjOzcVJX\nUETEZyRNIzubAdYXI5XGn5RBUbl4UTOsXJlDZ6Xss6gERXd3jo7q7HSzlJlNGfWOeroY+AywCRCw\nTNJbRxke2ziVzuxmaW/PPouenpyv8b3vZTPU2rU5SuqUU7Km0d4OXV3Z59HW5vAws0mp3qanvwQu\njYj1AJLOAr4IXNiogpWqND0124wZ2bHd1ZUjpvbsyTkcg4M5UW/WLHj22RwhNTAAr3hFln3aNHjw\nwQyV887Lbf39h46kOngQtmzJYy9blrWWyuVff/CDvN/XlwE0c2auY9XWBhdeWB2xZWY2Bur9Rmmv\nhARARPxI0jCTGMbJRAkKyGaorq6cY7F8eS5z/vzzuUQIZJhs3pyhcd99+VxfX76mpaU6OXDatPzC\nP+OMnLfR1ZXHaWvLYy9cCHPnZnjs2FFtehsczGHClT6btjY499z86Y52MxsD9QbFWkl/D3yuePwm\nYG1jilSHZjc9DXXCCdUZ29Om5TDb+fMP3WfDhhwtdeAALFiQE/qeey5DoqMjZ4fPmJGhsHdvhs78\n+flZt26FjRvzi3/evGzWGs6OHXmcrq4MkXnzMkBmzcqmMTOzo1BvUPw28LtAZTjsvwOfaEiJ6jWR\ngqIeK1ce/lxtmFT6PFpbs4+jojIktx6LFmUw7NuXfSfPPpuPOzqyWWzBgpFDxsxsBPWOejoIfKS4\nTQwTpelpLM2YcezHaGnJvo7a/o7u7qxpHDyYAbtkSe7X3p7B5D4NMytR+g0h6faIeIOk/yLXdjpE\nRLy0YSUbzWSrUTRTZX2q9euzWWrbtgyN9vZs2jr9dDjzzGaX0swmqNH+lHxb8fPnG12QIxaH5ZaN\n5uyz8+fzz+coqoMHc3TWli0ZFkOXWzczY5TrUURE5Yo+u4AtEbEZmA6cBzzd4LKVc3PJ0ZszJ/s+\nZszIBQ7378/hug89lB3pZmY16v22vRf4b5JOAP6VXEL8jeTop/FXuRSqjY3ubvjxj3OY7fPP51Dc\nhQtz9vny5Xm/tzfPeUQuwy5lk9aJJ/p3YTbF1RsUiogDkq4DPhERfyHpoUYWrFQzl++Yis44I38O\nDGRgdHXBD3+YNY29e3N47QsvZJ9Gb28+X5lpPnt2BsaKFRk0O3dWH5vZlFB3UEh6FVmDuK54rnkN\n2s1aEHCqa23NTu+BgTy/Us7N2Lcvh+zu3ZvzRRYsqK7g29OTiyR2dWWI9PTk/I1du3K2+QUX5NyS\nF17ITvT+/qyl9PXlPt3d2QQ2d24Gja8gaDbh1BsUfwD8EfBPxcWHTgfuaVyxrKlqO7Vrrzt+0knV\n+y0teevoyBFTvb1Z45By/sa2bRkavb156+7OmkhLS3aeHzxYHa47Y0aGzrx5uaDiwYNZa5wxI/tT\nurszbBYvzqYuMxtX9c6j+Dbw7ZrHG6lOvht/EdkcYhPHtGnV+wsW5M/+/pzo19l5aE1hpBrh9u3V\n/ffty8Bobc3jtLbCM89kU1ft1Q0XLsxwev75rPGcfHLOExmLOSlmBow+j+L/RsQfSPoyw8+juLJh\nJStz5pke9TQZtLUdvpQJjNxsWDsjfbjX7dhRbeKq9FNt3ZpBEpHBsnNn9rPMnQuvetXYfA6z49xo\n37a3Fj//T6MLYjaq2mawMn19GSrf/W42Zy1enMFhZkelNCgi4oHi7lqgOyIGASS1kvMpzCae9vZs\nvnriiRyBtXVrNlW1tmZozJiRTVaeYGhWl3rbb74BXALsKx7PJOdT/FQjCmV2zObNyxtk/0VPTzZb\n7dhRvSrhokUexmtWh3qDYkZEVEKCiNgnyb3JNjlUFkisjNrq68tl3ytDcxcu9HBrsxL1BsV+SRdE\nxA8AJF0IdDeuWGYN1N6eAyI2bcraxuLF2TRVGb4bkR3kM2dWF0usXY3X7DhzJPMo7pD0NHnN7EXk\nEh5mk1NbW14jZP/+7MuYNSuH7fb25hDc/ftzGO4zz2R4VCYannxybjc7jtQ7j+J+SecAxfKjrI+I\nvsYVy2yczJ5dvsT64GDOKt+0KS9pW5kPMn16NlvNm5fbBwZyZFVlhV6zKaSuoCj6I94OLI+I6yWd\nKensiPhKY4tn1mQtLRkAleG1PT358+mnqxfPqix5Mnt2Xt72nHOyH2TOnAwTr0tmk1y9TU//ADwA\nVGYwbQPuABwUdnypzPg+/fTDtx04AE8+WZ0UOH16BszLX37ozHWzSabeoDgjIt4o6RqAYiVZ/5lk\nVmvWrKxNVAwOwsaNObpq7twMixNOyFnnXoLGJpF6g6JX0kyKZTwknQEcbFipzKaClpbsMO/qgh/9\nKJ+bNSs7xRcuzFFVg4O5dIlrHDaB1RsU7wX+BVgm6fPAq4FrG1UosynlhBOqneADA9k89fTT2XfR\n3p6jrjo6clhuZan3jo7mltmsxqhBUTQx/RD4BeCV5PDYt0XErgaXzWzqaW3NWkat7dtzGO60abmw\nYVdXDt9tb89ggQyRzk4477w8hlt+bRyNGhQREZLuiohzga+OQ5nMji+nnHLoyrk9PRkYe/ZUR011\nd2eY7NmTkwHb2qq3iOr1QKZPz9dOn15dYfn88x0sdkzqbXr6gaSXR8T9DS2NmWU4VK76VzFvXgbC\n/v1Zs+jpyfuDg1kTaWmB3buzttHSkgshVi5X29cHy5blthNOcH+IHbF6g+IVwJslbQL2k81PEREv\nbVTBzGwIqdp3MXv24TPEK4sgQs4ghwyJJ57I5q2BgQyK007LIJo2La8Y6NqGjaLeoPi5hpbCzBqj\nvf3Q2eKbNuWVAAcGctRVR0e1BlPpUJ8/PycLeqFEK4x2hbsZwG8BK4H/Am6JiP56Dy7pMuCjQCvw\n9xHx5yPs94vAncDLI2Jtvcc3syNUu6x6f38uilhppqqYNStrG3PmZC3l1FO9vtVxbrQaxWeAPuDf\ngcuBVcDb6jlwcXGjjwM/C2wF7pe0JiLWDdmvszjm946s6GZ2TNrasunpxBMP39bbm7WPyrXK583L\nGkdfX9Y6Zs/OPpTK8iWdnb488RQ22m92VTHaCUm3AN8/gmNfBGyIiI3F628DrgLWDdnvT4EPAe86\ngmObWSNNmwZnnZX3N23KCz61tFSDoTJ8t3Lt8spVBZcu9cWgpqDRguInK8RGRP8RrtqxBNhS83gr\n2Sn+E5IuAJZFxFcljRgUkm4AbgA4tdJJZ2bjo54v/spyJS+8AM8+m9f4WLKk4UWz8TFaUJwn6fni\nvoCZxePKqKejvpqLpBbgI9QxwzsibgZuBlh99tlxtO9pZg1SWa5k9+68euCOHbB+fc7naGnJfo/F\ni3P5Ept0SoMiIo7l6vPbgGU1j5cWz1V0Ai8BvlXUVBYBayRd6Q5ts0mq0udR6Sjfswf2FVdRfuqp\nDIzp03PEVWdnNnGddFK1Cauzs7nlt2E1svfpfuBMSaeRAXE18CuVjRGxF5hfeSzpW8A7HRJmU0Cl\no7xWRC7Fvn17dopXJgfOmlWdbT5zZgZJe3t2mE+bljUVz/VoqoYFRdGncSNwNzk89lMR8Zik9wNr\nI2JNo97bzCYgKb/8h7uWB2SQVPo49u+vLlGyeXMGSHt7dpYvXlytgdi4aOh4toi4C7hryHPvGWHf\nixtZFjOb4KScuzFnSNfnvn2wc2cuW7J9e9ZABgaqtY7KNT5aW7OmcuKJed/GjAc+m9nE1tFRXbok\nIkOitTVrH088kbPKa+dwzJ2bzVVLlzanvFOQg8LMJg+pGgrD1T4gr/exfz88/njWOjo6ssN8+XIv\nS3KUHBRmNrWcdlp2jvf356irHTuyn2PLlmyqammpdpjPnZuzzr2ibikHhZlNPZWO8EWL8gY5PHfn\nzrxeR1tbNl+1tWWtZOnS6kx0O4yDwsyOD6eeOvzzW7dmM9WmTdWmquXLc3KgR1YBDgozO94tXVrt\nJN+zpzqzfNq0vLW15c8lS47bDnIHhZlZpZN8/vy8QXV2+b59+fOZZ3KU1cyZ8LKXHVf9Gg4KM7Ph\n1M4uX7w4g2P37gyM3bszMGbOzM7wRYuGH4E1RTgozMzq0dYGCxfmrb8furqyf2P6dPjxj7NPoxIs\nJ56YEwGnSB+Hg8LM7Ei1tWUwVFbDPXAgm6U2bqxeq6NyXfPOTli1alLP4XBQmJkdq1mzDh1eG5GT\n/p56KoNj9+5smlq0KMNlki0x4qAwMxtrUg6zXbkyH2/alOtUbd6coTJ7dv5csGBSXODJQWFm1mi1\nVwncvz87xA8cyDDZsCFrG+eeO2GvOz4xS2VmNlVV+i4qtm7NeRu9vbBsWY6wmmAcFGZmzbR0aS6P\nvmFDLjGyYUPWNBYsyLWoJsCwWweFmVmztbfDOefk7PDt27OW8dRTOVKqMrqqtTUXPGwCB4WZ2UTR\n2nroMiH9/dkRvmVLzgR/5pnsIJ8/f8RDNIKDwsxsomprq46c6u/PZqk9e/L6GitWVOdxNLoY4/Iu\nZmZ2bNrasnmqcmW/nTtz+ZDzz8+hto1864Ye3czMxlZnZ976+vJqfv398KIX5dIiDeKgMDObjNrb\n4fTTsw+juztrFa2t+fypp1Yv2DQGHBRmZpNVpQ+jtzf7Lg4ezGXRn3mmOvt70aJDJ/wdzduMTWnN\nzKxppk07vOlp165smtq1K5unjoGDwsxsKqpchGnXLli/ntkw82gPNXnXvTUzs9HNnw8dHbQdQ8XA\nQWFmNtVNn05AHO3LHRRmZlbKQWFmZqUcFGZmVspBYWZmpRwUZmZWykFhZmalHBRmZlbKQWFmZqUa\nGhSSLpO0XtIGSe8eZvvbJa2T9Iikb0ha3sjymJnZkWtYUEhqBT4OXA6sAq6RtGrIbg8CqyPipcCd\nwF80qjxmZnZ0GlmjuAjYEBEbI6IXuA24qnaHiLgnIg4UD+8DlmJmZhNKI4NiCbCl5vHW4rmRXAd8\nbbgNkm6QtFbS2mf37h3DIpqZ2WgmRGe2pDcDq4EPD7c9Im6OiNURsXrB3LnjWzgzs+NcI69HsQ1Y\nVvN4afHcISRdAvwx8JqIONjA8piZ2VFoZI3ifuBMSadJmgZcDayp3UHS+cAngSsjYmcDy2JmZkep\nYUEREf3AjcDdwOPA7RHxmKT3S7qy2O3DQAdwh6SHJK0Z4XBmZtYkDb0UakTcBdw15Ln31Ny/pJHv\nb2Zmx25CdGabmdnE5aAwM7NSDgozMyvloDAzs1IOCjMzK+WgMDOzUg4KMzMr5aAwM7NSDgozMyvl\noDAzs1IOCjMzK+WgMDOzUg4KMzMr5aAwM7NSDgozMyvloDAzs1IOCjMzK+WgMDOzUg4KMzMr5aAw\nM7NSDgozMyvloDAzs1IOCjMzK+WgMDOzUg4KMzMr5aAwM7NSDgozMyvloDAzs1IOCjMzK+WgMDOz\nUg4KMzMr5aAwM7NSDgozMyvloDAzs1IOCjMzK9XQoJB0maT1kjZIevcw26dL+lKx/XuSVjSyPGZm\nduQaFhSSWoGPA5cDq4BrJK0astt1QFdErAT+CvhQo8pjZmZHp62Bx74I2BARGwEk3QZcBayr2ecq\n4Kbi/p3AxyQpImLEo0ZATw+0NbLoZmZTSG8vOoaXN/LbdgmwpebxVuAVI+0TEf2S9gInAbtqd5J0\nA3ADgKB3zsUXPzFykhw/+uCEduhqdjkmAp+LKp+LKp+LJNA+OPVoXz8p/iyPiJuBmwEkrX0+YnWT\nizQhSFrb43MB+FzU8rmo8rmokrT2aF/byM7sbcCymsdLi+eG3UdSGzAXeK6BZTIzsyPUyKC4HzhT\n0mmSpgFXA2uG7LMGeGtx/5dqAEK2AAAE6UlEQVSAb5b2T5iZ2bhrWNNT0edwI3A30Ap8KiIek/R+\nYG1ErAFuAW6VtAHYTYbJaG5uVJknIZ+LKp+LKp+LKp+LqqM+F/If8GZmVsYzs83MrJSDwszMSk3Y\noPDyH1V1nIu3S1on6RFJ35C0vBnlHA+jnYua/X5RUkiaskMj6zkXkt5Q/Nt4TNIXxruM46WO/yOn\nSrpH0oPF/5MrmlHORpP0KUk7JT06wnZJ+uviPD0i6YK6DhwRE+5Gdn4/AZwOTAMeBlYN2ed3gL8t\n7l8NfKnZ5W7iufgZYFZx/7eP53NR7NcJ3AvcB6xudrmb+O/iTOBB4ITi8cJml7uJ5+Jm4LeL+6uA\nTc0ud4POxU8DFwCPjrD9CuBrgIBXAt+r57gTtUbxk+U/IqIXqCz/Uesq4DPF/TuB10o6llnqE9Wo\n5yIi7omIA8XD+8g5K1NRPf8uAP6UXDesZzwLN87qORfXAx+PiC6AiNg5zmUcL/WciwDmFPfnAk+P\nY/nGTUTcS44gHclVwGcj3QfMk3TKaMedqEEx3PIfS0baJyL6gcryH1NNPeei1nXkXwxT0ajnoqhK\nL4uIr45nwZqgnn8XZwFnSfqOpPskXTZupRtf9ZyLm4A3S9oK3AX83vgUbcI50u8TYJIs4WH1kfRm\nYDXwmmaXpRkktQAfAa5tclEmijay+elispZ5r6RzI2JPU0vVHNcAn46Iv5T0KnL+1ksiYrDZBZsM\nJmqNwst/VNVzLpB0CfDHwJURcXCcyjbeRjsXncBLgG9J2kS2wa6Zoh3a9fy72AqsiYi+iHgS+BEZ\nHFNNPefiOuB2gIj4LjADmD8upZtY6vo+GWqiBoWX/6ga9VxIOh/4JBkSU7UdGkY5FxGxNyLmR8SK\niFhB9tdcGRFHvRjaBFbP/5F/JmsTSJpPNkVtHM9CjpN6zsVTwGsBJL2IDIpnx7WUE8Ma4C3F6KdX\nAnsjYvtoL5qQTU/RuOU/Jp06z8WHgQ7gjqI//6mIuLJphW6QOs/FcaHOc3E3cKmkdcAA8K6ImHK1\n7jrPxTuAv5P0P8mO7Wun4h+Wkr5I/nEwv+iPeS/QDhARf0v2z1wBbAAOAL9W13Gn4LkyM7MxNFGb\nnszMbIJwUJiZWSkHhZmZlXJQmJlZKQeFmZmVclCYDSFpQNJDkh6V9GVJ88b4+NdK+lhx/yZJ7xzL\n45uNNQeF2eG6I+JlEfESco7O7za7QGbN5KAwK/ddahZNk/QuSfcXa/m/r+b5txTPPSzp1uK51xXX\nSnlQ0tclndyE8psdswk5M9tsIpDUSi77cEvx+FJyraSLyPX810j6aXKNsT8Bfioidkk6sTjEfwCv\njIiQ9BvAH5IzhM0mFQeF2eFmSnqIrEk8Dvxb8fylxe3B4nEHGRznAXdExC6AiKhcD2Ap8KVivf9p\nwJPjU3yzseWmJ7PDdUfEy4DlZM2h0kch4INF/8XLImJlRNxScpz/B3wsIs4FfpNciM5s0nFQmI2g\nuGrg7wPvKJayvxv4dUkdAJKWSFoIfBP4ZUknFc9Xmp7mUl3C+a2YTVJuejIrEREPSnoEuCYibi2W\nqP5usUrvPuDNxUqlHwC+LWmAbJq6lryq2h2SusgwOa0Zn8HsWHn1WDMzK+WmJzMzK+WgMDOzUg4K\nMzMr5aAwM7NSDgozMyvloDAzs1IOCjMzK/X/AdoMgj31ZBnMAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IJG-6Ad5UdVb", + "colab_type": "text" + }, + "source": [ + "### 7. Conclusion №2\n", + "We can see that pretrained ResNet-50 showed better performance and gained higher values of important metrics, however the model itself is still not that great." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iB3VYOAFWnzC", + "colab_type": "text" + }, + "source": [ + "### 8. Compressing and publishing\n", + "In order to meet stupid GitHub restriction of max file size 100Mb, I would like to use TFLite to compress the model." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "LSwtLkdvW0g-", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 287 + }, + "outputId": "8139ad1c-8f7d-48e3-b4cf-c2eef8562f6a" + }, + "source": [ + "# Converting Keras model to TensorFlow Lite one\n", + "converter = lite.TFLiteConverter.from_keras_model_file('cnn-cv-iphone-resnet-50-pretrained.hdf5')\n", + "tflite_model = converter.convert()\n", + "open(\"cnn-cv-iphone-resnet-50-pretrained.tflite\", \"wb\").write(tflite_model)" + ], + "execution_count": 70, + "outputs": [ + { + "output_type": "stream", + "text": [ + "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:96: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with distribution=normal is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "`normal` is a deprecated alias for `truncated_normal`\n", + "WARNING:tensorflow:Error in loading the saved optimizer state. As a result, your model is starting with a freshly initialized optimizer.\n", + "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/lite/python/lite.py:591: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Use tf.compat.v1.graph_util.convert_variables_to_constants\n", + "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/graph_util_impl.py:245: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Use tf.compat.v1.graph_util.extract_sub_graph\n", + "INFO:tensorflow:Froze 324 variables.\n", + "INFO:tensorflow:Converted 324 variables to const ops.\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "96312484" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 70 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "nBDj-FeCXxNg", + "colab_type": "code", + "colab": {} + }, + "source": [ + "# Generate the requirements file with all of the dependencies\n", + "pip freeze > requirements.txt" + ], + "execution_count": 0, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/bevstratov/model.tflite b/bevstratov/model.tflite new file mode 100644 index 0000000..a1e0032 Binary files /dev/null and b/bevstratov/model.tflite differ diff --git a/bevstratov/predict.py b/bevstratov/predict.py new file mode 100644 index 0000000..7cdc927 --- /dev/null +++ b/bevstratov/predict.py @@ -0,0 +1,80 @@ +# Importing libraries +import argparse +import pandas as pd +import numpy as np +import os +import tensorflow as tf +from keras.preprocessing import image + +# Function for loading and rescaling images +def load_image(img_path): + img = image.load_img(img_path, target_size=(224, 224)) + img_tensor = image.img_to_array(img) + img_tensor = np.expand_dims(img_tensor, axis=0) + img_tensor /= 255. + return img_tensor + +# Function for calculating probabilities +def get_predictions(model_path, in_folder, out_file): + + print('Starting image processing...') + + # Load TFLite model and allocate tensors. + interpreter = tf.lite.Interpreter(model_path=model_path) + interpreter.allocate_tensors() + print('Model has been successfully loaded!') + + # Get input and output tensors. + input_details = interpreter.get_input_details() + output_details = interpreter.get_output_details() + + # Set up some variables + preds = [] + filenames = [] + counter = 0 + + # Loop through test images and get predictions for each one + for subdir, dir, files in os.walk(in_folder): + for file in files: + try: + input_data = load_image(os.path.join(subdir, file)) + interpreter.set_tensor(input_details[0]['index'], input_data) + interpreter.invoke() + output_data = interpreter.get_tensor(output_details[0]['index']) + + # Append results to the array + preds.append(output_data[0]) + filenames.append(file) + + # Track progress + percentage = round(counter/len(files)*100, 2) + print('Progress:',percentage,'%', end='\r') + counter += 1 + except: + print('Unsupported format:', file) + + # Generate dataframe with filenames and iphone probabilities + print('Image processing has been completed') + out_df = pd.DataFrame() + out_df['image_name'] = filenames + out_df['iphone_probability'] = [pred[0] for pred in preds] + + # Export dataframe to CSV + out_df.to_csv(out_file, index=False) + +if __name__ == '__main__': + # Disable warnings + os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' + + # Parse arguments + parser = argparse.ArgumentParser(description='iPhone detector') + + parser.add_argument('--model', type=str, default='model.tflite', help='path to the model') + parser.add_argument('--input', type=str, default='test', help='path to the folder containing pictures') + parser.add_argument('--output', type=str, default='predictions.csv', help='output file path') + args = parser.parse_args() + print('model=',args.model,' input_data=',args.input,' output_data=',args.output) + + # Execute the predict function + get_predictions(args.model, args.input, args.output) + print('Predictions file', args.output,'has been successfully saved!') \ No newline at end of file diff --git a/bevstratov/predictions.csv b/bevstratov/predictions.csv new file mode 100644 index 0000000..d076791 --- /dev/null +++ b/bevstratov/predictions.csv @@ -0,0 +1,69 @@ +image_name,iphone_probability +htc-636.jpg,0.0002446103608235717 +iphone (7415).jpg,0.9999991655349731 +iphone (7403).jpg,1.0 +iphone (7395).jpg,1.0 +htc-637.jpg,0.06424905359745026 +htc-635.jpg,0.0368439257144928 +huawei-2539.jpg,0.00014503127022180706 +iphone (7419).jpg,0.9999998807907104 +iphone (7399).jpg,1.0 +iphone (7423).jpg,0.9999997615814209 +huawei-2538.jpg,0.0001576247886987403 +htc-634.jpg,1.215667089127237e-05 +htc-630.jpg,0.0002865386486519128 +huawei-2528.jpg,1.3843765023224819e-09 +iphone (7422).jpg,1.0 +iphone (7398).jpg,0.9999955892562866 +iphone (7418).jpg,1.0 +huawei-2529.jpg,1.7707568744640412e-08 +htc-631.jpg,9.673178738012211e-07 +htc-633.jpg,0.0020064341370016336 +htc-627.jpg,0.0002645572239998728 +iphone (7394).jpg,1.0 +iphone (7402).jpg,1.0 +iphone (7414).jpg,1.0 +htc-626.jpg,0.0003447848139330745 +htc-632.jpg,0.35063329339027405 +iphone (7409).jpg,1.0 +iphone (7413).jpg,1.0 +iphone (7405).jpg,0.9999998807907104 +iphone (7393).jpg,1.0 +iphone (7404).jpg,1.0 +iphone (7412).jpg,1.0 +huawei-2548.jpg,5.6224249419756234e-05 +iphone (7408).jpg,1.0 +huawei-2549.jpg,0.18368463218212128 +huawei-2550.jpg,0.00911719910800457 +huawei-2544.jpg,6.959380698390305e-05 +huawei-2545.jpg,0.0002948904875665903 +huawei-2547.jpg,0.0004999940283596516 +iphone (7407).jpg,1.0 +iphone (7411).jpg,0.9999984502792358 +huawei-2546.jpg,0.020786777138710022 +huawei-2542.jpg,7.604515985804028e-07 +iphone (7410).jpg,1.0 +iphone (7406).jpg,1.0 +huawei-2543.jpg,0.03468066081404686 +huawei-2541.jpg,3.3432322652515722e-06 +huawei-2540.jpg,2.375311725089091e-09 +huawei-2533.jpg,0.014947078190743923 +huawei-2527.jpg,0.00026377796893939376 +iphone (7397).jpg,0.9999942779541016 +iphone (7401).jpg,1.0 +iphone (7417).jpg,1.0 +huawei-2526.jpg,0.0001296536938752979 +huawei-2532.jpg,0.05978754162788391 +htc-628.jpg,2.0107416276005097e-05 +huawei-2530.jpg,0.0010379767045378685 +iphone (7421).jpg,1.0 +huawei-2531.jpg,3.427090167207325e-09 +htc-629.jpg,0.004859833512455225 +huawei-2535.jpg,0.039645083248615265 +iphone (7420).jpg,1.0 +huawei-2534.jpg,0.0011331156129017472 +huawei-2536.jpg,0.044090207666158676 +iphone (7416).jpg,1.0 +iphone (7400).jpg,1.0 +iphone (7396).jpg,1.0 +huawei-2537.jpg,8.651422573085589e-20 diff --git a/bevstratov/requirements.txt b/bevstratov/requirements.txt new file mode 100644 index 0000000..ae31ea5 --- /dev/null +++ b/bevstratov/requirements.txt @@ -0,0 +1,365 @@ +absl-py==0.7.1 +alabaster==0.7.12 +albumentations==0.1.12 +altair==3.1.0 +astor==0.8.0 +astropy==3.0.5 +atari-py==0.1.15 +atomicwrites==1.3.0 +attrs==19.1.0 +audioread==2.1.8 +autograd==1.2 +Babel==2.7.0 +backcall==0.1.0 +backports.tempfile==1.0 +backports.weakref==1.0.post1 +beautifulsoup4==4.6.3 +bleach==3.1.0 +blis==0.2.4 +bokeh==1.0.4 +boto==2.49.0 +boto3==1.9.162 +botocore==1.12.162 +Bottleneck==1.2.1 +branca==0.3.1 +bs4==0.0.1 +bz2file==0.98 +cachetools==3.1.1 +certifi==2019.3.9 +cffi==1.12.3 +chainer==5.4.0 +chardet==3.0.4 +cityhash==0.2.3.post9 +Click==7.0 +cloudpickle==0.6.1 +cmake==3.12.0 +colorlover==0.3.0 +community==1.0.0b1 +contextlib2==0.5.5 +convertdate==2.1.3 +coverage==3.7.1 +coveralls==0.5 +crcmod==1.7 +cufflinks==0.14.6 +cupy-cuda100==5.4.0 +cvxopt==1.2.3 +cvxpy==1.0.15 +cycler==0.10.0 +cymem==2.0.2 +Cython==0.29.10 +daft==0.0.4 +dask==1.1.5 +dataclasses==0.6 +datascience==0.10.6 +decorator==4.4.0 +defusedxml==0.6.0 +dill==0.2.9 +distributed==1.25.3 +Django==2.2.2 +dlib==19.16.0 +dm-sonnet==1.33 +docopt==0.6.2 +docutils==0.14 +dopamine-rl==1.0.5 +easydict==1.9 +ecos==2.0.7.post1 +editdistance==0.5.3 +en-core-web-sm==2.1.0 +entrypoints==0.3 +enum34==1.1.6 +ephem==3.7.6.0 +et-xmlfile==1.0.1 +fa2==0.3.5 +fancyimpute==0.4.3 +fastai==1.0.52 +fastcache==1.1.0 +fastdtw==0.3.2 +fastprogress==0.1.21 +fastrlock==0.4 +fbprophet==0.5 +feather-format==0.4.0 +featuretools==0.4.1 +filelock==3.0.12 +fix-yahoo-finance==0.0.22 +Flask==1.0.3 +folium==0.8.3 +future==0.16.0 +gast==0.2.2 +GDAL==2.2.2 +gdown==3.6.4 +gensim==3.6.0 +geographiclib==1.49 +geopy==1.17.0 +gevent==1.4.0 +gin-config==0.1.4 +glob2==0.6 +google==2.0.2 +google-api-core==1.11.1 +google-api-python-client==1.6.7 +google-auth==1.4.2 +google-auth-httplib2==0.0.3 +google-auth-oauthlib==0.3.0 +google-cloud-bigquery==1.8.1 +google-cloud-core==0.29.1 +google-cloud-language==1.0.2 +google-cloud-storage==1.13.2 +google-cloud-translate==1.3.3 +google-colab==1.0.0 +google-resumable-media==0.3.2 +googleapis-common-protos==1.6.0 +googledrivedownloader==0.4 +graph-nets==1.0.4 +graphviz==0.10.1 +greenlet==0.4.15 +grpcio==1.15.0 +gspread==3.0.1 +gspread-dataframe==3.0.2 +gunicorn==19.9.0 +gym==0.10.11 +h5py==2.8.0 +HeapDict==1.0.0 +holidays==0.9.10 +html5lib==1.0.1 +httpimport==0.5.16 +httplib2==0.11.3 +humanize==0.5.1 +hyperopt==0.1.2 +ideep4py==2.0.0.post3 +idna==2.8 +image==1.5.27 +imageio==2.4.1 +imagesize==1.1.0 +imbalanced-learn==0.4.3 +imblearn==0.0 +imgaug==0.2.9 +importlib-metadata==0.17 +imutils==0.5.2 +inflect==2.1.0 +intel-openmp==2019.0 +intervaltree==2.1.0 +ipykernel==4.6.1 +ipython==5.5.0 +ipython-genutils==0.2.0 +ipython-sql==0.3.9 +ipywidgets==7.4.2 +itsdangerous==1.1.0 +jdcal==1.4.1 +jedi==0.13.3 +jieba==0.39 +Jinja2==2.10.1 +jmespath==0.9.4 +joblib==0.13.2 +jpeg4py==0.1.4 +jsonschema==2.6.0 +jupyter==1.0.0 +jupyter-client==5.2.4 +jupyter-console==6.0.0 +jupyter-core==4.4.0 +kaggle==1.5.4 +kapre==0.1.3.1 +Keras==2.2.4 +Keras-Applications==1.0.8 +Keras-Preprocessing==1.1.0 +keras-vis==0.4.1 +kiwisolver==1.1.0 +knnimpute==0.1.0 +librosa==0.6.3 +lightgbm==2.2.3 +llvmlite==0.29.0 +lmdb==0.94 +lucid==0.3.8 +lunardate==0.2.0 +lxml==4.2.6 +magenta==0.3.19 +Markdown==3.1.1 +MarkupSafe==1.1.1 +matplotlib==3.0.3 +matplotlib-venn==0.11.5 +mesh-tensorflow==0.0.5 +mido==1.2.6 +mir-eval==0.5 +missingno==0.4.1 +mistune==0.8.4 +mkl==2019.0 +mlxtend==0.14.0 +mock==3.0.5 +more-itertools==7.0.0 +moviepy==0.2.3.5 +mpi4py==3.0.1 +mpmath==1.1.0 +msgpack==0.5.6 +multiprocess==0.70.7 +multitasking==0.0.9 +murmurhash==1.0.2 +music21==5.5.0 +natsort==5.5.0 +nbconvert==5.5.0 +nbformat==4.4.0 +networkx==2.3 +nibabel==2.3.3 +nltk==3.2.5 +nose==1.3.7 +notebook==5.2.2 +np-utils==0.5.10.0 +numba==0.40.1 +numexpr==2.6.9 +numpy==1.16.4 +nvidia-ml-py3==7.352.0 +oauth2client==4.1.3 +oauthlib==3.0.1 +okgrade==0.4.3 +olefile==0.46 +opencv-contrib-python==3.4.3.18 +opencv-python==3.4.5.20 +openpyxl==2.5.9 +osqp==0.5.0 +packaging==19.0 +pandas==0.24.2 +pandas-datareader==0.7.0 +pandas-gbq==0.4.1 +pandas-profiling==1.4.1 +pandocfilters==1.4.2 +parso==0.4.0 +pathlib==1.0.1 +patsy==0.5.1 +pexpect==4.7.0 +pickleshare==0.7.5 +Pillow==4.3.0 +pip-tools==3.6.1 +plac==0.9.6 +plotly==3.6.1 +pluggy==0.7.1 +portpicker==1.2.0 +prefetch-generator==1.0.1 +preshed==2.0.1 +pretty-midi==0.2.8 +prettytable==0.7.2 +progressbar2==3.38.0 +prometheus-client==0.7.0 +promise==2.2.1 +prompt-toolkit==1.0.16 +protobuf==3.7.1 +psutil==5.4.8 +psycopg2==2.7.6.1 +ptyprocess==0.6.0 +py==1.8.0 +pyarrow==0.13.0 +pyasn1==0.4.5 +pyasn1-modules==0.2.5 +pycocotools==2.0.0 +pycparser==2.19 +pydot==1.3.0 +pydot-ng==2.0.0 +pydotplus==2.0.2 +pyemd==0.5.1 +pyglet==1.3.2 +Pygments==2.1.3 +pygobject==3.26.1 +pymc3==3.7 +pymongo==3.8.0 +pymystem3==0.2.0 +PyOpenGL==3.1.0 +pyparsing==2.4.0 +pyrsistent==0.15.2 +pysndfile==1.3.3 +PySocks==1.7.0 +pystan==2.19.0.0 +pytest==3.6.4 +python-apt==1.6.4 +python-chess==0.23.11 +python-dateutil==2.5.3 +python-louvain==0.13 +python-rtmidi==1.3.0 +python-slugify==3.0.2 +python-utils==2.3.0 +pytz==2018.9 +PyWavelets==1.0.3 +PyYAML==3.13 +pyzmq==17.0.0 +qtconsole==4.5.1 +requests==2.21.0 +requests-oauthlib==1.2.0 +resampy==0.2.1 +retrying==1.3.3 +rpy2==2.9.5 +rsa==4.0 +s3fs==0.2.1 +s3transfer==0.2.1 +scikit-image==0.15.0 +scikit-learn==0.21.2 +scipy==1.3.0 +screen-resolution-extra==0.0.0 +scs==2.1.0 +seaborn==0.9.0 +semantic-version==2.6.0 +Send2Trash==1.5.0 +setuptools-git==1.2 +Shapely==1.6.4.post2 +simplegeneric==0.8.1 +six==1.12.0 +sklearn==0.0 +sklearn-pandas==1.8.0 +smart-open==1.8.4 +snowballstemmer==1.2.1 +sortedcontainers==2.1.0 +spacy==2.1.4 +Sphinx==1.8.5 +sphinxcontrib-websupport==1.1.2 +SQLAlchemy==1.3.4 +sqlparse==0.3.0 +srsly==0.0.5 +stable-baselines==2.2.1 +statsmodels==0.9.0 +sympy==1.1.1 +tables==3.4.4 +tabulate==0.8.3 +tblib==1.4.0 +tensor2tensor==1.11.0 +tensorboard==1.13.1 +tensorboardcolab==0.0.22 +tensorflow==1.13.1 +tensorflow-estimator==1.13.0 +tensorflow-hub==0.4.0 +tensorflow-metadata==0.13.0 +tensorflow-probability==0.6.0 +termcolor==1.1.0 +terminado==0.8.2 +testpath==0.4.2 +text-unidecode==1.2 +textblob==0.15.3 +textgenrnn==1.4.1 +tfds-nightly==1.0.2.dev201906060105 +tflearn==0.3.2 +Theano==1.0.4 +thinc==7.0.4 +toolz==0.9.0 +torch==1.1.0 +torchsummary==1.5.1 +torchtext==0.3.1 +torchvision==0.3.0 +tornado==4.5.3 +tqdm==4.28.1 +traitlets==4.3.2 +tweepy==3.6.0 +typing==3.6.6 +tzlocal==1.5.1 +umap-learn==0.3.9 +uritemplate==3.0.0 +urllib3==1.24.3 +vega-datasets==0.7.0 +wasabi==0.2.2 +wcwidth==0.1.7 +webencodings==0.5.1 +Werkzeug==0.15.4 +widgetsnbextension==3.4.2 +wordcloud==1.5.0 +wrapt==1.11.1 +xarray==0.11.3 +xgboost==0.90 +xkit==0.0.0 +xlrd==1.1.0 +xlwt==1.3.0 +yellowbrick==0.9.1 +zict==0.1.4 +zipp==0.5.1 +zmq==0.0.0 diff --git a/bevstratov/test/htc-626.jpg b/bevstratov/test/htc-626.jpg new file mode 100755 index 0000000..e6be7f1 Binary files /dev/null and b/bevstratov/test/htc-626.jpg differ diff --git a/bevstratov/test/htc-627.jpg b/bevstratov/test/htc-627.jpg new file mode 100755 index 0000000..cf723d1 Binary files /dev/null and b/bevstratov/test/htc-627.jpg differ diff --git a/bevstratov/test/htc-628.jpg b/bevstratov/test/htc-628.jpg new file mode 100755 index 0000000..2116125 Binary files /dev/null and b/bevstratov/test/htc-628.jpg differ diff --git a/bevstratov/test/htc-629.jpg b/bevstratov/test/htc-629.jpg new file mode 100755 index 0000000..d89c43f Binary files /dev/null and b/bevstratov/test/htc-629.jpg differ diff --git a/bevstratov/test/htc-630.jpg b/bevstratov/test/htc-630.jpg new file mode 100755 index 0000000..8ee035a Binary files /dev/null and b/bevstratov/test/htc-630.jpg differ diff --git a/bevstratov/test/htc-631.jpg b/bevstratov/test/htc-631.jpg new file mode 100755 index 0000000..0f847c9 Binary files /dev/null and b/bevstratov/test/htc-631.jpg differ diff --git a/bevstratov/test/htc-632.jpg b/bevstratov/test/htc-632.jpg new file mode 100755 index 0000000..99834f1 Binary files /dev/null and b/bevstratov/test/htc-632.jpg differ diff --git a/bevstratov/test/htc-633.jpg b/bevstratov/test/htc-633.jpg new file mode 100755 index 0000000..e5716b4 Binary files /dev/null and b/bevstratov/test/htc-633.jpg differ diff --git a/bevstratov/test/htc-634.jpg b/bevstratov/test/htc-634.jpg new file mode 100755 index 0000000..d3a8966 Binary files /dev/null and b/bevstratov/test/htc-634.jpg differ diff --git a/bevstratov/test/htc-635.jpg b/bevstratov/test/htc-635.jpg new file mode 100755 index 0000000..a477288 Binary files /dev/null and b/bevstratov/test/htc-635.jpg differ diff --git a/bevstratov/test/htc-636.jpg b/bevstratov/test/htc-636.jpg new file mode 100755 index 0000000..5d4d138 Binary files /dev/null and b/bevstratov/test/htc-636.jpg differ diff --git a/bevstratov/test/htc-637.jpg b/bevstratov/test/htc-637.jpg new file mode 100755 index 0000000..3f5c104 Binary files /dev/null and b/bevstratov/test/htc-637.jpg differ diff --git a/bevstratov/test/huawei-2526.jpg b/bevstratov/test/huawei-2526.jpg new file mode 100755 index 0000000..7e38c82 Binary files /dev/null and b/bevstratov/test/huawei-2526.jpg differ diff --git a/bevstratov/test/huawei-2527.jpg b/bevstratov/test/huawei-2527.jpg new file mode 100755 index 0000000..e011346 Binary files /dev/null and b/bevstratov/test/huawei-2527.jpg differ diff --git a/bevstratov/test/huawei-2528.jpg b/bevstratov/test/huawei-2528.jpg new file mode 100755 index 0000000..66d3ace Binary files /dev/null and b/bevstratov/test/huawei-2528.jpg differ diff --git a/bevstratov/test/huawei-2529.jpg b/bevstratov/test/huawei-2529.jpg new file mode 100755 index 0000000..1992773 Binary files /dev/null and b/bevstratov/test/huawei-2529.jpg differ diff --git a/bevstratov/test/huawei-2530.jpg b/bevstratov/test/huawei-2530.jpg new file mode 100755 index 0000000..5b0c05c Binary files /dev/null and b/bevstratov/test/huawei-2530.jpg differ diff --git a/bevstratov/test/huawei-2531.jpg b/bevstratov/test/huawei-2531.jpg new file mode 100755 index 0000000..f3408f3 Binary files /dev/null and b/bevstratov/test/huawei-2531.jpg differ diff --git a/bevstratov/test/huawei-2532.jpg b/bevstratov/test/huawei-2532.jpg new file mode 100755 index 0000000..b3a0913 Binary files /dev/null and b/bevstratov/test/huawei-2532.jpg differ diff --git a/bevstratov/test/huawei-2533.jpg b/bevstratov/test/huawei-2533.jpg new file mode 100755 index 0000000..1374e47 Binary files /dev/null and b/bevstratov/test/huawei-2533.jpg differ diff --git a/bevstratov/test/huawei-2534.jpg b/bevstratov/test/huawei-2534.jpg new file mode 100755 index 0000000..886f1d3 Binary files /dev/null and b/bevstratov/test/huawei-2534.jpg differ diff --git a/bevstratov/test/huawei-2535.jpg b/bevstratov/test/huawei-2535.jpg new file mode 100755 index 0000000..e0ad04b Binary files /dev/null and b/bevstratov/test/huawei-2535.jpg differ diff --git a/bevstratov/test/huawei-2536.jpg b/bevstratov/test/huawei-2536.jpg new file mode 100755 index 0000000..4c057c5 Binary files /dev/null and b/bevstratov/test/huawei-2536.jpg differ diff --git a/bevstratov/test/huawei-2537.jpg b/bevstratov/test/huawei-2537.jpg new file mode 100755 index 0000000..0702e16 Binary files /dev/null and b/bevstratov/test/huawei-2537.jpg differ diff --git a/bevstratov/test/huawei-2538.jpg b/bevstratov/test/huawei-2538.jpg new file mode 100755 index 0000000..1ad3045 Binary files /dev/null and b/bevstratov/test/huawei-2538.jpg differ diff --git a/bevstratov/test/huawei-2539.jpg b/bevstratov/test/huawei-2539.jpg new file mode 100755 index 0000000..1350c50 Binary files /dev/null and b/bevstratov/test/huawei-2539.jpg differ diff --git a/bevstratov/test/huawei-2540.jpg b/bevstratov/test/huawei-2540.jpg new file mode 100755 index 0000000..86451e6 Binary files /dev/null and b/bevstratov/test/huawei-2540.jpg differ diff --git a/bevstratov/test/huawei-2541.jpg b/bevstratov/test/huawei-2541.jpg new file mode 100755 index 0000000..9e766f5 Binary files /dev/null and b/bevstratov/test/huawei-2541.jpg differ diff --git a/bevstratov/test/huawei-2542.jpg b/bevstratov/test/huawei-2542.jpg new file mode 100755 index 0000000..7aa40d4 Binary files /dev/null and b/bevstratov/test/huawei-2542.jpg differ diff --git a/bevstratov/test/huawei-2543.jpg b/bevstratov/test/huawei-2543.jpg new file mode 100755 index 0000000..bc0b793 Binary files /dev/null and b/bevstratov/test/huawei-2543.jpg differ diff --git a/bevstratov/test/huawei-2544.jpg b/bevstratov/test/huawei-2544.jpg new file mode 100755 index 0000000..cb3bd5a Binary files /dev/null and b/bevstratov/test/huawei-2544.jpg differ diff --git a/bevstratov/test/huawei-2545.jpg b/bevstratov/test/huawei-2545.jpg new file mode 100755 index 0000000..894d386 Binary files /dev/null and b/bevstratov/test/huawei-2545.jpg differ diff --git a/bevstratov/test/huawei-2546.jpg b/bevstratov/test/huawei-2546.jpg new file mode 100755 index 0000000..bb38904 Binary files /dev/null and b/bevstratov/test/huawei-2546.jpg differ diff --git a/bevstratov/test/huawei-2547.jpg b/bevstratov/test/huawei-2547.jpg new file mode 100755 index 0000000..27e00d0 Binary files /dev/null and b/bevstratov/test/huawei-2547.jpg differ diff --git a/bevstratov/test/huawei-2548.jpg b/bevstratov/test/huawei-2548.jpg new file mode 100755 index 0000000..fc32e83 Binary files /dev/null and b/bevstratov/test/huawei-2548.jpg differ diff --git a/bevstratov/test/huawei-2549.jpg b/bevstratov/test/huawei-2549.jpg new file mode 100755 index 0000000..204566a Binary files /dev/null and b/bevstratov/test/huawei-2549.jpg differ diff --git a/bevstratov/test/huawei-2550.jpg b/bevstratov/test/huawei-2550.jpg new file mode 100755 index 0000000..03f588f Binary files /dev/null and b/bevstratov/test/huawei-2550.jpg differ diff --git a/bevstratov/test/iphone (7393).jpg b/bevstratov/test/iphone (7393).jpg new file mode 100755 index 0000000..3dd3b1e Binary files /dev/null and b/bevstratov/test/iphone (7393).jpg differ diff --git a/bevstratov/test/iphone (7394).jpg b/bevstratov/test/iphone (7394).jpg new file mode 100755 index 0000000..d9af1cb Binary files /dev/null and b/bevstratov/test/iphone (7394).jpg differ diff --git a/bevstratov/test/iphone (7395).jpg b/bevstratov/test/iphone (7395).jpg new file mode 100755 index 0000000..99f6f5e Binary files /dev/null and b/bevstratov/test/iphone (7395).jpg differ diff --git a/bevstratov/test/iphone (7396).jpg b/bevstratov/test/iphone (7396).jpg new file mode 100755 index 0000000..2ad6b68 Binary files /dev/null and b/bevstratov/test/iphone (7396).jpg differ diff --git a/bevstratov/test/iphone (7397).jpg b/bevstratov/test/iphone (7397).jpg new file mode 100755 index 0000000..72d11f5 Binary files /dev/null and b/bevstratov/test/iphone (7397).jpg differ diff --git a/bevstratov/test/iphone (7398).jpg b/bevstratov/test/iphone (7398).jpg new file mode 100755 index 0000000..468f62e Binary files /dev/null and b/bevstratov/test/iphone (7398).jpg differ diff --git a/bevstratov/test/iphone (7399).jpg b/bevstratov/test/iphone (7399).jpg new file mode 100755 index 0000000..c538719 Binary files /dev/null and b/bevstratov/test/iphone (7399).jpg differ diff --git a/bevstratov/test/iphone (7400).jpg b/bevstratov/test/iphone (7400).jpg new file mode 100755 index 0000000..10415ab Binary files /dev/null and b/bevstratov/test/iphone (7400).jpg differ diff --git a/bevstratov/test/iphone (7401).jpg b/bevstratov/test/iphone (7401).jpg new file mode 100755 index 0000000..af07c9f Binary files /dev/null and b/bevstratov/test/iphone (7401).jpg differ diff --git a/bevstratov/test/iphone (7402).jpg b/bevstratov/test/iphone (7402).jpg new file mode 100755 index 0000000..5ff6b2e Binary files /dev/null and b/bevstratov/test/iphone (7402).jpg differ diff --git a/bevstratov/test/iphone (7403).jpg b/bevstratov/test/iphone (7403).jpg new file mode 100755 index 0000000..69731ad Binary files /dev/null and b/bevstratov/test/iphone (7403).jpg differ diff --git a/bevstratov/test/iphone (7404).jpg b/bevstratov/test/iphone (7404).jpg new file mode 100755 index 0000000..d0f16aa Binary files /dev/null and b/bevstratov/test/iphone (7404).jpg differ diff --git a/bevstratov/test/iphone (7405).jpg b/bevstratov/test/iphone (7405).jpg new file mode 100755 index 0000000..0ed218c Binary files /dev/null and b/bevstratov/test/iphone (7405).jpg differ diff --git a/bevstratov/test/iphone (7406).jpg b/bevstratov/test/iphone (7406).jpg new file mode 100755 index 0000000..3f911a5 Binary files /dev/null and b/bevstratov/test/iphone (7406).jpg differ diff --git a/bevstratov/test/iphone (7407).jpg b/bevstratov/test/iphone (7407).jpg new file mode 100755 index 0000000..ecd03c6 Binary files /dev/null and b/bevstratov/test/iphone (7407).jpg differ diff --git a/bevstratov/test/iphone (7408).jpg b/bevstratov/test/iphone (7408).jpg new file mode 100755 index 0000000..6647b74 Binary files /dev/null and b/bevstratov/test/iphone (7408).jpg differ diff --git a/bevstratov/test/iphone (7409).jpg b/bevstratov/test/iphone (7409).jpg new file mode 100755 index 0000000..1ee4552 Binary files /dev/null and b/bevstratov/test/iphone (7409).jpg differ diff --git a/bevstratov/test/iphone (7410).jpg b/bevstratov/test/iphone (7410).jpg new file mode 100755 index 0000000..7233681 Binary files /dev/null and b/bevstratov/test/iphone (7410).jpg differ diff --git a/bevstratov/test/iphone (7411).jpg b/bevstratov/test/iphone (7411).jpg new file mode 100755 index 0000000..7480302 Binary files /dev/null and b/bevstratov/test/iphone (7411).jpg differ diff --git a/bevstratov/test/iphone (7412).jpg b/bevstratov/test/iphone (7412).jpg new file mode 100755 index 0000000..f0d7c80 Binary files /dev/null and b/bevstratov/test/iphone (7412).jpg differ diff --git a/bevstratov/test/iphone (7413).jpg b/bevstratov/test/iphone (7413).jpg new file mode 100755 index 0000000..f686f52 Binary files /dev/null and b/bevstratov/test/iphone (7413).jpg differ diff --git a/bevstratov/test/iphone (7414).jpg b/bevstratov/test/iphone (7414).jpg new file mode 100755 index 0000000..9eb6bfd Binary files /dev/null and b/bevstratov/test/iphone (7414).jpg differ diff --git a/bevstratov/test/iphone (7415).jpg b/bevstratov/test/iphone (7415).jpg new file mode 100755 index 0000000..100e38b Binary files /dev/null and b/bevstratov/test/iphone (7415).jpg differ diff --git a/bevstratov/test/iphone (7416).jpg b/bevstratov/test/iphone (7416).jpg new file mode 100755 index 0000000..18692a9 Binary files /dev/null and b/bevstratov/test/iphone (7416).jpg differ diff --git a/bevstratov/test/iphone (7417).jpg b/bevstratov/test/iphone (7417).jpg new file mode 100755 index 0000000..c3049b1 Binary files /dev/null and b/bevstratov/test/iphone (7417).jpg differ diff --git a/bevstratov/test/iphone (7418).jpg b/bevstratov/test/iphone (7418).jpg new file mode 100755 index 0000000..24c7d8a Binary files /dev/null and b/bevstratov/test/iphone (7418).jpg differ diff --git a/bevstratov/test/iphone (7419).jpg b/bevstratov/test/iphone (7419).jpg new file mode 100755 index 0000000..dd18619 Binary files /dev/null and b/bevstratov/test/iphone (7419).jpg differ diff --git a/bevstratov/test/iphone (7420).jpg b/bevstratov/test/iphone (7420).jpg new file mode 100755 index 0000000..e63ecbc Binary files /dev/null and b/bevstratov/test/iphone (7420).jpg differ diff --git a/bevstratov/test/iphone (7421).jpg b/bevstratov/test/iphone (7421).jpg new file mode 100755 index 0000000..a5f577f Binary files /dev/null and b/bevstratov/test/iphone (7421).jpg differ diff --git a/bevstratov/test/iphone (7422).jpg b/bevstratov/test/iphone (7422).jpg new file mode 100755 index 0000000..a3dba95 Binary files /dev/null and b/bevstratov/test/iphone (7422).jpg differ diff --git a/bevstratov/test/iphone (7423).jpg b/bevstratov/test/iphone (7423).jpg new file mode 100755 index 0000000..56debbd Binary files /dev/null and b/bevstratov/test/iphone (7423).jpg differ