test02_3.py

from __future__ import print_function

# python3 -m tensorflow.tensorboard --logdir=run1:/tmp/ufLearn2/1 --port=6006

# These are all the modules we'll be using later. Make sure you can import them
# before proceeding further.

import numpy as np
import tensorflow as tf
import six.moves.cPickle as pickle
from six.moves import range

logs_path1 = "/tmp/ufLearn2/1"

pickle_file = 'notMNIST.pickle'

with open(pickle_file, 'rb') as f:
    save = pickle.load(f)
    train_dataset = save['train_dataset']
    train_labels = save['train_labels']
    valid_dataset = save['valid_dataset']
    valid_labels = save['valid_labels']
    test_dataset = save['test_dataset']
    test_labels = save['test_labels']
    del save  # hint to help gc free up memory
    print('Training set', train_dataset.shape, train_labels.shape)
    print('Validation set', valid_dataset.shape, valid_labels.shape)
    print('Test set', test_dataset.shape, test_labels.shape)

image_size = 28
num_labels = 10


def reformat(dataset, labels):
    dataset = dataset.reshape((-1, image_size * image_size)).astype(np.float32)
    # Map 0 to [1.0, 0.0, 0.0 ...], 1 to [0.0, 1.0, 0.0 ...]
    labels = (np.arange(num_labels) == labels[:, None]).astype(np.float32)
    return dataset, labels


def weight_variable(shape):
    """Create a weight variable with appropriate initialization."""
    initial = tf.truncated_normal(shape, stddev=0.1)
    return tf.Variable(initial)


def bias_variable(shape):
    """Create a bias variable with appropriate initialization."""
    initial = tf.constant(0.1, shape=shape)
    return tf.Variable(initial)


def variable_summaries(var, name):
    """Attach a lot of summaries to a Tensor."""
    with tf.name_scope('summaries'):
        mean = tf.reduce_mean(var)
        tf.scalar_summary('mean/' + name, mean)
        with tf.name_scope('stddev'):
            stddev = tf.sqrt(tf.reduce_sum(tf.square(var - mean)))
        tf.scalar_summary('sttdev/' + name, stddev)
        tf.scalar_summary('max/' + name, tf.reduce_max(var))
        tf.scalar_summary('min/' + name, tf.reduce_min(var))
        tf.histogram_summary(name, var)


def nn_layer(input_tensor, input_dim, output_dim, layer_name, act=tf.nn.relu):
    """Reusable code for making a simple neural net layer.
It does a matrix multiply, bias add, and then uses relu to nonlinearize.
It also sets up name scoping so that the resultant graph is easy to read,
and adds a number of summary ops.
"""
    # Adding a name scope ensures logical grouping of the layers in the graph.
    with tf.name_scope(layer_name):
        # This Variable will hold the state of the weights for the layer
        with tf.name_scope('weights'):
            weights = weight_variable([input_dim, output_dim])
            variable_summaries(weights, layer_name + '/weights')
        with tf.name_scope('biases'):
            biases = bias_variable([output_dim])
            variable_summaries(biases, layer_name + '/biases')
        with tf.name_scope('Wx_plus_b'):
            preactivate = tf.matmul(input_tensor, weights) + biases
            tf.histogram_summary(layer_name + '/pre_activations', preactivate)
        activations = act(preactivate, 'activation')
        tf.histogram_summary(layer_name + '/activations', activations)
        return activations


def conv2d(x, W):
    with tf.name_scope('conv2d'):
        return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')


def WconvX_plus_b(x, w, b):
    with tf.name_scope('WconvX_plus_b'):
        return tf.nn.relu(tf.matmul(x, w) + b)
        # return tf.nn.relu(conv2d(x, w) + b)

train_dataset, train_labels = reformat(train_dataset, train_labels)
valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)
test_dataset, test_labels = reformat(test_dataset, test_labels)
print('Training set', train_dataset.shape, train_labels.shape)
print('Validation set', valid_dataset.shape, valid_labels.shape)
print('Test set', test_dataset.shape, test_labels.shape)


# With gradient descent training, even this much data is prohibitive.
# Subset the training data for faster turnaround.
train_subset = 10000

graph = tf.Graph()
with graph.as_default():
    # Input data.
    # Load the training, validation and test data into constants that are
    # attached to the graph.
    with tf.name_scope('input'):
        with tf.name_scope('train'):
            tf_train_dataset = tf.constant(train_dataset[:train_subset, :], name='train_dataset')
            tf_train_labels = tf.constant(train_labels[:train_subset], name='train_labels')
        with tf.name_scope('valid'):
            tf_valid_dataset = tf.constant(valid_dataset, name='valid_dataset')
        with tf.name_scope('test'):
            tf_test_dataset = tf.constant(test_dataset, name='test_dataset')

    with tf.name_scope('Layer'):
        with tf.name_scope("weights"):
            weights = tf.Variable(tf.truncated_normal([image_size * image_size, 200]), name='W')
        with tf.name_scope("biases"):
            biases = tf.Variable(tf.zeros([200]), name='b')
        with tf.name_scope('train_Wx_b'):
            # train_logits = tf.matmul(tf_train_dataset, weights) + biases
            train_logits = WconvX_plus_b(tf_train_dataset, weights, biases)
        with tf.name_scope('valid_Wx_b'):
            # valid_logits = tf.matmul(tf_valid_dataset, weights) + biases
            valid_logits = WconvX_plus_b(tf_valid_dataset, weights, biases)
        with tf.name_scope('test_Wx_b'):
            # test_logits = tf.matmul(tf_test_dataset, weights) + biases
            test_logits = WconvX_plus_b(tf_test_dataset, weights, biases)

    with tf.name_scope('Layer2'):
        with tf.name_scope("weights2"):
            weights2 = tf.Variable(tf.truncated_normal([200, num_labels]), name='W')
        with tf.name_scope("biases2"):
            biases2 = tf.Variable(tf.zeros([num_labels]), name='b')
        with tf.name_scope('train_Wx_b2'):
            # train_logits2 = tf.matmul(train_logits, weights2) + biases2
            train_logits2 = WconvX_plus_b(train_logits, weights2, biases2)
        with tf.name_scope('valid_Wx_b2'):
            # valid_logits2 = tf.matmul(valid_logits, weights2) + biases2
            valid_logits2 = WconvX_plus_b(valid_logits, weights2, biases2)
        with tf.name_scope('test_Wx_b2'):
            # test_logits2 = tf.matmul(test_logits, weights2) + biases2
            test_logits2 = WconvX_plus_b(test_logits, weights2, biases2)

    with tf.name_scope('cross_entropy'):
        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=train_logits2))
        # tf.scalar_summary('cross entropy', loss)

    # Optimizer.
    # We are going to find the minimum of this loss using gradient descent.
    with tf.name_scope('train'):
        # train_op = tf.train.GradientDescentOptimizer(0.5).minimize(loss)
        train_op = tf.train.AdamOptimizer(0.01).minimize(loss)

    # Predictions for the training, validation, and test data.
    # These are not part of training, but merely here so that we can report
    # accuracy figures as we train.
    with tf.name_scope("softmax"):
        train_prediction = tf.nn.softmax(train_logits2, name='train_pred.')
        valid_prediction = tf.nn.softmax(valid_logits2, name='valid_pred.')
        test_prediction = tf.nn.softmax(test_logits2, name='test_pred.')

    with tf.name_scope('Accuracy'):
        correct_prediction = tf.equal(tf.argmax(train_prediction, 1), tf.argmax(tf_train_labels, 1))
        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
        # tf.scalar_summary('Accuracy', accuracy)


num_steps = 801


def accuracy(predictions, labels):
    with tf.name_scope('Accuracy'):
        return 100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1)) / predictions.shape[0]


summary_op = tf.merge_all_summaries()

with tf.Session(graph=graph) as session:
    # This is a one-time operation which ensures the parameters get initialized as
    # we described in the graph: random weights for the matrix, zeros for the
    # biases.

    # tf.initialize_all_variables().run()
    session.run(tf.initialize_all_variables())

    writer1 = tf.train.SummaryWriter(logs_path1, graph=tf.get_default_graph())

    print('Initialized')
    for step in range(num_steps):
        # Run the computations. We tell .run() that we want to run the optimizer,
        # and get the loss value and the training predictions returned as numpy
        # arrays.
        _, l, predictions = session.run([train_op, loss, train_prediction])

        # write log
        # writer1.add_summary(l, step)

        if step % 100 == 0:
            # summary_str = session.run(summary_op)
            # writer1.add_summary(l, step)
            print('Loss at step %d: %f' % (step, l))
            print('Training accuracy: %.1f%%' % accuracy(predictions, train_labels[:train_subset, :]))
            # Calling .eval() on valid_prediction is basically like calling run(), but
            # just to get that one numpy array. Note that it recomputes all its graph
            # dependencies.
            print('Validation accuracy: %.1f%%' % accuracy(valid_prediction.eval(), valid_labels))
    print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval(), test_labels))
print("### End 5")

writer1.close()