from datetime import datetime
import time
import tensorflow as tf
import input_data
mnist = input_data.read_data_sets('data/', one_hot=True)
print("MNIST READY")
# 定義網(wǎng)絡(luò)參數(shù)
n_input = 784 # 輸入的維度
n_output = 10 # 標(biāo)簽的維度
learning_rate = 0.001
dropout = 0.75
# 定義函數(shù)print_activations來顯示網(wǎng)絡(luò)每一層結(jié)構(gòu)，展示每一個(gè)卷積層或池化層輸出tensor的尺寸
def print_activations(t):
    print(t.op.name, '', t.get_shape().as_list())
# 定義卷積操作
def conv2d(input, w, b):
    return tf.nn.relu(tf.nn.bias_add(tf.nn.conv2d(input, w, strides=[1, 1, 1, 1], padding='SAME'), b)) # 參數(shù)分別指定了卷積核的尺寸、多少個(gè)channel、filter的個(gè)數(shù)即產(chǎn)生特征圖的個(gè)數(shù)                                                                                       # 步長(zhǎng)為1，即掃描全圖像素,[1, 1, 1, 1]分別代表batch_size、h、w、c的stride
# 定義池化操作
def max_pool(input):
    return tf.nn.max_pool(input, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') # padding有兩種選擇：'SAME'（窗口滑動(dòng)時(shí)，像素不夠會(huì)自動(dòng)補(bǔ)0）或'VALID'（不夠就跳過）兩種選擇
# 定義全連接操作
def fc(input, w, b):
    return tf.nn.relu(tf.add(tf.matmul(input, w), b)) # w*x+b，再通過非線性激活函數(shù)relu
# 定義網(wǎng)絡(luò)結(jié)構(gòu)
def alex_net(_input, _weights, _biases, _keep_prob):
    _input_r = tf.reshape(_input, [-1, 28, 28, 1])  # 對(duì)圖像做一個(gè)預(yù)處理，轉(zhuǎn)換為tf支持的格式，即[n, h, w, c],-1是確定好其它3維后，讓tf去推斷剩下的1維
    with tf.name_scope('conv1'):
        _conv1 = conv2d(_input_r, _weights['wc1'], _biases['bc1'])
        print_activations(_conv1) # 將這一層最后輸出的tensor conv1的結(jié)構(gòu)打印出來

    # # 這里參數(shù)基本都是AlexNet論文中的推薦值，但目前其他經(jīng)典卷積神經(jīng)網(wǎng)絡(luò)模型基本都放棄了LRN（主要是效果不明顯），
    # # 并且使用LRN也會(huì)讓前饋、反饋的速度大大下降（整體速度降到1/3）
    # with tf.name_scope('_lrn1'):
    #     _lrn1 = tf.nn.lrn(_conv1, 4, bias=1.0, alpha=0.001/9, beta=0.75)

    with tf.name_scope('pool1'):
        _pool1 = max_pool(_conv1)
        print_activations(_pool1)

    with tf.name_scope('conv2'):
        _conv2 = conv2d(_pool1, _weights['wc2'], _biases['bc2'])
        print_activations(_conv2)

    # with tf.name_scope('_lrn2'):
    #     _lrn2 = tf.nn.lrn(_conv2, 4, bias=1.0, alpha=0.001/9, beta=0.75)

    with tf.name_scope('pool2'):
        _pool2 = max_pool(_conv2)
        print_activations(_pool2)

    with tf.name_scope('conv3'):
        _conv3 = conv2d(_pool2, _weights['wc3'], _biases['bc3'])
        print_activations(_conv3)

    with tf.name_scope('conv4'):
        _conv4 = conv2d(_conv3, _weights['wc4'], _biases['bc4'])
        print_activations(_conv4)

    with tf.name_scope('conv5'):
        _conv5 = conv2d(_conv4, _weights['wc5'], _biases['bc5'])
        print_activations(_conv5)

    with tf.name_scope('pool3'):
        _pool3 = max_pool(_conv5)
        print_activations(_pool3)

    _densel = tf.reshape(_pool3, [-1, _weights['wd1'].get_shape().as_list()[0]])  # 定義全連接層的輸入，把pool2的輸出做一個(gè)reshape，變?yōu)橄蛄康男问?

    # pool_shape = _pool3.get_shape().as_list()
    # nodes = pool_shape[1] * pool_shape[2] * pool_shape[3]

    with tf.name_scope('fc1'):
        _fc1 = fc(_densel, _weights['wd1'], _biases['bd1'])
        _fc1_drop = tf.nn.dropout(_fc1, _keep_prob) # 為了減輕過擬合，使用Dropout層
        print_activations(_fc1_drop)

    with tf.name_scope('fc2'):
        _fc2 = fc(_fc1_drop, _weights['wd2'], _biases['bd2'])
        _fc2_drop = tf.nn.dropout(_fc2, _keep_prob)
        print_activations(_fc2_drop)

    with tf.name_scope('out'):
        _out = tf.add(tf.matmul(_fc2_drop, _weights['wd3']), _biases['bd3'])
        print_activations(_out)

    return _out
print("CNN READY")

x = tf.placeholder(tf.float32, [None, n_input]) # 用placeholder先占地方，樣本個(gè)數(shù)不確定為None
y = tf.placeholder(tf.float32, [None, n_output]) # 用placeholder先占地方，樣本個(gè)數(shù)不確定為None
keep_prob = tf.placeholder(tf.float32)

# 存儲(chǔ)所有的網(wǎng)絡(luò)參數(shù)
weights = {
    # 使用截?cái)嗟恼龖B(tài)分布（標(biāo)準(zhǔn)差0.1）初始化卷積核的參數(shù)kernel，卷積核大小為3*3，channel為1，個(gè)數(shù)64
    'wc1': tf.Variable(tf.truncated_normal([3, 3, 1, 64], dtype=tf.float32, stddev=0.1), name='weights1'),
    'wc2': tf.Variable(tf.truncated_normal([3, 3, 64, 128], dtype=tf.float32, stddev=0.1), name='weights2'),
    'wc3': tf.Variable(tf.truncated_normal([3, 3, 128, 256], dtype=tf.float32, stddev=0.1), name='weights3'),
    'wc4': tf.Variable(tf.truncated_normal([3, 3, 256, 256], dtype=tf.float32, stddev=0.1), name='weights4'),
    'wc5': tf.Variable(tf.truncated_normal([3, 3, 256, 128], dtype=tf.float32, stddev=0.1), name='weights5'),
    'wd1': tf.Variable(tf.truncated_normal([4*4*128, 1024], dtype=tf.float32, stddev=0.1), name='weights_fc1'),
    'wd2': tf.Variable(tf.random_normal([1024, 1024], dtype=tf.float32, stddev=0.1), name='weights_fc2'),
    'wd3': tf.Variable(tf.random_normal([1024, n_output], dtype=tf.float32, stddev=0.1), name='weights_output')
}
biases = {
    'bc1': tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32), trainable=True, name='biases1'),
    'bc2': tf.Variable(tf.constant(0.0, shape=[128], dtype=tf.float32), trainable=True, name='biases2'),
    'bc3': tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32), trainable=True, name='biases3'),
    'bc4': tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32), trainable=True, name='biases4'),
    'bc5': tf.Variable(tf.constant(0.0, shape=[128], dtype=tf.float32), trainable=True, name='biases5'),
    'bd1': tf.Variable(tf.constant(0.0, shape=[1024], dtype=tf.float32), trainable=True, name='biases_fc1'),
    'bd2': tf.Variable(tf.constant(0.0, shape=[1024], dtype=tf.float32), trainable=True, name='biases_fc2'),
    'bd3': tf.Variable(tf.constant(0.0, shape=[n_output], dtype=tf.float32), trainable=True, name='biases_output')
}

pred = alex_net(x, weights, biases, keep_prob) # 前向傳播的預(yù)測(cè)值
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y)) # 交叉熵?fù)p失函數(shù)，參數(shù)分別為預(yù)測(cè)值_pred和實(shí)際label值y，reduce_mean為求平均loss
optm = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) # 梯度下降優(yōu)化器
corr = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1)) # tf.equal()對(duì)比預(yù)測(cè)值的索引和實(shí)際label的索引是否一樣，一樣返回True，不一樣返回False
accuracy = tf.reduce_mean(tf.cast(corr, tf.float32)) # 將pred即True或False轉(zhuǎn)換為1或0,并對(duì)所有的判斷結(jié)果求均值
# 初始化所有參數(shù)
init = tf.global_variables_initializer()
print("FUNCTIONS READY")

# 上面神經(jīng)網(wǎng)絡(luò)結(jié)構(gòu)定義好之后，下面定義一些超參數(shù)
training_epochs = 1000 # 所有樣本迭代1000次
batch_size = 1 # 每進(jìn)行一次迭代選擇50個(gè)樣本
display_step = 10

sess = tf.Session() # 定義一個(gè)Session
sess.run(init) # 在sess里run一下初始化操作
for epoch in range(training_epochs):
    avg_cost = 0.
    total_batch = int(mnist.train.num_examples/batch_size)
    start_time = time.time()
    for i in range(total_batch):
        batch_xs, batch_ys = mnist.train.next_batch(batch_size) # 逐個(gè)batch的去取數(shù)據(jù)
        # 獲取批數(shù)據(jù)
        sess.run(optm, feed_dict={x: batch_xs, y: batch_ys, keep_prob:dropout})
        avg_cost += sess.run(cost, feed_dict={x: batch_xs, y: batch_ys, keep_prob:1.0})/total_batch
    if epoch % display_step == 0:
        train_accuracy = sess.run(accuracy, feed_dict={x: batch_xs, y: batch_ys, keep_prob: 1.0})
        test_accuracy = sess.run(accuracy, feed_dict={x: mnist.test.images, y: mnist.test.labels, keep_prob:1.0})
        print("Epoch: %03d/%03d cost: %.9f TRAIN ACCURACY: %.3f TEST ACCURACY: %.3f" % (epoch, training_epochs, avg_cost, train_accuracy, test_accuracy))
    # 計(jì)算每輪迭代的平均耗時(shí)mn和標(biāo)準(zhǔn)差sd，并顯示
    duration = time.time() - start_time
    print('%s: step %d, duration = %.3f' % (datetime.now(), epoch, duration))

print("DONE")