MNINST (Deep Learning 역사 ver) [he's intializer]

# 1. Training Data Set 확보

 

import numpy as np

import pandas as pd

from sklearn.model_selection import train_test_split

from sklearn.preprocessing import MinMaxScaler

 

# Raw Data Loading

df = pd.read_csv('/content/drive/MyDrive/융복합 프로젝트형 AI 서비스 개발(2021.06)/09월/16일(목요일)/mnist/train.csv')

 

# train 데이터와 test 데이터를 분리

train_x_data, test_x_data, train_t_data, test_t_data = \

train_test_split(df.drop('label', axis=1, inplace=False),

                 df['label'],

                 random_state=0,

                 test_size=0.3,

                 stratify=df['label'])

 

# 결측치 이상치는 존재하지 않아요!

# 정규화는 진행해야 합니다.

scaler = MinMaxScaler()

scaler.fit(train_x_data)

train_x_data_norm = scaler.transform(train_x_data)

test_x_data_norm = scaler.transform(test_x_data)

 

# 2. sklearn으로 구현

 

from sklearn.linear_model import LogisticRegression

 

model = LogisticRegression(solver='saga')

 

#saga는 stochastic gradient algorithm 보다 data가 많을때는 saga

# 속성으로 C를 줄수 있으며 L2 강도조절가능

  L2는 기본 장착

 

model.fit(train_x_data_norm, train_t_data)

 

# model 평가

model_score = model.score(test_x_data_norm, test_t_data)

print('sklearn의 정확도 : {}'.format(model_score))

 

# sklearn의 정확도 : 0.917063492063492

 

# 3. tensorflow 1.15로 구현

 

# 3. tensorflow 1.15로 구현(multinomial classification)

# label의 class들에 대해 one-hot encoding처리를 해야 해요!

 

import tensorflow as tf

 

sess = tf.Session()

train_t_data_onehot = sess.run(tf.one_hot(train_t_data, depth=10))

test_t_data_onehot = sess.run(tf.one_hot(test_t_data, depth=10))

 

# placeholder

X = tf.placeholder(shape=[None,784], dtype=tf.float32)

T = tf.placeholder(shape=[None,10], dtype=tf.float32)

 

# Weight & bias

W = tf.Variable(tf.random.normal([784,10]))

b = tf.Variable(tf.random.normal([10]))

 

# hypothesis

logit = tf.matmul(X,W) + b

H = tf.nn.softmax(logit)

 

# loss

loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=logit,

                                                                 labels=T))

 

# train

train = tf.train.GradientDescentOptimizer(learning_rate=1e-1).minimize(loss)

 

# session과 초기화

sess.run(tf.global_variables_initializer())

# 여기서 초기화는 w초기화가 아닌 tf의 변수를 만들때 초기에 사용한느 기법 /

  나중에 2.0에서는 필요 없어서 빠졌다

 

# batch를 이용해서 training data를 분할해서 학습을 진행(메모리문제때문에...)해야 해요!

# 여기서는 batch처리를 생략해서 구현할께요!

 

# 학습을 진행

for step in range(3000):

    

    tmp, loss_val = sess.run([train,loss], feed_dict={X:train_x_data_norm,

                                                      T:train_t_data_onehot})

    

    if step % 300 == 0:

        print('loss : {}'.format(loss_val))

 

# Evaluation(accuracy를 측정해보아요!)

 

predict = tf.argmax(H,1)  # 숫자 1은 axis=1의 의미.

correct = tf.equal(predict, tf.argmax(T,1))

acc = tf.reduce_mean(tf.cast(correct, dtype=tf.float32))

 

result = sess.run(acc, feed_dict={X:test_x_data_norm,

                                  T:test_t_data_onehot})

 

print('Tensorflow 1.15의 정확도(multinomial) : {}'.format(result))

 

# Tensorflow 1.15의 정확도(multinomial) : 0.8509523868560791        

 

# 4. tensorflow 2.6로 구현(multinomial classification)

 

import tensorflow as tf

from tensorflow.keras.models import Sequential

from tensorflow.keras.layers import Flatten, Dense

from tensorflow.keras.optimizers import Adam

 

keras_model = Sequential()

 

keras_model.add(Flatten(input_shape=(train_x_data_norm.shape[1],)))

keras_model.add(Dense(10, activation='softmax'))

 

keras_model.compile(optimizer=Adam(learning_rate=1e-1),

                    loss='sparse_categorical_crossentropy',

                    metrics=['accuracy'])

 

histroy = keras_model.fit(train_x_data_norm,

                          train_t_data,

                          epochs=100,

                          verbose=0,

                          validation_split=0.2,

                          batch_size=100)

 

result = keras_model.evaluate(test_x_data_norm,test_t_data)

 

print('Tensorflow 2.6의 정확도(multinomial) : {}'.format(result))

 

# Tensorflow 2.6의 정확도(multinomial) : 0.8784920573234558

 

# 5. tensorflow 1.15로 구현(multilayer neural network을 이용한 multinomial classification)

 

# label의 class들에 대해 one-hot encoding처리를 해야 해요!

 

# - hiddent layer가 포함되요! 3개의 hidden layer를 이용

 

# - weight의 초기값을 random이 아닌 2가지 기법 중 하나를 이용

# 많은 w초기화 기법이 있지만 많이 사용하는 2개를 사용 , 현재까지도 많이 연구중

#   => Xavier[자비어] 초기화, He's 초기화(Xavier 초기화의 개량버전)

#      weight의 값을 랜덤으로 하지 않고 계산하는 기존에 사용한 복잡한 수식이 존재했었어요.

#      그래서 2가지 알고리즘이 나왔는데 그게 Xavier 초기화, He's 초기화

#      입력의 개수와 출력의 개수를 기반으로 적절한 수식을 이용해서 초기값을 지정.

 

#       W = np.random.randn(input의 개수, output의 개수) / np.sqrt[루트](input의 개수)    Xavier

#       W = np.random.randn(input의 개수, output의 개수) / np.sqrt(input의 개수 / 2)     he's

 

# - activation function을 기존의 sigmoid에서 relu(Rectified Linear Unit) [0이상] 변경

                                                           [banishing gradient 없애기위해]

# anaconda prompt => conda install tensorflow-estimator==1.15.1

 

import tensorflow as tf

 

sess = tf.Session()

train_t_data_onehot = sess.run(tf.one_hot(train_t_data, depth=10))

test_t_data_onehot = sess.run(tf.one_hot(test_t_data, depth=10))

 

# placeholder

X = tf.placeholder(shape=[None,784], dtype=tf.float32)

T = tf.placeholder(shape=[None,10], dtype=tf.float32)

 

# Weight & bias

W2 = tf.get_variable('weight2', shape=[784,512],

                             initializer=tf.contrib.layers.variance_scaling_initializer())

b2 = tf.Variable(tf.random.normal([512]))

layer2 = tf.nn.relu(tf.matmul(X,W2) + b2)

 

W3 = tf.get_variable('weight3', shape=[512,256],

                    initializer=tf.contrib.layers.variance_scaling_initializer())

b3 = tf.Variable(tf.random.normal([256]))

layer3 = tf.nn.relu(tf.matmul(layer2,W3) + b3)

 

W4 = tf.get_variable('weight4', shape=[256,128],

                    initializer=tf.contrib.layers.variance_scaling_initializer())

b4 = tf.Variable(tf.random.normal([128]))

layer4 = tf.nn.relu(tf.matmul(layer3,W4) + b4)

 

W5 = tf.get_variable('weight5', shape=[128,10],

                    initializer=tf.contrib.layers.variance_scaling_initializer())

b5 = tf.Variable(tf.random.normal([10]))

# hypothesis

logit = tf.matmul(layer4,W5) + b5

H = tf.nn.softmax(logit)

 

# loss

loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=logit,

                                                                 labels=T))

 

# train

train = tf.train.GradientDescentOptimizer(learning_rate=1e-1).minimize(loss)

 

# session과 초기화

sess.run(tf.global_variables_initializer())

 

# batch를 이용해서 training data를 분할해서 학습을 진행(메모리문제때문에...)해야 해요!

# 여기서는 batch처리를 생략해서 구현할께요!

 

# 학습을 진행

for step in range(3000):

    

    tmp, loss_val = sess.run([train,loss], feed_dict={X:train_x_data_norm,

                                                      T:train_t_data_onehot})

    

    if step % 300 == 0:

        print('loss : {}'.format(loss_val))

        

# Evaluation(accuracy를 측정해보아요!)

 

predict = tf.argmax(H,1)  # 숫자 1은 axis=1의 의미.

correct = tf.equal(predict, tf.argmax(T,1))

acc = tf.reduce_mean(tf.cast(correct, dtype=tf.float32))

 

result = sess.run(acc, feed_dict={X:test_x_data_norm,

                                  T:test_t_data_onehot})

 

print('Tensorflow 1.15의 정확도(neural network multinomial) : {}'.format(result))

 

# Tensorflow 1.15의 정확도(neural network multinomial) : 0.9688095450401306    

# anaconda prompt => conda install tensorflow-estimator==1.15.1

  2.0으로 넘어가면서 위에껄 안깔면 오류가 난다

 

# 6. tensorflow 2.6로 구현(multilayer neural network을 이용한 multinomial classification)

import tensorflow as tf

from tensorflow.keras.models import Sequential

from tensorflow.keras.layers import Flatten, Dense

from tensorflow.keras.optimizers import Adam

 

keras_model = Sequential()

 

keras_model.add(Flatten(input_shape=(train_x_data_norm.shape[1],)))

 

ㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡ 초기화는 알아서해줌 

keras_model.add(Dense(512, activation='relu'))

keras_model.add(Dense(256, activation='relu'))

keras_model.add(Dense(128, activation='relu'))

 

keras_model.add(Dense(10, activation='softmax'))

ㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡㅡ

 

keras_model.compile(optimizer=Adam(learning_rate=1e-4),

                    loss='sparse_categorical_crossentropy',

                    metrics=['accuracy'])

 

histroy = keras_model.fit(train_x_data_norm,

                          train_t_data,

                          epochs=100,

                          verbose=1,

                          validation_split=0.2,

                          batch_size=100)

 

result = keras_model.evaluate(test_x_data_norm,test_t_data)

 

print('Tensorflow 2.6의 정확도(multilayer neural network) : {}'.format(result))

 

# Tensorflow 2.6의 정확도(multilayer neural network) : 0.9721428751945496