-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathneural_net_noise.py
159 lines (136 loc) · 4.76 KB
/
neural_net_noise.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
import numpy as np
from sklearn.datasets import fetch_mldata
import pylab as pl
import matplotlib.pyplot as plt
import random as ra
from numpy.random import *
import sys,time
mnist = fetch_mldata('MNIST original')
mnist.data = mnist.data.astype(np.float32)
mnist.data /= 255
mnist.target = mnist.target.astype(np.int32)
N = 60000
data_num = 28*28
input_vector_length = data_num
inter_vector1_length = 300
inter_vector2_length = 300
output_vector_length = 10
learn_ratio = 0.01
max_learn_ratio = 0.5
max_rand_ratio = 0.25
rand_ratio = 0.01
sig_gain = 1
def sigmoid(x):
return 1.0/(1+np.exp(x*(-1)*sig_gain))
def softmax(x):
e = np.exp(x - np.max(x)) # prevent overflow
if e.ndim == 1:
return e / np.sum(e, axis=0)
else:
return e / np.array([np.sum(e, axis=1)]).T
def draw_digit(data):
size = 28
plt.figure(figsize = (2.5,3))
X,Y = np.meshgrid(range(size),range(size))
Z = data.reshape(size,size)
Z = Z[::-1,:]
plt.xlim(0,27)
plt.ylim(0,27)
plt.pcolor(X,Y,Z)
plt.gray()
plt.tick_params(labelbottom = "off")
plt.tick_params(labelleft = "off")
plt.show()
def make_delta_k(vector,err):
return sig_gain*err*vector*(1-vector)
def make_delta_h(vector,sigma_vector):
return sig_gain*vector*(1-vector)*sigma_vector
def make_delta_i(vector,sigma_vector):
return sig_gain*vector*(1-vector)*sigma_vector
def neural_net(learn_gain,rand_rate):
mnist = fetch_mldata('MNIST original')
mnist.data = mnist.data.astype(np.float32)
mnist.data /= 255
mnist.target = mnist.target.astype(np.int32)
x_train,x_test = np.split(mnist.data,[N])
y_train,y_test = np.split(mnist.target,[N])
N_test = y_test.size
weight_vector1 = rand(input_vector_length,inter_vector1_length)/np.sqrt(input_vector_length)
weight_vector2 = rand(inter_vector1_length , inter_vector2_length)/np.sqrt(inter_vector1_length)
weight_vector3 = rand(inter_vector2_length , output_vector_length)/np.sqrt(inter_vector2_length)
b1 = 0
b2 = 0
db1 = 0
db2 = 0
dw1 = 0
dw3 = 0
train_num = N
warukazu = 1
order=np.arange(train_num)
np.random.shuffle(order)
for var in range(0,train_num/warukazu):
seed = order[var]
out = var * 100 / train_num
out_str = "\rlearning now [" +str(out) + "%]"
sys.stdout.write(out_str)
sys.stdout.flush()
input_vector = x_train[seed]
for i in range (0,data_num):
if rand() < rand_rate:
input_vector[i] = rand()
inter_vector1 = sigmoid(np.dot(input_vector,weight_vector1)+b1)
output_vector = softmax(np.dot(inter_vector1,weight_vector3)+b2)
minus = np.zeros(output_vector_length)
minus[y_train[seed]] = 1
reverse_vector3 = output_vector - minus
db2 = reverse_vector3
dw3 = np.dot(inter_vector1.reshape(inter_vector1_length,1),reverse_vector3.reshape(1,output_vector_length))
reverse_vector2 = np.dot(reverse_vector3,weight_vector3.T)
reverse_vector1 = reverse_vector2 * (1.0-inter_vector1) * inter_vector1
db1 = reverse_vector1
dw1 = np.dot(input_vector.reshape(input_vector_length,1),reverse_vector1.reshape(1,inter_vector1_length))
b1 -= learn_gain * db1
b2 -= learn_gain * db2
weight_vector3 -= learn_gain * dw3
weight_vector1 -= learn_gain * dw1
sys.stdout.write("\r=======================================================\n")
sys.stdout.flush()
time.sleep(0.01)
print "learning finished"
print "rand ratio is "+ str(rand_rate)
test_num = 10000
correct_count = 0
count = 0
for var in range(0,test_num):
seed = ra.randint(0,test_num-1)
input_vector = x_test[seed]
for i in range (0,data_num):
if rand() < rand_rate:
input_vector[i] = rand()
inter_vector1 = sigmoid(np.dot(input_vector,weight_vector1)+b1)
output_vector = softmax(np.dot(inter_vector1,weight_vector3)+b2)
if np.argmax(output_vector) == y_test[seed]:
correct_count += 1
print "presition is " + str(correct_count * 100.0 / test_num) + " %"
return correct_count
'''rand_ratio_step = 0.01
learn_count = int((max_rand_ratio - rand_ratio) / rand_ratio_step + 1)
#print learn_count
learn_count_single = 1
learn_count_all = learn_count * learn_count_single
#learn_count = 5
result=np.zeros(learn_count)
for var in range(0,learn_count):
for a in range(0,learn_count_single):
tmp = neural_net(0.1,rand_ratio)
print tmp
result[var] += tmp
rand_ratio += rand_ratio_step
ans = (np.argmax(result) + 1) * 0.01
print "best rand ratio is " + str(ans)
print result
'''
ans = 0
for i in range(0,10):
ans += neural_net(0.1,0.25)
print ans / 10.0