5.6 Deep Neural Network

5.6.1 Dropout

In[]:=
session=StartExternalSession["Python"]
Out[]=
ExternalSessionObject
System: Python
EvaluationCount:
None
UUID: aaf53c8f-e53e-4593-8e8a-43649c051d74

Test
〉
In[]:=
5+6
Out[]=
11
In[]:=
drop=DropoutLayer[0.65,"Input"NetEncoder["Image"],"Output"NetDecoder["Image"]]
Out[]=
DropoutLayer

Parameters
​
Dropout probability:
0.65
Method:
Dropout
Ports
Input:
image
Output:
image

In[]:=
drop
,NetEvaluationMode"Train"
Out[]=

5.6.2 ReLU

In[]:=
Clear[x]
In[]:=
Plot[Ramp[x],{x,-2,2}]
Out[]=
In[]:=
Simplify[Ramp[x]-xUnitStep[x]]
Out[]=
0

5.6.3 Softmax Layer

In[]:=
softmax=SoftmaxLayer["Input"{3}]
Out[]=
SoftmaxLayer

Level:
-1
Output:
vector
(size: 3)

In[]:=
softmax[{2,1,0.1}]
Out[]=
{0.659001,0.242433,0.0985659}

5.6.4 Cross Entropy Loss

In[]:=
Plot[{-Log[y],-Log[1-y]},{y,0,1},FrameTrue,FrameLabel{"y"},PlotLegends{"d = 1  -Log(y)","d = 0  -Log(1-y)"}]
Out[]=
d = 1  -Log(y)
d = 0  -Log(1-y)
In[]:=
loss=CrossEntropyLossLayer["Probabilities"]
Out[]=
CrossEntropyLossLayer

Parameters
​
Target form:
Probabilities
Ports
Input:
array of rank ≥
1
Target:
array of rank ≥
1
Loss:
real

In[]:=
loss[<|"Input"->{0.2,0.7},"Target"->{0.3,0.8}|>]
Out[]=
0.768171
In[]:=
loss=MeanSquaredLossLayer["Input"->{2}]
Out[]=
MeanSquaredLossLayer

Ports
​
Input:
vector
(size: 2)
Target:
length-
2
vector
Loss:
real

In[]:=
loss[<|"Input"->{0.2,0.7},"Target"->{0.3,0.8}|>]
Out[]=
0.01

5.6.9 Classifying Double Spirals

In[]:=
spirals=Import["M:\\spiral.dat"];
https://datascience.stackexchange.com/questions/22830/deep-neural-network-using-keras-tensorflow-solves-spiral-dataset-classification
In[]:=
n=Length[spirals]
Out[]=
194
In[]:=
s1={};s2={};
In[]:=
Do[If[spirals[[i,3]]0,AppendTo[s1,{spirals[[i,1]],spirals[[i,2]]}],AppendTo[s2,{spirals[[i,1]],spirals[[i,2]]}]],{i,1,n}]
In[]:=
S1=ListPlot[s1,PlotStyle{RGBColor[1,0,0],PointSize[0.02]},AspectRatio1];
In[]:=
S2=ListPlot[s2,PlotStyle{RGBColor[0,0,1],PointSize[0.015]},AspectRatio1];
In[]:=
pspiral=Show[{S1,S2},PlotRangeAll]
Out[]=
In[]:=
S1=Map[#{0}&,s1];S2=Map[#{1}&,s2];
In[]:=
trainingdata=Join[S1,S2];
In[]:=
net=NetChain[{16,Ramp,16,Ramp,16,Ramp,16,ElementwiseLayer[1+Tanh[#]&],1},"Input"{2},"Output"{1}]
〉
In[]:=
import numpy as np
import pandas as pd
np.random.seed(4375689)
〉
In[]:=
from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import RMSprop
〉
In[]:=
train_data = pd.read_csv('M:\\spiralom.csv').values
train_X = train_data[:,0:2]
train_y = train_data[:,2]
〉
In[]:=
model = Sequential()
model.add(Dense(16, activation='relu', input_shape=(2,)))
model.add(Dense(16, activation='relu'))
model.add(Dense(16, activation='relu'))
model.add(Dense(16, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
〉
In[]:=
model.compile(loss='binary_crossentropy', optimizer=RMSprop(),
metrics=['accuracy'])
〉
In[]:=
history = model.fit(train_X, train_y, batch_size=32, epochs=2000, verbose=0)
〉
In[]:=
score = model.evaluate(train_X, train_y, verbose=0)
〉
In[]:=
score
〉
In[]:=
​
u=model.predict(train_X)
u