In[]:=
NetToExpression[net_]:=Module[{layersList,batchNormFunction,layerFunctions},layersList=Information[net,"LayersList"];batchNormFunction=Function[Block[{sd=Sqrt[#MovingVariance+#Epsilon]},(#2*#Scaling/sd)+(#Biases-(#Scaling*#MovingMean)/sd)]];layerFunctions=layersList/.{layer_LinearLayer:>(Normal@NetExtract[layer,"Weights"].#+Normal@NetExtract[layer,"Biases"]&),layer_ElementwiseLayer:>(ϕ/@#&),layer_BatchNormalizationLayer:>(batchNormFunction[<|"Scaling"->Normal@NetExtract[layer,"Scaling"],"Biases"->Normal@NetExtract[layer,"Biases"],"MovingMean"->Normal@NetExtract[layer,"MovingMean"],"MovingVariance"->Normal@NetExtract[layer,"MovingVariance"],"Epsilon"->Normal@NetExtract[layer,"Epsilon"]|>,#]&)};Fold[#2@#1&,{x},layerFunctions][[1]]];
In[]:=
nsmall=With[{size=3,act="SELU"},NetTrain[NetChain[{LinearLayer[size],BatchNormalizationLayer[],ElementwiseLayer[act],LinearLayer[size],ElementwiseLayer[act],LinearLayer[{}]},"Input"->"Real"],Table[x->Sin[x],{x,0,10,0.01}],RandomSeeding->6543]];
In[]:=
NumberForm[FullSimplify[NetToExpression[nsmall]],2]
Out[]//NumberForm=
0.11-0.17ϕ[0.2+0.72ϕ[2.2-0.49x]-0.42ϕ[1.4-0.28x]-0.11ϕ[-1.7+0.39x]]-0.81ϕ[-0.016-0.36ϕ[2.2-0.49x]+0.59ϕ[1.4-0.28x]+0.3ϕ[-1.7+0.39x]]+0.58ϕ[0.15+0.24ϕ[2.2-0.49x]+0.068ϕ[1.4-0.28x]+0.49ϕ[-1.7+0.39x]]
In[]:=
NetToExpression[NetInitialize[NetChain[{LinearLayer[2],BatchNormalizationLayer[],ElementwiseLayer["SELU"],LinearLayer[2],ElementwiseLayer["SELU"],LinearLayer[{}]},"Input"->"Real"]]]
Out[]=
0.+0.101079ϕ[0.+1.89365ϕ[0.+0.9995(0.-0.508336x)]-0.93114ϕ[0.+0.9995(0.-0.0706019x)]]-0.773974ϕ[0.-1.12706ϕ[0.+0.9995(0.-0.508336x)]+1.08654ϕ[0.+0.9995(0.-0.0706019x)]]
In[]:=
NetToExpression[NetInitialize[NetChain[{LinearLayer[2],ElementwiseLayer["SELU"],LinearLayer[2],ElementwiseLayer["SELU"],LinearLayer[{}]},"Input"->"Real"]]]
Out[]=
0.+0.101079ϕ[0.+1.89365ϕ[0.-0.508336x]-0.93114ϕ[0.-0.0706019x]]-0.773974ϕ[0.-1.12706ϕ[0.-0.508336x]+1.08654ϕ[0.-0.0706019x]]
In[]:=
NetToExpression[NetInitialize[NetChain[{LinearLayer[2],ElementwiseLayer["SELU"],LinearLayer[{}]},"Input"->"Real"]]]
Out[]=
0.-1.12706ϕ[0.-0.508336x]+1.08654ϕ[0.-0.0706019x]
In[]:=
NetGraphPlot[NetInitialize[NetChain[{LinearLayer[2],ElementwiseLayer["SELU"],LinearLayer[{}]},"Input"->"Real"]]]
Out[]=
In[]:=
NetGraphPlot[NetInitialize[NetChain[{ElementwiseLayer["SELU"],LinearLayer[{}]},"Input"->"Real"]]]
Out[]=
In[]:=
NetToExpression[NetInitialize[NetChain[{LinearLayer[{}],ElementwiseLayer["SELU"]},"Input"->"Real"]]]
Out[]=
ϕ[0.-0.508336x]
In[]:=
Plot[ElementwiseLayer["ReLU"][x],{x,-2,2}]
Out[]=
In[]:=
Manipulate[Plot[ElementwiseLayer["ReLU"][ax],{x,-2,2}],{a,-1,1}]
Out[]=
In[]:=
Plot[ElementwiseLayer["SELU"][x],{x,-2,2}]
Out[]=
In[]:=
NetToExpression[NetInitialize[NetChain[{LinearLayer[2],ElementwiseLayer["SELU"],LinearLayer[{}],ElementwiseLayer["SELU"]},"Input"->"Real"]]]
Out[]=
ϕ[0.-1.12706ϕ[0.-0.508336x]+1.08654ϕ[0.-0.0706019x]]
In[]:=
NetGraphPlot[NetInitialize[NetChain[{LinearLayer[2],ElementwiseLayer["SELU"],LinearLayer[{}],ElementwiseLayer["SELU"]},"Input"->"Real"]]]
Out[]=
A polynomial of degree n can go through n+1 points
[[ Should also have a bias at each node ]]
NetInitialize
Is the number of changes of slope simply proportional to the total number of neurons ?