IT-Berater: Theo Gottwald (IT-Consultant) > Source Code and Discussions

Neuronal Network revisited

(1/1)

Theo Gottwald:
This code simulates a learning Neuronal Network. To understand what Neural Networks can be  used for think like this:
1. You need a program that converts Binary Numbers (e.g. "0101" to dezimal number e.g. "5").
2. You are too lazy to find an appropriate Algorhytm, or the Algo is too complex (like for example in Face recognition).

So what you do is, you set up an Neuronal Network. See it like a "Self configuring Algorhytm".
Set up properly it will search itself the right "Formula" to solve your problem.

Here its the "Binary to Decimal" Conversion. Which is just an example.

It has 4 Input Neurons, 8 Neurons in the Hidden Layer and 8 Output-Neurons.

0 I ---- H ---- O1
1 I ---- H ---- O2
0 I ---- H ---- O3
1 I ---- H ---- O4

The Input Neurons are given a Binary Number from 1 to 8 (The Input Neurons are set to 0011 for "3" for example).
Then The Output Neuron O3 must have the highest value of all Output Neurons.

When the Net starts the result-values are rather random. After some learning, the net recognizes all binary numbers from 1 to 8.

Note that this version does not have any visible Output. You need to add your "Debug Print" into X_AU () to see something.

--- Code: ---'***********************************************************************************************
'
'
'***********************************************************************************************

#DEBUG ERROR on
#COMPILE EXE

'#INCLUDE ONCE "private\G_MainLib.inc"

MACRO DebS(P1)=STR\$(INT(P1*100000)/100000)
MACRO INST = INSTANCE
MACRO G_REG()
REGISTER R01 AS LONG,R02 AS LONG
END MACRO
MACRO G_S01()
LOCAL S01 AS STRING
END MACRO
MACRO G_S02()
LOCAL S01,S02 AS STRING
END MACRO

MACRO G_S03()
LOCAL S01,S02,S03 AS STRING
END MACRO

' Your Debug Print comes in here
SUB X_AU(BYVAL P1 AS STRING)
' Print String P1
END SUB

'***********************************************************************************************
GLOBAL NA AS NeoNet_Interface

FUNCTION PBMAIN () AS LONG

NA=CLASS "NeoNet"
NA.TrainNet()

END FUNCTION

'***********************************************************************************************

'***********************************************************************************************
CLASS NeoNet
INST ACnt,BCnt,OCnt AS DWORD
INST LearnSteps AS LONG
INST LearnRate AS SINGLE
INST Inputs(), Outputs() AS SINGLE
INST Epsilon AS SINGLE
INST WeightsInput(),WeightsHidden() AS SINGLE
INST BiasHidden(),BiasOut(),HIDDEN() AS SINGLE
INST DeltaHidden(),DeltaOut(),Target() AS SINGLE
INST Net() AS BYTE
INST Total_Error AS SINGLE
' Minimaler Error bei X, Largest Output bei X
INST LAON,EMIN AS LONG
' Soll-Ausgabe
INST Soll AS LONG
'-------------------------------

CLASS METHOD CREATE()

END METHOD
'-------------------------------

'-------------------------------
INTERFACE NeoNet_Interface
INHERIT IUNKNOWN

'-------------------------------

'-------------------------------
METHOD TrainNet()
G_REG
G_S01
ME.Init_Mem()
ME.InitWeights()
FOR R02=1 TO LearnSteps
FOR R01=1 TO OCnt
ME.SetInputsAndTargets(R01)
ME.FeedForward()
ME.FeedBackward()
ME.DrawNet(R02)
NEXT
NEXT
END METHOD
'-------------------------------
PROPERTY GET Inputs(BYVAL A1 AS LONG) AS SINGLE
PROPERTY=Inputs(A1)
END PROPERTY

PROPERTY SET Inputs(BYVAL A1 AS LONG,BYREF A2 AS SINGLE)
Inputs(A1)=A2
END PROPERTY
'-------------------------------
END INTERFACE
'-------------------------------
CLASS METHOD Init_Mem()
ME.InitVars()
DIM Inputs(ACnt)
DIM Outputs(OCnt),Target(OCnt),BiasOut(OCnt)
DIM WeightsInput(BCnt,ACnt),WeightsHidden(OCnt,BCnt)
DIM BiasHidden(BCnt),DeltaHidden(BCnt),HIDDEN(BCnt)
DIM DeltaOut(OCnt)
END METHOD
'-------------------------------
' A1 - Pass-Nummer
CLASS METHOD DrawNet(BYVAL A1 AS DWORD)
REGISTER R01 AS DWORD,R02 AS LONG
G_S02
X_AU "Pass-Nr."+STR\$(A1)+" Total Error: "+TRIM\$(Total_Error)
S01="":S02=""
FOR R01=0 TO ACnt
S01+="I("+TRIM\$(R01)+")="+TRIM\$(Inputs(R01))+" -- "
NEXT r01
X_AU S01

S01=" Soll="+TRIM\$(Soll)+"   Ist="+TRIM\$(LAON)+"  with: "+TRIM\$(Outputs(LAON))
X_AU S01
X_AU "-------------------------------------------------------------------"
'Sleep 10
END METHOD
'-------------------------------
CLASS METHOD InitVars()
G_REG
ACnt=4:BCnt=8:OCnt=8
LearnSteps=130000
LearnRate=0.03
Epsilon=LearnRate
END METHOD
'-------------------------------
CLASS METHOD Activation(BYREF A1 AS SINGLE) AS SINGLE
LOCAL S1 AS SINGLE
S1=1/(1+EXP(-A1))
METHOD=S1
END METHOD
'-------------------------------
CLASS METHOD Abl(BYREF P1 AS SINGLE) AS SINGLE
LOCAL S1 AS SINGLE
S1=P1*(1-P1)
METHOD=S1
END METHOD
'-------------------------------
CLASS METHOD InitWeights()
REGISTER i AS LONG
REGISTER j AS LONG
FOR i=0 TO OCnt
FOR j=0 TO BCnt
WeightsHidden(i,j)=RND()*2 - 1
NEXT
BiasOut(i)=RND()*2 - 1
NEXT

FOR i=0 TO BCnt
FOR j=0 TO ACnt
WeightsInput(i,j)=RND()*2 - 1
NEXT
BiasHidden(i) =RND()*2 - 1
NEXT
END METHOD
'-------------------------------
CLASS METHOD SetInputsAndTargets(BYVAL A1 AS LONG)
G_REG
G_S01
S01=BIN\$(A1,4)
FOR R01=1 TO 4
Inputs(R01)=VAL(MID\$(S01,R01,1))
NEXT r01

FOR R01=1 TO OCnt
Target(R01) = 0
NEXT
Target(A1)=1
Soll=A1
END METHOD

'-------------------------------
CLASS METHOD FeedForward()
REGISTER i AS LONG
REGISTER j AS LONG
LOCAL aH,aI AS SINGLE

FOR i = 0 TO BCnt
aI=0
FOR j=0 TO ACnt
aI+=Inputs(j) * WeightsInput (i,j)
NEXT
HIDDEN(i)  = ME.Activation(aI+BiasHidden(i))
NEXT

FOR i = 0 TO OCnt
aH=0
FOR j=0 TO BCnt
aH+=HIDDEN(j) * WeightsHidden(i,j)
NEXT
Outputs(i) = ME.Activation(aH+BiasOut(i)   )
NEXT

END METHOD

'-------------------------------
CLASS METHOD FeedBackward()
REGISTER i AS LONG
REGISTER j AS LONG
LOCAL S1,S2 AS SINGLE
' Errormin und Largest Output
LOCAL EMI,LAO AS SINGLE
EMI=1:LAO=-1:EMIN=-1:LAON=-1
FOR i=0 TO OCnt
S1=Target(i)-Outputs(i)
S2+=ABS(S1)
DeltaOut(i) = S1 * ME.Abl(Outputs(i))
IF S1<EMI THEN EMI=S1:EMIN=i
IF Outputs(i)>LAO THEN LAO=Outputs(i):LAON=i
NEXT
Total_Error=S2

FOR i=0 TO OCnt
FOR j=0 TO BCnt
WeightsHidden(i,j)+=(Epsilon*DeltaOut(i)*HIDDEN(j))
NEXT
BiasOut(i)+=(Epsilon*DeltaOut(i))
NEXT

FOR i=0 TO OCnt
DeltaHidden(i)=0
FOR j=0 TO BCnt
DeltaHidden(i)+=(DeltaOut(j)*WeightsHidden(i,j))
NEXT
DeltaOut(i)*=ME.Abl(HIDDEN(i))
NEXT

FOR i=0 TO BCnt
FOR j=0 TO ACnt
WeightsInput(i,j)=WeightsInput(i,j) + (Epsilon * DeltaHidden(i) * Inputs(j))
NEXT
BiasHidden(i)+=(Epsilon * DeltaHidden(i))
NEXT
END METHOD
END CLASS

--- End code ---

Theo Gottwald:
Here is the next logical Step. This program gets Binary Numbers from 0 to 127 from Inputs (0 to 5).
It will then Activate the Output-Neuron with that number .... after long time of learning.

Let it run for an hour before you read what the Output is.
It also demonstrates why larger Neuronal Nets are often trained usingthe GPU.
Because it takes too much time using the CPU.

--- Code: ---#COMPILE EXE

'#INCLUDE ONCE "private\G_MainLib.inc"

MACRO DebS(P1)=STR\$(INT(P1*100000)/100000)
MACRO INST = INSTANCE
MACRO G_REG()
REGISTER R01 AS LONG,R02 AS LONG
END MACRO
MACRO G_S01()
LOCAL S01 AS STRING
END MACRO
MACRO G_S02()
LOCAL S01,S02 AS STRING
END MACRO

MACRO G_S03()
LOCAL S01,S02,S03 AS STRING
END MACRO

' Your Debug Print comes in here
SUB X_AU(BYVAL P1 AS STRING)
' Print String P1
END SUB

'***********************************************************************************************
GLOBAL NA AS NeoNet_Interface

FUNCTION PBMAIN () AS LONG

NA=CLASS "NeoNet"
NA.TrainNet()

END FUNCTION

'***********************************************************************************************

'***********************************************************************************************
Class NeoNet
INST ACnt,BCnt,OCnt as dword
INST LearnSteps as long
INST LearnRate as single
INST Inputs(), Outputs() as single
INST Epsilon as single
INST WeightsInput(),WeightsHidden() as single
INST BiasHidden(),BiasOut(),Hidden() as single
INST DeltaHidden(),DeltaOut(),Target() as single
INST TE() as SIngle
INST Epsilondec as Single
INST Net() as byte
INST Total_Error as single
INST Last_Error as single
INST LastChange as Dword
' Minimaler Error bei X, Largest Output bei X
INST LAON,EMIN as long
' Soll-Ausgabe
INST Soll as long
' Pass Nr.
Inst Pass as dword
' Letzter Totaler Fehler über alle Eingaben
INST Last_TE as SIngle
' Richtige Ergebnise
INST Ri as dword
Inst Ok() as Byte
'-------------------------------
'
CLASS METHOD CREATE()

end method
'-------------------------------

'-------------------------------
interface NeoNet_Interface
Inherit Iunknown

'-------------------------------

'-------------------------------
Method TrainNet()
G_REG
G_S01
X_AU "Hallo"
ME.Init_Mem()
ME.InitWeights()
FOr R02=1 to LearnSteps
Pass=R02
for R01=0 to OCnt
ME.SetInputsAndTargets(R01)
ME.FeedForward()
ME.FeedBackward()
ME.DrawNet(R02)
next
ME.CheckEpsilon()
next
end method
'-------------------------------
Property Get Inputs(byval A1 as long) as Single
Property=Inputs(A1)
end property

Property Set Inputs(byval A1 as long,byref A2 as Single)
Inputs(A1)=A2
end property
'-------------------------------

end interface
'-------------------------------
class method InitVars()
G_REG
' Outp=1.23*Hidden
ACnt=6:BCnt=360:OCnt=127
LearnSteps=1300000
LearnRate=0.9
Epsilon=LearnRate
EpsilonDec=0.999
end method
'-------------------------------
class method Init_Mem()
ME.InitVars()
dim Inputs(ACnt)
dim Outputs(OCnt),Target(OCnt),BiasOut(OCnt)
dim WeightsInput(BCnt,ACnt),WeightsHidden(OCnt,BCnt)
dim BiasHidden(BCnt),DeltaHidden(BCnt),Hidden(BCnt)
dim DeltaOut(OCnt),TE(OCnt),Ok(OCnt)
end method
'-------------------------------
' A1 - Pass-Nummer
class Method DrawNet(byval A1 as dword)
Register R01 as DWORD,R02 as Long
G_T02
G_S04

X_AU "Pass-Nr."+STR\$(A1)+" Total Error: "+DebS(Total_Error)+" -- Epsilon="+DebS(Epsilon)+"  Last_TE="+DebS(Last_TE)
if Last_TE>1 or Total_Error>0.1 then exit method
#if 0
S01="":S02=""
For R01=0 to ACnt
S01+="I("+trim\$(R01)+")="+trim\$(Inputs(R01))+" -- "
next r01
X_AU S01
#ENDIF

S01="":S02="":S03="":T01=0:T02=0
For R01=0 to OCnt
'   S01+="O("+trim\$(R01)+")="+DebS(Outputs(R01))+" -- "
'   S02+="Target("+trim\$(R01)+")="+DebS(Target(R01))+" -- "
if Ok(R01)=1 then S04="OK":incr T02 else S04="ER"
S03+=trim\$(R01)+"="+S04+"**"
incr T01
if T01>16 then S03+=\$crlf:T01=0
next r01
S03+=\$crlf+"Total: "+STR\$(T02)+"/"+STR\$(OCnt+1)
if T02=OCnt+1 then SLEEP 4000
Ri=T02
if Ri<(OCnt/2) then exit method
'X_AU S01
'X_AU S02
X_AU S03

S01=" Soll="+trim\$(Soll)+"   Ist="+trim\$(LAON)+"  with: "+trim\$(Outputs(LAON))
X_AU S01
X_AU "-------------------------------------------------------------------"
'Sleep 10
end method

'-------------------------------
class method Activation(byref A1 as single) as single
local S1 as Single
S1=1/(1+exp(-A1))
method=S1
end method
'-------------------------------
clASs Method Abl(byref P1 as single) as SIngle
local S1 as Single
S1=P1*(1-P1)
method=S1
end method
'-------------------------------
class method InitWeights()
Register i as long
Register j as long
for i=0 to OCnt
for j=0 to BCnt
WeightsHidden(i,j)=rnd()*2 - 1
next
BiasOut(i)=rnd()*2 - 1
next

for i=0 to BCnt
for j=0 to ACnt
WeightsInput(i,j)=rnd()*2 - 1
next
BiasHidden(i) =rnd()*2 - 1
next
end method
'-------------------------------
class Method SetInputsAndTargets(byval A1 as Long)
G_REG
G_S01
S01=BIN\$(A1,ACnt+1)
FOR R01=0 TO ACnt
Inputs(R01)=VAL(MID\$(S01,R01+1,1))
NEXT r01

For R01=0 to OCnt
Target(R01) = 0
next
Target(A1)=1
Soll=A1
end method

'-------------------------------
class method FeedForward()
Register i as long
Register j as long
local aH,aI as SIngle

for i = 0 to BCnt
aI=0
for j=0 to ACnt
aI+=Inputs(j) * WeightsInput (i,j)
next
Hidden(i)  = ME.Activation(aI+BiasHidden(i))
next

for i = 0 to OCnt
aH=0
for j=0 to BCnt
aH+=Hidden(j) * WeightsHidden(i,j)
next
Outputs(i) = ME.Activation(aH+BiasOut(i))
next

end method

'-------------------------------
class method FeedBackward()
Register i as long
Register j as long
Local S1,S2 as single
' Errormin und Largest Output
Local EMI,LAO as Single
EMI=1:LAO=-1:EMIN=-1:LAON=-1
for i=0 to OCnt
S1=Target(i)-Outputs(i)
S2+=ABS(S1)
DeltaOut(i) = S1 * ME.Abl(Outputs(i))
if S1<EMI then EMI=S1:EMIN=i
if Outputs(i)>LAO  then LAO=Outputs(i):LAON=i
next
If Soll=LAON then OK(Soll)=1 else Ok(Soll)=0
Total_Error=S2:TE(Soll)=S2

for i=0 to OCnt
for j=0 to BCnt
WeightsHidden(i,j)+=(Epsilon*DeltaOut(i)*Hidden(j))
next
BiasOut(i)+=(Epsilon*DeltaOut(i))
next

for i=0 to OCnt
DeltaHidden(i)=0
for j=0 to BCnt
DeltaHidden(i)+=(DeltaOut(j)*WeightsHidden(i,j))
next
DeltaOut(i)*=ME.Abl(Hidden(i))
next

for i=0 to BCnt
for j=0 to ACnt
WeightsInput(i,j)=WeightsInput(i,j) + (Epsilon * DeltaHidden(i) * Inputs(j))
next
BiasHidden(i)+=(Epsilon * DeltaHidden(i))
next
end method
'-------------------------------
class Method CheckEpsilon()
G_REG
Local S1 as Single
if Epsilon<0.002 then exit method
For R01=0 to OCnt
S1+=TE(R01)
next r01
S1=S1/OCnt
X_AU "Pass: "+trim\$(Pass)+" Total Error="+STR\$(S1)+" Epsilon="+DebS(Epsilon)
if Last_TE<S1 then Epsilon*=Epsilondec
Last_TE=S1
end method
END CLASS

--- End code ---