Switch Net 4 neural network

[Luxan]

Just updated diagram to include sign flip at appropriate location .

Code: Select all

'  sciwiseg@gmail.com

'   Rectangle drawing [ updated ]


screen 12
window screen (0,0)-(100,100)



type pt
     x as single
     y as single
end type

dim as integer n,i,cm,j
n=64


dim pts(0 to n-1) as pt
dim chrome(0 to n-1) as integer
'
' Read data
'
restore datax
for i=0 to n-1
  read pts(i).x
next i
restore datay
for i=0 to n-1
  read pts(i).y
next i
restore colorp
for j=0 to (n/2)-1
  read chrome(j)
next j  
'
'  Display connected pts
'
dim as single x,y,x1,y1
' 
'  Line [target,] [[STEP]|(x1, y1)]-[STEP] (x2, y2) [, [color][, [B|BF][, style]]]
' or
'  Line - (x2, y2) [, [color][, [B|BF][, style]]]
' 
dim as single xmov
xmov=10
for i=0 to n-1 step 2 'n-1
    x=pts(i).x +xmov
    y=pts(i).y  
   cm=chrome(int(i/2))
   x1=pts(i+1).x + xmov
   y1=pts(i+1).y
 if x1 > 0 and x > 0 then line(x,y)-(x1,y1),cm,b
   
next i

restore datax2
read x
read x1
restore datay2
read y
read y1
restore colorp2
read cm
    
 if x1 > 0 and x > 0 then line(x,y)-(x1,y1),cm,b
   
color 2,0
locate 2,3
print "in"
'
' ______________________________________________________________________
'
dim as integer txmov
txmov=8
color 10,0
locate 2,2+txmov
print "sign";
locate 3,2+txmov
print "flip";
color 13,0
locate 2,8+txmov
print "wht"
color 14,0
locate 2,12+txmov
print " switch"
color 15,0
locate 2,21+txmov
print "main banks"
color 6,0
locate 2,32+txmov
print " extra"
color 13,0
locate 2, 40+txmov
print "wht"
color 9,0
locate 2,46+txmov
print "out"
color 8,0
locate 18,3+txmov
print "stored banks"
color 11,0
locate 14,50+txmov
print "- banks"
color 4,0
locate 15,50+txmov
print "+ banks"



' within SwNet4()
' wht( result() )
' wht( result(), pow2_shl )
color 13,0
locate 12,1+txmov
print "wht(result())"
locate 12,38 +txmov
print "wht(result(), pow2_shl )"
sleep
end
'
' ======================================================================
'
' ----------------------------------- x --------------------------------
'
datax:
'input
data 2,5
' wht
data 7,15
' switch
data 17,20
' block
data 22,39
' - bank
data 24,27,24,27,29,32,29,32,34,37,34,37
' + bank
data 24,27,24,27,29,32,29,32,34,37,34,37
' xtra
data 41,44
' wht 2
data 46,54
' output
data 56,59
'
' -------------------------- banks 2 ----------------------------------
'
' block
data 22,39
' - bank
data 24,27,24,27,29,32,29,32,34,37,34,37
' + bank
data 24,27,24,27,29,32,29,32,34,37,34,37
'
' ------------------------------ y ------------------------------------
'
datay:
' input
data 10,30
' wht
data 10,30
' switch
data 10,52
' block
data 8,53
' - bank 
data 10,18,21,30,10,18,21,30,10,18,21,30
' + bank 
data 32,40,43,51,32,40,43,51,32,40,43,51
' xtra
data 10, 52
'  wht 2
data 10,30
' output
data 10,30
'
'  ................... banks 2 ......................
'
' block
data 55,100
' - bank 
data 57,65,68,76,57,65,68,76,57,65,68,76
' + bank 
data 79,87,90,98,79,87,90,98,79,87,90,98
'
' ------------------------- colour -------------------------------------
'

colorp:
data 10,13,14,15,11,11,11,11,11,11,4,4,4,4,4,4,6,13,9
data 8,11,11,11,11,11,11,4,4,4,4,4,4

datax2:
data 2,5
datay2:
data 10,30
colorp2:
data 2

[Luxan]

I believe I've sorted out what is happening with the wht used with
switchnet.

The parameter hs_pow2_shl sets the particular passes to use when applying
the wht to the data.
With hs_pow2_shl = 0, all of the passes are evaluated.
For N = 8 samples and hs_pow2_shl = 2, only the last pass is evaluated;
this is pass 3.

It's as if succesive passes are being replaced with neural
network layers.

The situation is illustrated through this code :

Code: Select all

'
'    draw_wht.bas
'
'     wht basics

screen 12

dim as integer x1,y1,x2,y2,chrome

x1=10
y1=40
x2=x1+20
y2=y1+100
chrome=15
locate 1,2
print "in"
line (x1,y1)-(x2,y2),chrome,b

x1=x1+30

x2=x1+20
y2=y1+100
chrome=10
color chrome,0
locate 1,6
print "sign"
locate 2,6
print"flip"
line (x1,y1)-(x2,y2),chrome,b

x1=x1+50

x2=x1+20
y2=y1+100
chrome=11
color chrome,0
locate 1,11
print "Pass 1"
line (x1,y1)-(x2,y2),chrome,b


x1=x1+60

x2=x1+20
y2=y1+100
chrome=11
color chrome,0
locate 1,19
print "Pass 2"
line (x1,y1)-(x2,y2),chrome,b

x1=x1+60

x2=x1+20
y2=y1+100
chrome=11
color chrome,0
locate 1,27
print "Pass 3"
line (x1,y1)-(x2,y2),chrome,b

x1=75
y1=0
x2=260
y2=175
chrome=15
color 11,0
locate 11,20
print "WHT"
line (x1,y1)-(x2,y2),chrome,b

x1=210+60
y1=40
x2=x1+20
y2=y1+100
chrome=12
color chrome,0
locate 1,35
print "Switch"
line (x1,y1)-(x2,y2),chrome,b

x1=x1+30
y1=40
x2=x1+120
y2=y1+100
chrome=15
color chrome,0
locate 1,46
print "NN"
line (x1,y1)-(x2,y2),chrome,b

x1=x2+20

x2=x1+20
y2=y1+100
chrome=11
color chrome,0
locate 1,56
print "Pass 3"
line (x1,y1)-(x2,y2),chrome,b

x1=x1-8
y1=0
x2=x2+30
y2=175
chrome=15
color 11,0
locate 11,57
print "WHT"
line (x1,y1)-(x2,y2),chrome,b

color 15,0
locate 25,4
print "N = 8"
locate 26,4
print"hs_pow2_shl = 2"

locate 13,15
print "wht(result(),0)"

locate 13,55
print "wht(result(),hs_pow2_shl)"

sleep
end
'
' ======================================================================
'

[dafhi]

nice! GreenInks demo uses 2 WHT subs
1. full
2. partial

from the p5js demo they have the same core

Code: Select all

// Fast Walsh Hadamard Transform
function wht(vec) {
  const n = vec.length;
  let hs = 1;
  while (hs < n) {
    let i = 0;
    while (i < n) {
      const j = i + hs;
      while (i < j) {
        var a = vec[i];
        var b = vec[i + hs];
        vec[i] = a + b;
        vec[i + hs] = a - b;
        i += 1;
      }
      i += hs;
    }
    hs += hs;
  }
}

// Partial Fast Walsh Hadamard Transform
function wht4(vec) {
  const n = vec.length;
  let hs = 4;
  while (hs < n) {
    let i = 0;
    while (i < n) {
      const j = i + hs;
      while (i < j) {
        var a = vec[i];
        var b = vec[i + hs];
        vec[i] = a + b;
        vec[i + hs] = a - b;
        i += 1;
      }
      i += hs;
    }
    hs += hs;
  }
}

SwNet4.recall() also looked like a contender for code reduction

my freebasic version SwNet4._abcd() is what i came up with

[dafhi]

if you can think of a more useful name than sh_pow2_shl, feel free to do so.

i'll try to come up with something also

[dafhi]

i translated SwNet4 because it looked faster. SwitchNet is shorter.

he also recommends it

[Luxan]

For the wht, where multiplications aren't involved or twiddle factors, your shorter
sub using while statements is a useful approach.
However being able to compare this to the more traditional representation helps
illuminate what's being attempted.

Now, so far, I've illustrated the situation for an 8 point wht ; what is the flow for larger numbers
of samples, say 16 point. Do we now find that the last two passes are calculated just prior to the
output.

[Luxan]

For the last, illustrated, example pow2_shl = 1, within your SwNet4.recall it's const pow2_shl = 2 .
Hence N>8 is required before the last pass[es] of the wht will be evaluated .

[Luxan]

I updated some code I'd been using previously to examine
the WHT.

With L=4, N=16, hs_pow2_shl = 2, it's apparent that the first two passes
are skipped and the remaining are evaluated.
Is setting hs_pow2_shl = 3, ever considered ?

Elsewhere I examined this via your SwNet4.recall(), with certain
pieces of code commented out.

Here's the updated code I then used to double check the results from there :

Code: Select all

/'


  wht_cmp5.bas
  
  sciwiseg@gmail.com


  Oct 1 2023 ; L=4 => N = 16

Data from SwNet4.recall()

i=[0,...,15],inVec=[1,...,16]
i=[0,...,15],wht(,0)=[136,-8,-16,0,-32,0,0,0,-64,0,0,0,0,0,0,0]
previous as input, returns :
i=[0,...,15],wht(,2)=[40,-8,-16,0,104,-8,-16,0,168,-8,-16,0,232,-8,-16,0]



  fast Walsh Hadamard Transform .
  
  https://au.mathworks.com/help/signal/ug/walshhadamard-transform.html
  
       0 1 2 3 4 5  6 7 
  x = [4 2 2 0 0 2 -2 0]
  y = fwht(x)    "sequency"  format
  y = [1 1 0 1 0 0 1 0]
  
  x1 = ifwht(y)
  x1 = [4 2 2 0 0 2 -2 0]



  Using GNU Octave , with signal package loaded and this script :
  
  x = [4 2 2 0 0 2 -2 0]
  y = fwht(x)
  y = fwht(x,8)
  y = fwht(x,8,"hadamard")
  
  returns :
  
  x =    4   2   2   0   0   2  -2   0

  y =    1   1   0   1   0   0   1   0

  y =    1   1   0   1   0   0   1   0

  y =    1   0   1   0   1   1   0   0
      

'/



declare sub xwht( vec() as single, hs_pow2_shl as long = 0 )
declare sub WHT(A() as single, L as long)

dim as long L
dim as integer n,i
dim as single x

L=4
n=2^L


dim vec(0 to n-1) as single
'
' ..................
'
print " ............................................... "
for i=0 to n-1
    vec(i) =i+1
next i
'
print " vec "
for i=0 to n-1
   print vec(i);" , ";
next i
print
print
'
print " xwht(vec(),0) "
xwht(vec() , 0 )
print
'
'       Now   xwht( , hs_pow2_shl)
'
for i=0 to n-1
    vec(i) = i+1
next i
print " xwht(vec(),2) "
xwht(vec() , 2 )
print

end
'
'=======================================================================
'
'
  '' Fast Walsh Hadamard Transform
  '' As illustrated  
  '' Input vec() , output vec()
  sub xwht( vec() as single, hs_pow2_shl as long = 0 ) '' 1 or 2 for Partial Fast W.H.T.
    static as integer k
    static as long n , p
    n = ubound(vec)+1
    var hs = 1 shl hs_pow2_shl
 '   print " xwht, n , hs ";n;" , ";hs
    p= hs_pow2_shl + 1
    while (hs < n) 
      var i = 0
      while (i < n) 
        var j = i + hs
       ' print " xwht, j ";j
    '   print "xwht, i, i + hs "
        while (i < j) 
     '   print i;" ";i+hs
          var a = vec(i)
          var b = vec(i + hs)
          vec(i) = a + b
          vec(i + hs) = a - b
          i += 1
        wend
        i += hs
      wend
      
      print" ....... "
      print "pass "; p
      print
      p=p+1
      
      hs += hs
      
      
      for k=0 to n-1
        print vec(k);",";
      next k
      print
    wend
  end sub
' _________________________________________________________________
'
'

[dafhi]

i think 3 could be appropriate.

also thinking that hs_pow2_shl could be renamed 'depth'
xwht() is good.

a rash of ideas just an hour ago thinking about 1 to 4 bit neurons and irrational-friendly activation functions, like

0: 0
1: 1
2: e
3: pi

[edit] - i'll use primes

[Luxan]

You're being whimsical , right ?

There are a few things I'm attempting to do , one in particular though tends
to preempt the others; home renovation .

Eventually I'll bring all of the bits and pieces of Switch nets together into,
hopefully, a functioning whole that I , and anyone else, can comprehend.

[dafhi]

whimsical in the sense that i plan to code it later. i've been busy myself

[Luxan]

It's just your option 1. 0, leaves me puzzled; do you mean to dynamically
prune the NN connections .

I've noticed my time stamp, and yours, we appear to be on opposite sides
of the world.

[dafhi]

my SparseNet prunes. where do you see 1 and 0?

my time zone is Pacific (same as Los Angeles)

[Luxan]

It's option 0: 0 .

SparseNet, is this new to the Freebasic forum ?

[dafhi]

if you mean the WHT,
i use optional as often as i can, for less text later

SparseNet i linked to earlier with blue white connections.


[edit] - actually, i'm not certain my SparseNet prunes