Sorting samples into the most abundant.

General FreeBASIC programming questions.
srvaldez
Posts: 3379
Joined: Sep 25, 2005 21:54

Re: Sorting samples into the most abundant.

Post by srvaldez »

hi dodicat
I tweaked your code to use the Intel Decimal Floating-Point Math Library viewtopic.php?t=29203
I hope you guys don't mind, if this post is a bother then feel free to delete it

Code: Select all

#include once "libbid.bi"

type range
    as BID.Decimal128 _max,_min,sd
    as long _maxi,_mini
end type

function mean(a() as BID.Decimal128,R as range) as BID.Decimal128
    'R=type(-1e10,1e10,0,0,0)
    R._min="1e-10"
    R._max="-1e-10"
    R.sd=0
    R._maxi=0
    R._mini=0
    dim as long lb=lbound(a),ub=ubound(a)
    dim as BID.Decimal128 acc,m
    for n as long=lb to ub
        acc+=a(n)
        if R._max<a(n) then R._max=a(n):R._maxi=n
        if R._min>a(n) then R._min=a(n):R._mini=n
    next
    m=acc/(ub-lb+1)
    acc=0
    for n as long=lb to ub
        acc+=(a(n)-m)*(a(n)-m)
    next
    R.sd =sqr(acc/(ub-lb))
    return m
end function
Type vector
    Dim As BID.Decimal128 element(Any)
End Type

Type matrix 
    Dim As BID.Decimal128 element(Any,Any)
    Declare Function inverse() As matrix
    Declare Function transpose() As matrix
    private:
    Declare Function GaussJordan(As vector) As vector
End Type

'mult operators
Operator *(m1 As matrix,m2 As matrix) As matrix
Dim rows As Integer=Ubound(m1.element,1)
Dim columns As Integer=Ubound(m2.element,2)
If Ubound(m1.element,2)<>Ubound(m2.element,1) Then
    Print "Can't do"
    Exit Operator
End If
Dim As matrix ans
Redim ans.element(rows,columns)
Dim rxc As BID.Decimal128
For r As Integer=1 To rows
    For c As Integer=1 To columns
        rxc=0
        For k As Integer = 1 To Ubound(m1.element,2)
            rxc=rxc+m1.element(r,k)*m2.element(k,c)
        Next k
        ans.element(r,c)=rxc
    Next c
Next r
Operator= ans
End Operator

Operator *(m1 As matrix,m2 As vector) As vector
Dim rows As Integer=Ubound(m1.element,1)
Dim columns As Integer=Ubound(m2.element,2)
If Ubound(m1.element,2)<>Ubound(m2.element) Then
    Print "Can't do"
    Exit Operator
End If
Dim As vector ans
Redim ans.element(rows)
Dim rxc As BID.Decimal128
For r As Integer=1 To rows
    rxc=0
    For k As Integer = 1 To Ubound(m1.element,2)
        rxc=rxc+m1.element(r,k)*m2.element(k)
    Next k
    ans.element(r)=rxc
Next r
Operator= ans
End Operator

Function matrix.transpose() As matrix
    Dim As matrix b
    Redim b.element(1 To Ubound(this.element,2),1 To Ubound(this.element,1))
    For i As Long=1 To Ubound(this.element,1)
        For j As Long=1 To Ubound(this.element,2) 
            b.element(j,i)=this.element(i,j)
        Next
    Next
    Return b
End Function

Function matrix.GaussJordan(rhs As vector) As vector
    Dim As Integer n=Ubound(rhs.element)
    Dim As vector ans=rhs,r=rhs
    Dim As matrix b=This
    #macro pivot(num)
    For p1 As Integer  = num To n - 1
        For p2 As Integer  = p1 + 1 To n  
            If Abs(b.element(p1,num))<Abs(b.element(p2,num)) Then
                Swap r.element(p1),r.element(p2)
                For g As Integer=1 To n
                    Swap b.element(p1,g),b.element(p2,g)
                Next g
            End If
        Next p2
    Next p1
    #endmacro
    For k As Integer=1 To n-1
        pivot(k)
        For row As Integer =k To n-1
            If b.element(row+1,k)=0 Then Exit For
            Var f=b.element(k,k)/b.element(row+1,k)
            r.element(row+1)=r.element(row+1)*f-r.element(k)
            For g As Integer=1 To n
                b.element((row+1),g)=b.element((row+1),g)*f-b.element(k,g)
            Next g
        Next row
    Next k
    'back substitute 
    For z As Integer=n To 1 Step -1
        ans.element(z)=r.element(z)/b.element(z,z)
        For j As Integer = n To z+1 Step -1
            ans.element(z)=ans.element(z)-(b.element(z,j)*ans.element(j)/b.element(z,z))
        Next j
    Next    z
    Function = ans
End Function

Function matrix.inverse() As matrix
    Var ub1=Ubound(this.element,1),ub2=Ubound(this.element,2)
    Dim As matrix ans
    Dim As vector temp,null_
    Redim temp.element(1 To ub1):Redim null_.element(1 To ub1)
    Redim ans.element(1 To ub1,1 To ub2)
    For a As Integer=1 To ub1
        temp=null_
        temp.element(a)=1
        temp=GaussJordan(temp)
        For b As Integer=1 To ub1
            ans.element(b,a)=temp.element(b)
        Next b
    Next a
    Return ans
End Function

'vandermode of x
Function vandermonde(x_values() As BID.Decimal128,w As Long) As matrix
    Dim As matrix mat
    Var n=Ubound(x_values)
    Redim mat.element(1 To n,1 To w)
    For a As Integer=1 To n
        For b As Integer=1 To w
            mat.element(a,b)=x_values(a)^(b-1)
        Next b
    Next a
    Return mat
End Function

'main preocedure
Sub regress(x_values() As BID.Decimal128,y_values() As BID.Decimal128,ans() As BID.Decimal128,n As Long)
    Redim ans(1 To Ubound(x_values))
    Dim As matrix m1= vandermonde(x_values(),n)
    Dim As matrix T=m1.transpose
    Dim As vector y 
    Redim y.element(1 To Ubound(ans))
    For n As Long=1 To Ubound(y_values)
        y.element(n)=y_values(n)
    Next n
    Dim As vector result=(((T*m1).inverse)*T)*y
    Redim Preserve ans(1 To n)
    For n As Long=1 To Ubound(ans)
        ans(n)=result.element(n)
    Next n
End Sub

'Evaluate a polynomial at x
Function polyeval(Coefficients() As BID.Decimal128,Byval x As BID.Decimal128) As BID.Decimal128
    Dim As BID.Decimal128 acc
    For i As Long=Ubound(Coefficients) To Lbound(Coefficients) Step -1
        acc=acc*x+Coefficients(i)
    Next i
    Return acc
End Function

Dim As BID.Decimal128 sample()

Dim As Long f = Freefile
Dim As Double d
Open "Dinosaur.txt" For Input As #f
Do Until EOF(f)
    Redim Preserve sample(Ubound(sample) + 1)
    Input #f, d
    sample(Ubound(sample))=d
Loop
Close #f

Screenres 1070,500
width 1070\8,500\16

dim as BID.Decimal128 x(),y()
dim as long counter
For I As Long = 0 To Ubound(sample)
    counter+=1
    redim preserve x(1 to counter)
    redim preserve y(1 to counter)
    x(counter)=i
    y(counter)=sample(i)
    If I = 0 Then  
        Pset(I+200, sample(I) - 26500)
    Else
        Line -(I+200, sample(I) - 26500)
    End If
Next I

Redim As BID.Decimal128 ans()
redim as BID.Decimal128 result(0 to Ubound(sample))
    regress(x(),y(),ans(),9)
    
   For x As Single=0 To Ubound(sample)
       result(x)=polyeval(ans(),x)
        Var f=polyeval(ans(),x)-26500
        If x=0 Then Pset(x+200,f),5 Else Line-(x+200,f),2
    Next

dim as range R
dim as BID.Decimal128 m
var mn=mean(sample(),R)
print "Mean ";mn;"  Standard dev. ";R.sd;" Min ";R._min;" Max ";R._max;" difference ";R._max-R._min
print
  mn=mean(result(),R) 
  color 2
  print "Mean ";mn;"  Standard dev. ";R.sd;" Min ";R._min;" Max ";R._max;" difference ";R._max-R._min
   

Sleep
<edit>
question
could this problem be considered as noise in data?
if so, then perhaps an algorithm to reduce noise could be used
dodicat
Posts: 7983
Joined: Jan 10, 2006 20:30
Location: Scotland

Re: Sorting samples into the most abundant.

Post by dodicat »

Thanks srvaldez, I can get a regression curve degree 20 with 128 bits (line 221ish.(regress(x(),y(),ans(),20)
Regression doesn't ignore any peaks, but uses all the data to statistically create a smooth curve to approximate the data.
https://statisticsbyjim.com/regression/ ... egression/
If the peaks (spikes)are rubbish values, and can safely be ignored then a regression curve is maybe not the route to go.
degree 9 is a pretty good curve, degree 20 also, gives a standard deviation of about 13.
Dinosaur
Posts: 1481
Joined: Jul 24, 2005 1:13
Location: Hervey Bay (.au)

Re: Sorting samples into the most abundant.

Post by Dinosaur »

Hi All

WOW, I am glad there are some members here that have strengths in that area, as I have no hope of understanding what you have done.

Using a number of data files that I have saved, each time the result looks reliable.
I will have to add it to the running code to see physically what happens.
Theoretically if there is only the load of the Rpi running, then the result should slowly creep down as the battery drains.
Will be interesting to see how long the filtering takes on the Rpi.

I slowed down the samples per second to 64 and only read 250 samples, to see the difference.
The larger negative spikes had gone, but an interesting pattern appeared.
Every 4 or 5 samples in the correct range, one sample would drop by 10-15 and then repeat all the way through the saved file.
At 64 samples per second each sample takes 15.6 mSec , so I wait 16 msec between getting samples.
I guess statistically you can't really sync with the A2D converter in free running mode.
Previously if I read to fast, I would simply get a repeat of the previous sample, so ????

Reading more on forums, some comments suggested that reading the chip in continuous mode is prone to noise and other difficulties.
Today I will test using single shot mode and it is suggested to wait 10% + 250 uSec between reads.
Initially just do single reads and see how it fluctuates.

Going bush solo next Sunday for 15 days, so battery has to behave itself.
Regards
fxm
Moderator
Posts: 12110
Joined: Apr 22, 2009 12:46
Location: Paris suburbs, FRANCE

Re: Sorting samples into the most abundant.

Post by fxm »

In my last 2 codes, I have just improved the recursive filtering procedure ("Sub filtering()") by taking into account now 2 cases for the criterion of final convergence:
- stop when there are no more samples to eliminate,
- or stop before there are no more samples at all.
fxm
Moderator
Posts: 12110
Joined: Apr 22, 2009 12:46
Location: Paris suburbs, FRANCE

Re: Sorting samples into the most abundant.

Post by fxm »

Dinosaur wrote: Apr 16, 2023 20:40 Using a number of data files that I have saved .....

It would be nice to have other data files too.
Dinosaur
Posts: 1481
Joined: Jul 24, 2005 1:13
Location: Hervey Bay (.au)

Re: Sorting samples into the most abundant.

Post by Dinosaur »

Hi All

Two new files.
I suspected a "passive Cell balancer for causing sudden discharges into lower cells.
Also changed the reading method to single shot type. That meant I had to communicate with the chip twice for every read.
Looking at the chart in LibreCalc the tops of the bar chart looks much more stable , BUT still those spikes to lower values. This one is with the Balancer connected.

Code: Select all

26685,26687,26668,26684,26684,26685,26660,26683,26684,26685,26688,26685,26686,26687,26653,26688,26687,26686,26686,26688,26687,26686,26650,26688,26689,26686,26686,26688,26687,26686,26649,26685,26688,26687,26687,26686,26687,26689,26649,26685,26687,26688,26687,26686,26687,26686,26650,26687,26686,26686,26688,26688,26685,26687,26686,26688,26688,26685,26688,26688,26688,26687,26687,26688,26687,26685,26686,26687,26688,26688,26686,26686,26689,26688,26688,26687,26687,26688,26689,26687,26686,26687,26687,26687,26685,26669,26689,26686,26684,26685,26687,26690,26688,26647,26642,26682,26683,26684,26685,26683,26685,26646,26685,26685,26686,26687,26687,26687,26688,26647,26687,26686,26686,26683,26665,26686,26688,26650,26610,26648,26616,26684,26687,26684,26684,26648,26685,26684,26684,26685,26685,26683,26683,26648,26683,26686,26687,26687,26688,26687,26687,26650,26688,26688,26685,26685,26686,26688,26688,26647,26686,26686,26687,26687,26686,26687,26688,26648,26687,26686,26687,26687,26687,26687,26686,26650,26688,26685,26688,26690,26688,26687,26686,26682,26688,26687,26685,26686,26688,26688,26688,26673,26687,26689,26690,26685,26685,26687,26688,26687,26686,26688,26688,26688,26686,26687,26680,26689,26686,26687,26688,26687,26687,26686,26652,26688,26689,26687,26686,26687,26688,26689,26648,26687,26687,26689,26688,26687,26687,26689,26648,26687,26688,26686,26688,26687,26687,26687,26651,26687,26685,26686,26686,26688,26687,26686,26649,26687,26687,26686,26687,26687,26687,26686,26667,26687,26688,26689,26684,26685,26687,26688,26689,26686,26684,26688,26687,26687,26687,26688,26687,26686,26687,26687,26687,26688,26685,26686,26687,26688,26687,26686,26686,26687,26688,26666,26684,26688,26688,26687,26686,26687,26688,26650,26683,26686,26687,26676,26669,26685,26686,26649,26608,26655,26620,26687,26687,26684,26684,26647,26685,26685,26684,26684,26685,26689,26687,26648,26688,26688,26689,26687,26685,26688,26687,26648,26687,26686,26688,26687,26686,26687,26689,26647,26687,26685,26686,26687,26687,26686,26688,26653,26687,26685,26687,26687,26688,26687,26686,26685,26687,26687,26686,26687,26686,26687,26686,26686,26687,26689,26686,26688,26687,26688,26680,26688,26686,26687,26688,26689,26687,26686,26655,26688,26688,26686,26687,26687,26687,26687,26650,26688,26688,26687,26687,26686,26687,26688,26637,26684,26687,26686,26686,26684,26666,26666,26628,26667,26666,26666,26667,26666,26666,26663,26643,26662,26667,26665,26666,26672,26688,26686,26686,26687,26687,26688,26687,26687,26686,26688,26687,26686,26688,26688,26688,26685,26687,26688,26687,26687,26685,26688,26688,26686,26686,26676,26688,26688,26686,26687,26688,26688,26687,26679,26688,26687,26688,26688,26686,26687,26685,26650,26685,26687,26688,26688,26687,26686,26686,26650,26687,26686,26687,26690,26689,26686,26686,26652,26688,26687,26686,26621,26688,26687,26686,26587,26648,26651,26682,26683,26682,26684,26685,26645,26682,26687,26686,26685,26683,26682,26687,26652,26685,26686,26687,26688,26686,26687,26687,26679,26687,26687,26687,26687,26688,26687,26686,26676,26688,26688,
And this one is with it dis-connected.

Code: Select all

26686,26685,26664,26684,26684,26684,26684,26685,26685,26686,26687,26686,26686,26684,26685,26685,26686,26686,26685,26685,26685,26623,26684,26682,26611,26646,26623,26682,26682,26682,26683,26683,26683,26684,26683,26682,26684,26683,26681,26685,26687,26685,26684,26685,26685,26685,26685,26686,26686,26685,26686,26685,26685,26686,26685,26686,26686,26686,26679,26686,26684,26685,26686,26684,26685,26685,26655,26686,26685,26685,26684,26686,26686,26686,26648,26686,26685,26687,26685,26685,26684,26686,26648,26685,26685,26685,26685,26685,26685,26686,26649,26686,26685,26686,26686,26685,26685,26685,26649,26684,26685,26685,26685,26685,26686,26686,26669,26684,26687,26686,26685,26685,26686,26685,26680,26684,26686,26686,26685,26684,26684,26686,26685,26686,26685,26686,26685,26685,26685,26686,26685,26685,26685,26685,26686,26687,26686,26663,26685,26686,26686,26685,26686,26686,26685,26647,26685,26686,26685,26686,26686,26685,26686,26650,26685,26685,26685,26685,26685,26685,26685,26648,26685,26686,26685,26686,26685,26686,26686,26647,26685,26685,26685,26685,26685,26685,26686,26656,26685,26686,26686,26685,26686,26686,26685,26678,26686,26685,26685,26686,26686,26685,26686,26684,26685,26620,26685,26686,26686,26625,26633,26656,26683,26681,26682,26684,26663,26664,26667,26666,26668,26668,26666,26664,26667,26669,26655,26667,26683,26687,26687,26687,26686,26686,26648,26685,26687,26687,26686,26688,26686,26687,26647,26684,26686,26685,26687,26687,26686,26686,26650,26687,26686,26687,26687,26687,26687,26687,26650,26686,26687,26686,26688,26688,26686,26687,26649,26685,26686,26685,26688,26687,26687,26686,26670,26686,26687,26687,26687,26687,26687,26686,26686,26687,26687,26687,26687,26688,26687,26686,26688,26686,26687,26687,26687,26688,26688,26687,26686,26686,26686,26687,26688,26688,26687,26669,26687,26687,26686,26687,26687,26687,26686,26647,26688,26687,26688,26687,26686,26687,26686,26648,26687,26686,26687,26687,26686,26686,26685,26651,26687,26688,26687,26685,26688,26687,26687,26648,26687,26688,26687,26687,26686,26687,26687,26651,26687,26687,26687,26686,26688,26686,26687,26666,26686,26686,26687,26687,26687,26687,26686,26687,26686,26685,26686,26686,26687,26686,26688,26686,26686,26688,26686,26685,26687,26686,26685,26687,26686,26664,26685,26683,26683,26604,26613,26612,26683,26683,26685,26682,26683,26683,26671,26683,26685,26684,26684,26685,26686,26687,26681,26685,26686,26686,26687,26688,26686,26687,26665,26688,26687,26686,26686,26687,26685,26685,26647,26685,26687,26687,26686,26685,26687,26686,26650,26686,26686,26687,26687,26687,26687,26686,26650,26688,26686,26686,26686,26687,26686,26688,26650,26686,26687,26686,26687,26685,26686,26687,26662,26688,26688,26687,26686,26686,26686,26687,26686,26686,26687,26686,26686,26687,26687,26687,26687,26686,26687,26685,26687,26663,26683,26687,26687,26686,26686,26686,26686,26686,26687,26679,26686,26686,26686,26686,26686,26686,26688,26652,26688,26686,26686,26687,26686,26685,26687,26650,26687,26686,26687,26686,26686,26685,26686,26647,26686,26687,26686,26687,26687,26687,
Regards
EDIT:
This is a 250 sample file that was done before changing the reading method. The puzzling spikes are almost on a predictable pattern.

Code: Select all

26386, 26381, 26388, 26389, 26371, 26389, 26389, 26388, 26388, 26377, 26386, 26388, 26388, 26389, 26389, 26372, 26386, 26392, 26391, 26389, 26390, 26371, 26389, 26389, 26390, 26387, 26393, 26370, 26390, 26389, 26389, 26389, 26390, 26370, 26388, 26394, 26389, 26391, 26372, 26388, 26389, 26390, 26389, 26393, 26372, 26389, 26391, 26388, 26388, 26389, 26371, 26387, 26389, 26392, 26390, 26389, 26370, 26389, 26390, 26389, 26388, 26390, 26373, 26389, 26389, 26389, 26389, 26371, 26389, 26349, 26388, 26392, 26389, 26325, 26354, 26345, 26386, 26387, 26384, 26372, 26387, 26388, 26389, 26388, 26389, 26371, 26390, 26386, 26391, 26388, 26389, 26372, 26388, 26389, 26390, 26388, 26369, 26382, 26384, 26387, 26389, 26388, 26370, 26389, 26388, 26385, 26389, 26387, 26371, 26387, 26388, 26389, 26388, 26386, 26370, 26391, 26388, 26388, 26388, 26388, 26370, 26388, 26387, 26391, 26387, 26370, 26388, 26387, 26388, 26388, 26388, 26368, 26389, 26390, 26388, 26387, 26389, 26370, 26386, 26388, 26387, 26391, 26389, 26370, 26388, 26389, 26388, 26387, 26389, 26366, 26391, 26390, 26387, 26387, 26370, 26387, 26389, 26387, 26388, 26391, 26370, 26388, 26388, 26388, 26387, 26388, 26369, 26386, 26393, 26388, 26388, 26387, 26369, 26389, 26387, 26386, 26389, 26390, 26370, 26389, 26387, 26387, 26389, 26370, 26387, 26387, 26388, 26389, 26387, 26369, 26388, 26388, 26388, 26386, 26390, 26372, 26370, 26364, 26388, 26387, 26333, 26319, 26370, 26390, 26387, 26385, 26384, 26368, 26384, 26385, 26385, 26384, 26373, 26389, 26387, 26389, 26387, 26387, 26371, 26388, 26385, 26390, 26388, 26387, 26370, 26388, 26387, 26389, 26386, 26387, 26375, 26387, 26388, 26387, 26388, 26388, 26371, 26388, 26392, 26388, 26387, 26371, 26387, 26387, 26388, 26388, 26387, 26372, 26391, 26388,
fxm
Moderator
Posts: 12110
Joined: Apr 22, 2009 12:46
Location: Paris suburbs, FRANCE

Re: Sorting samples into the most abundant.

Post by fxm »

I have just improved my 2 codes (viewtopic.php?p=298160#p298160) by adding an automatic graph framing (automatic offset on y axis).
Dinosaur
Posts: 1481
Joined: Jul 24, 2005 1:13
Location: Hervey Bay (.au)

Re: Sorting samples into the most abundant.

Post by Dinosaur »

Hi All

@fxm
Is there a chance you could reverse the graph so that the spikes show negative from the hi samples ?

Regards
fxm
Moderator
Posts: 12110
Joined: Apr 22, 2009 12:46
Location: Paris suburbs, FRANCE

Re: Sorting samples into the most abundant.

Post by fxm »

Dinosaur wrote: Apr 17, 2023 20:09 @fxm
Is there a chance you could reverse the graph so that the spikes show negative from the hi samples ?

Done (viewtopic.php?p=298160#p298160).
Dinosaur
Posts: 1481
Joined: Jul 24, 2005 1:13
Location: Hervey Bay (.au)

Re: Sorting samples into the most abundant.

Post by Dinosaur »

Hi All

Perfect, thank you.

Duty calls this morning , but will test stability this afternoon.
Many thanks for all the support.

Regards
Dinosaur
Posts: 1481
Joined: Jul 24, 2005 1:13
Location: Hervey Bay (.au)

Re: Sorting samples into the most abundant.

Post by Dinosaur »

Hi All

I measured the actual voltages with a high quality Fluke meter
and then calibrated the A2D so it read precisely what my meter read.
Using the Filter routine I have a stable output within about 1mV.
As you may realise it is a bit of a fluid situation, where the balancer will continuously try to even all the cells.
I am very happy with the end result.

I have an issue with the device itself but talking with TI on that.
At the moment the Rpi powers the A2D board, however the input voltages are always connected
so we have a situation where the A2D has no power but has signal feeding to it.
Going to rewire that, so that it always has power.(draws micro amps)
That may improve the negative spikes as well, never did like powering from the Rpi where you inherit all the noise as well.

I am very thankful for the time you guys have put into this, just wish I had the skill to contribute in the same way.

Regards
Post Reply