Array Descriptor (split from Wiki Improvements)

[coderJeff]

Fixed-dimension, variable-length, is handy for compile time type checking. Also, saves some space in the descriptor.

Code: Select all

'' a() must be 2 dimensional
sub proc( a(any,any) as integer )
end sub

dim a(1 to 10) as integer
proc( a() ) '' type mismatch

Yeah, I was just trying to work out if a "fixed-dim" flag is needed or not. Or if can be determined from the dimensions and dimTb() size only? If ERASE is allowed to reset the dimensions for arrray() then a "fixed-dim" flag would be convenient. It's possible could get it from FBARRAY_FLAGS_DIMENSIONS, but only if zero (0) takes on special meaning like in dimensions field, which I'd like to avoid. So the flag seems like better approach.

[Juergen Kuehlwein]

You could even make it one single (2 bit) flag, because the "states" are mutual exclusive:

0 = dynamic
1 = fixed dimensions, variable number of elements
2 = fixed size, fixed number of elements
3 = attached (only possible for former dynamic, so an attached array, which is reset, must become a dynamic array again)

This can save you some checks now, but expanding this flag (for future ideas) and adapting the code then would be be more tricky.


JK

[dodicat]

I have updated my RUNTIME tester for arrays (OK now with -O optimisations if they are used)
Gives the array pointer @a(lbound(a,1),lbound(a,2) . . . ).
And reports if the array is static or dynamic.
If in error it sleeps and reports ERROR.

Code: Select all

#macro __Arrayptr__(array)
Iif(Ubound(array,0)=1,@array(Lbound(array)), _
Iif(Ubound(array,0)=2,@array(Lbound(array,1),Lbound(array,2)), _
Iif(Ubound(array,0)=3,@array(Lbound(array,1),Lbound(array,2),Lbound(array,3)), _
Iif(Ubound(array,0)=4,@array(Lbound(array,1),Lbound(array,2),Lbound(array,3),Lbound(array,4)), _
Iif(Ubound(array,0)=5,@array(Lbound(array,1),Lbound(array,2),Lbound(array,3),Lbound(array,4),Lbound(array,5)), _
Iif(Ubound(array,0)=6,@array(Lbound(array,1),Lbound(array,2),Lbound(array,3),Lbound(array,4),Lbound(array,5),Lbound(array,6)), _
Iif(Ubound(array,0)=7,@array(Lbound(array,1),Lbound(array,2),Lbound(array,3),Lbound(array,4),Lbound(array,5),Lbound(array,6),Lbound(array,7)), _
Iif(Ubound(array,0)=8,@array(Lbound(array,1),Lbound(array,2),Lbound(array,3),Lbound(array,4),Lbound(array,5),Lbound(array,6),Lbound(array,7),Lbound(array,8)),0))))))))
#endmacro


#macro setptr(datatype)
Function arrayptr Overload(a() As datatype,Byref ret As boolean=0)  As datatype Ptr
    #macro GetSize(array,d)
    d=Ubound(array,0)
    For n As long=1 To d
        If n=1 Then d=1
        d=d*(Ubound(array,n)-Lbound(array,n)+1)
    Next
    d=d*Sizeof(array)
    #endmacro
    Static As datatype Ptr z
    Dim As Long size
    getsize(a,size)
    z= __arrayptr__(a)
    If  z >  @ret + size Then ret=true Else ret=false
    Return z
End Function
#endmacro


Type udt
    As Long n(100)
    As String s
End Type

'setup five datatypes (functions)
setptr(Double)
setptr(String)
setptr(udt)
setptr(Short)
setptr(Byte)

Dim As Long counter,outflag

#define AddRandomMemory  redim As String st(1 To 1+Rnd*5000)
Screen 12
randomize
Do
    If Inkey=Chr(27) Then Cls:outflag=1
    counter+=1
    Scope
        Dim As boolean b  'true=dynamic, false = static
        
        Locate 5
        Print "fn(ptr)","ptr","Is dynamic?"
        
        AddRandomMemory
        
        If Rnd>.5 Then
            AddRandomMemory
            Redim As Double d(4,5 To 7)
            Print arrayptr(d(),b),@d(0,5),b,"dynamic"
            If b=false Then Print "ERROR":Sleep
        Else
            AddRandomMemory
            Dim As Double d(4,5 To 7,3,4)
            Print arrayptr(d(),b),@d(0,5,0,0),b,"static"
            If b=true Then Print "ERROR":Sleep
        End If
        
        If Rnd>.5 Then
            AddRandomMemory
            Redim As String s(-1 To 5,4,1 To 3,-3 To 4)
            Print arrayptr(s(),b),@s(-1,0,1,-3),b,"dynamic"
            If b=false Then Print "ERROR":Sleep
        Else
            AddRandomMemory
            Dim As String s(-1 To 5,2 To 4,1 To 3,-3 To 4)
            Print arrayptr(s(),b),@s(-1,2,1,-3),b,"static"
            If b=true Then Print "ERROR":Sleep
        End If
        
        If Rnd>.5 Then
            AddRandomMemory
            Redim As udt z(2,3)
            Print arrayptr(z(),b),@z(0,0),b,"dynamic"
            If b=false Then Print "ERROR":Sleep
        Else
            AddRandomMemory
            Dim As udt z(2,3,4)
            Print arrayptr(z(),b),@z(0,0,0),b,"static"
            If b=true Then Print "ERROR":Sleep
        End If
        
        If Rnd>.5 Then
            AddRandomMemory
            Redim As Short f(1 To 67,2 To 8)
            Print arrayptr(f(),b),@f(1,2),b,"dynamic"
            If b=false Then Print "ERROR":Sleep
        Else
            AddRandomMemory
            Dim As Short f(1 To 67,1 To 8)
            Print arrayptr(f(),b),@f(1,1),b,"static"
            If b=true Then Print "ERROR":Sleep
        End If
        
        If Rnd>.5 Then
            AddRandomMemory
            Redim As Byte f(1 To 100,2 To 8,8,7,6,5)
            Print arrayptr(f(),b),@f(1,2,0,0,0,0),b,"dynamic"
            If b=false Then Print "ERROR":Sleep
        Else
            AddRandomMemory
            Dim As Byte f(1 To 50,2 To 8)
            Print arrayptr(f(),b),@f(1,2),b,"static"
            If b=true Then Print "ERROR":Sleep
        End If
        
        Print "press <esc> to exit"
       ' Sleep
    End Scope
    Print "number of tests  ";counter
    
Loop Until outflag
Sleep
   

[fxm]

Same remark as previously:

But the bigger problem is to distinguish between a var-len array in the heap and a fix-len array in the .BSS or .DATA section.
For example, in your Scope block, between:
Redim As Double d(4, 5 to 7) '' var-len array data in the heap
and
Static As Double d(4, 5 to 7) '' fix-len array data in the .BSS section
because we do not know by priori the value of the border between the heap and the .BSS section.

[coderJeff]

@dodicat, I am distracted by your cleverness ... again. You don't know how many times.. :)

Just for interest, must be compiled as 2 modules:

main module:

Code: Select all

'' Note: may need to check linker map to get actual names of these symbols
'' or underscore prefex '_' may vary by linker (i.e. prefix another '_')
'' though it's fairly common in gcc tool chain.
extern "c"
	extern bss_end    alias "_bss_end__" as byte
	extern bss_start  alias "_bss_start__" as byte
	extern data_end   alias "_data_end__" as byte
	extern data_start alias "_data_start__" as byte
end extern

'' prototypes intentionally different from procs in second module
declare function GetArrayDescPtr( array() as any ) as any ptr
declare function GetArrayDataPtr( array() as any ) as any ptr

'' compare an address the bss/data section label addresses
function getSection( byval p as any ptr ) as string
	if( p >= @bss_start ) and ( p < @bss_end ) then
		function = "bss"
	elseif( p >= @data_start ) and ( p < @data_end ) then
		function = "data"
	else
		function = "other"
	end if
end function

#macro show( array )
	print #array,
	print hex(getArrayDescPtr(array())),
	print getSection(getArrayDescPtr(array())),
	print hex(getArrayDataPtr(array())),
	print getSection(getArrayDataPtr(array()))
#endmacro

'' TEST
print "bss  end  : " & hex( @bss_end )                   
print "bss  start: " & hex( @bss_start )
print "data end  : " & hex( @data_end )
print "data start: " & hex( @data_start )
print

print "name", "descriptor", "section", "array", "section"

static a() as double
dim b() as double
redim c() as double

show( a )
show( b )
show( c )

static d(1 to 20) as double
dim e(1 to 20) as double
redim f(1 to 20) as double

show( d )
show( e )
show( f )

second module:

Code: Select all

'' these two functions must be in separate module
function getArrayDescPtr( byval array as any ptr ) as any ptr
	return array
end function 

function getArrayDataPtr( byval array as any ptr ptr ) as any ptr
	return array[1]
end function 

output (fbc win 32):

Code: Select all

bss  end  : 40EA54
bss  start: 409000
data end  : 407174
data start: 407000

name          descriptor    section       array         section
a             407004        data          0             other
b             18FE1C        other         0             other
c             18FDA4        other         0             other
d             40707C        data          409018        bss
e             18FCE0        other         18FCFC        other
f             18FCBC        other         582A60        other

I don't recommend as a general solution: Depends on crt and linker, so it's not portable, and if shared libraries (dll's) are involved there will be multiple bss/data sections for each executable.

EDIT:
was returning wrong data pointer, should be return array[1].

[coderJeff]

Further to the last post, with these 4 examples added:

Code: Select all

static g(1 to 20) as double = { 1, 2, 3 }
dim h(1 to 20) as double = { 1, 2, 3 }

show( g )
show( h )

type T
	a( 1 to 10 ) as double
end type

dim x as T
show( x.a )

dim px as T ptr = new T
show( px->a )
delete px

Code: Select all

name          descriptor    section       array         section
g             407148        data          4070A0        data     -- array data in data section (static)
h             18FBF8        other         18FC14        other    -- array data on stack
x.a           18FB84        other         18FBA0        other    -- array data on stack  
px->a         18FAEC        other         772B08        other    -- array data is on heap

The program doesn't know the difference between "stack" and "heap" and just reports it as "other". But from the address I can make a fair guess.

So, it really doesn't matter where the data is located, we really need to know if the array is fixed length or variable length to make good run time decisions.

[coderJeff]

The next step is to add the new "flags" field to the array descriptor and break everything.

I haven't created a new pull request as I felt it prudent to take a little detour. I was aiming for a 1.07.0 release around october 2019, but adding the new array stuff right might not be best choice.

Because fbc itself uses arrays, and we are changing the arrays, building the next version of fbc has to be done in 2 steps. What we traditionally call the 'chicken-egg' problem around here.

1) fbc (e.g. version 1.06.0) is used to build fbc 1.07.0 (A), which is linked with fbrtlib 1.06.

2) Assuming no bugs, fbc 1.07.0 (A) can produce valid executables. However to make sure fbc can compile itself, we must recompile fbc using fbc 1.07.0 (A) and linking to the new fbrtlib 1.07 to produce fbc 1.07.0 (B).

3) Optionally, we then use fbc 1.07.0 (B) to recompile itself again to produce fbc 1.07.0 (C) and check for any deviations. There must be none, otherwise we fix the problem and go back to step 1

Basically what I am considering:
- create a 1.07.0 branch for a release very soon. Maybe even in a week.
- allow the continuation of the new array stuff on master branch; I don't want to hold up development while I create a release for next 2+ weeks.
- I have been testing the possibility of a 1.07.0 release
- I have also been testing the validity of the next array descriptor change (adding the extra flags field) by writing tests and building multiple times.

[Juergen Kuehlwein]

Do whatever is necessary to push things forward.

One thought about presenting the array descriptor:
- If you choose to make the array descriptor available by reference (return a pointer to the array descriptor), then this includes the possibility to modify the descriptor members by the user. I think, this is not what you want. Because without knowing exactly what you do, this (modifying the descriptor members) is bound to end in a crash. Actually these values shouldn´t be modified outside the RTL.

- If you choose to return only the values of the descriptor members, a user can read each value, but cannot modify it (without a hack). This serves the purpose and leaves all options for futures modifications without the need to break existing code again.


JK

[fxm]

This is only a particular case of all the errors that can be made when we have the capacity of using pointers.

[speedfixer]

I agree. Pointer use is the programmer's problem. The language should give the freedom, not tie his hands.

I think access should be provided, but not casually.

These new array features are going to allow for some powerful extensions to FB.
After doing all this work just for exposure, getting full access shouldn't require another deep dive into the code source. The new access info can be put into a hacker's help page somewhere, with a few primitives exposed in the new added 'array.bi' to be provided.

Also, I think providing better TYPEOF functionality beyond what has been discussed would help. Another topic post.

[Lost Zergling]

@dodicat. Most code in your runtime tester might be very close to what I was looking for to improve lzae (overload redundancy, udts handling). It is simple & elegant.
#macro setptr(datatype)
Function arrayptr Overload(a() As datatype,...) As datatype Ptr
...
Static As datatype Ptr z
z= __arrayptr__(a)
Return z
End Function
..
with your graceful agreement and if it turns out to be relevant I may wish to reuse it to improve my code.

[MrSwiss]

Tried the new FBC 1.07.0 release vs. yesterdays dev version ... (the new array stuff).
Both seem to fail currently, displaying a identical error 8: undefined symbol ...

Testcode and results are:

Code: Select all

' NewArrayFeat-test1.bas -- (c) 2019-08-26, MrSwiss
'
' updated comments/results: 2019-08-27
'

/'
    fbc 1.07.0 -- WIN, 64 bit, date: 2019-08-26 -- inofficial dev
    together with, additional headers: 2019-08-22
'/

#Include "fbc-int/array.bi"

ReDim As UByte  uba()

Dim As FBC.FBARRAY Ptr  puba        ' up to this point: all okay
' get the pointer ...
puba = FBC.fb_ArrayGetDesc(uba())   ' <-- undefined symbol (error 8)
/'
    FBC.fb_ArrayGetDesc()   ??? not yet implemented in CRT ???
    retested with fbc 1.07.0 release version (2019-08-27), no change!
'/
Print "@uba() type-def: "; puba

Sleep

/'

results FBC 1.07.0:

26th, dev version:
C:\DEV_TOOLS\FreeBASIC\DEV_64\fbc -gen GCC -O 2 "NewArrayFeat-test1.bas"
NewArrayFeat-test1.bas(12) error 8: Undefined symbol, fb_ArrayGetDesc in 'puba = FBC.fb_ArrayGetDesc(uba())   ' <-- undefined symbol (error 8)'

Build error(s)

27th, release version:
C:\DEV_TOOLS\FreeBASIC\1070_64\fbc -gen GCC -O 2 "NewArrayFeat-test1.bas"
NewArrayFeat-test1.bas(13) error 8: Undefined symbol, fb_ArrayGetDesc in 'puba = FBC.fb_ArrayGetDesc(uba())   ' <-- undefined symbol (error 8)'

Build error(s)

'/

@coderJeff,
thanks for the new release, FBC 1.07.0.

[fxm]

puba = fb_ArrayGetDesc(uba())

[MrSwiss]

OK, it's declared outside of namespace "FBC", thanks, should have seen that myself ...

[dodicat]

@dodicat. Most code in your runtime tester might be very close to what I was looking for to improve lzae (overload redundancy, udts handling). It is simple & elegant.
#macro setptr(datatype)
Function arrayptr Overload(a() As datatype,...) As datatype Ptr
...
Static As datatype Ptr z
z= __arrayptr__(a)
Return z
End Function
..
with your graceful agreement and if it turns out to be relevant I may wish to reuse it to improve my code.

Sure thing Lost Zerling, good luck with your project.