IncFile() and IncArray() macros [Updated 22-1-2009]

Source-code only - please, don't post questions here.
voodooattack
Posts: 605
Joined: Feb 18, 2006 13:30
Location: Alexandria / Egypt
Contact:

IncFile() and IncArray() macros [Updated 22-1-2009]

Postby voodooattack » Jun 07, 2007 14:59

IncFile() and IncArray() are macros that allow you, as the names suggest, to embed files into your executable at compile-time.
The embedded files can then be accessed just like regular memory pointers (or as arrays, with IncArray).

incfile.bi

Code: Select all

#IFNDEF INCFILE_BI
#DEFINE INCFILE_BI

#MACRO IncFileEx(label, file, sectionName, attr)
#if __FUNCTION__ <> "__FB_MAINPROC__"
   
    Dim label As Const Ubyte Ptr = Any
    Dim label##_len As Uinteger = Any
   
    #if __FB_DEBUG__
        asm jmp .LT_END_OF_FILE_##label##_DEBUG_JMP
    #else
        ' Switch to/Create the specified section
        #if attr = ""
            asm .section sectionName
        #else
            asm .section sectionName, attr
        #endif
    #endif
   
    ' Assign a label to the beginning of the file
    asm .LT_START_OF_FILE_##label#:
    asm __##label##__start = .
    ' Include the file
    asm .incbin ##file
    ' Mark the end of the the file
    asm __##label##__len = . - __##label##__start
    asm .LT_END_OF_FILE_##label:
    ' Pad it with a NULL Integer (harmless, yet useful for text files)
    asm .LONG 0
#if __FB_DEBUG__
    asm .LT_END_OF_FILE_##label##_DEBUG_JMP:
#else
    ' Switch back to the .text (code) section               
    asm .section .text
    asm .balign 16
#endif
    asm .LT_SKIP_FILE_##label:
    asm mov dword ptr [label], offset .LT_START_OF_FILE_##label#
    asm mov dword ptr [label##_len], offset __##label##__len
#else

    Extern "c"
    Extern label As Ubyte Ptr
    Extern label##_len As Uinteger
    End Extern
   
    #if __FB_DEBUG__
        asm jmp .LT_END_OF_FILE_##label##_DEBUG_JMP
    #else
        ' Switch to/create the specified section
        #if attr = ""
            asm .section sectionName
        #else
            asm .section sectionName, attr
        #endif
    #endif
   
    ' Assign a label to the beginning of the file
    asm .LT_START_OF_FILE_##label#:
    asm __##label##__start = .
    ' Include the file
    asm .incbin ##file
    ' Mark the end of the the file
    asm __##label##__len = . - __##label##__start
    asm .LT_END_OF_FILE_##label:
    ' Pad it with a NULL Integer (harmless, yet useful for text files)
    asm .LONG 0
    asm label:
    asm .int .LT_START_OF_FILE_##label#
    asm label##_len:
    asm .int __##label##__len
#if __FB_DEBUG__
    asm .LT_END_OF_FILE_##label##_DEBUG_JMP:
#else
    ' Switch back to the .text (code) section               
    asm .section .text
    asm .balign 16
#endif
    asm .LT_SKIP_FILE_##label:
#endif       
#endmacro

#macro IncFile(label, file)
    IncFileEx(label, file, .data, "")        'Use the .data (storage) section (SHARED)
#endmacro

#endif


Usage:

IncFile(label, filename)
creates a variable named "label" that points to the file contents, stores the file in the .data section, the length of the contents is stored in another variable called "label_len".

IncFileEx(label, filename, section_name, attr)
same as the above, but lets you specify a name for the section to store data in and the attributes of the section (can be 0 for default).

I hope some one finds it useful :)

Change log:
* Fixed some glitches and replaced LEA instruction with MOV (OFFSET).
* Now works inside functions once more.
* removed all x86 run-time initialization code (all except the jump for debug mode), replaced by compile-time assembler directives. [ If used at module leve ]
* removed the SHARE parameter altogether..
* IncFile Automatically decides whether to declare the variable as SHARED or not, based on the context. (__FB_MAINPROC__ trick)
* Support for debug builds.
* Now skips updating the variables if not needed.
* IncFileEx now allows you to specify section attributes.
* Added the 'share' parameter.
* fixed a small glitch with the "attr" parameter.



[22-1-2009] IncArray() macro

Code: Select all

#IFNDEF __INCARRAY_BI__
#DEFINE __INCARRAY_BI__

#macro IncArray(name, filename)
#if __FUNCTION__ <> "__FB_MAINPROC__"
    #error "IncArray() cannot be used within a function."
    ' just to avoid extra pointless errors and keep things cleaner
    dim label() as ubyte
#else   
    extern "c"
    extern name() alias #name as ubyte
    end extern
   
    asm
    #IF __FB_DEBUG__
        jmp .LT_END_OF_FILE_##name##_DEBUG_JMP
    #ELSE       
        .section .data
    #ENDIF
        .balign 1
        __##name##__start = .
        .incbin filename
        __##name##__len = . - __##name##__start
        .long 0                 ' add one null int as padding
        .globl ##name
        .balign 4
        ##name:
        .int __##name##__start  'data
        .int __##name##__start  'ptr
        .int __##name##__len    'size
        .int 1                  'element_len
        .int 1                  'dimensions
       
        ' dimTB
        .int  __##name##__len   'elements
        .int 0                  'lbound
        .int  __##name##__len-1 'ubound
        .LT_END_OF_FILE_##name##_DEBUG_JMP:
        .section .text
        .balign 16
    End asm
   
    #undef Name##size
#endif
#endmacro

#ENDIF        '__INCARRAY_BI__


Notes:
  • the macro builds an array descriptor using assembler directives, then imports the generated array using the "extern" statement.
  • due to a bug in the FB preprocessor, the array's name must begin with an underscore (_test) [ FIXED, Not needed anymore ]
  • this will not work with debug builds. (maybe if you build an object file separately) [ Read below ]
  • you can use IncArrayEx() to specify the data-type of your array elements, the file will be divided into logical UDTs in memory just like normal arrays (variable-length strings are not supported)
  • you cannot redim the generated array
  • 0 instructions used, no executable code added to your application.
  • [UPDATE] This should work in debug builds now, once this bug gets fixed.
  • fixed some glitches with data-type alignment, now works as expected.
  • [22-1-2009] support for datatypes other than byte has been removed, assembler started spewing errors recently, probably due to a recent change in the emitter.

and here's a small example:

Code: Select all

#include "incarray.bi"
IncArray(myFile, "test.bas")

print "File size: " & ubound(myFile) - lbound(myFile) + 1
print "Listing file contents:"

for i as integer = lbound(myFile) to ubound(myFile)
   print chr(myFile(i));
next
Last edited by voodooattack on Jan 22, 2009 17:05, edited 9 times in total.
redcrab
Posts: 618
Joined: Feb 07, 2006 15:29
Location: France / Luxemburg
Contact:

Postby redcrab » Jun 07, 2007 15:13

Whaoo impressive ... Does it works for all platform (Linux, Win, DOs) ?
Not yet tested...
I thinks this is cool to include static resource without windows resource file stuff... So if this way is cross platform , sure i will adopt your way
there is another way to inclue binary resource in file (staticaly initialized array) but you've to use a program to generate the code ... your way is more direct... each time you change the resource file you do not have to generate a special code , you may use the resource directly without transformation.

That's cool
cha0s
Site Admin
Posts: 5317
Joined: May 27, 2005 6:42
Location: Illinois
Contact:

Postby cha0s » Jun 07, 2007 15:54

Yeah, this is great for the people who want everything in their exe but don't want to phuck with Windows resources and stuff. Really great (and greater if like redcrab asked, if it's cross-platofmr ;)
voodooattack
Posts: 605
Joined: Feb 18, 2006 13:30
Location: Alexandria / Egypt
Contact:

Postby voodooattack » Jun 07, 2007 16:44

Thanks redcrab and cha0s, as far as I know, it should be cross-platform, since GAS, the GNU assembler does most of the work. :)

EDIT:

Here's another useful macro, based on the original:

Code: Select all

#macro IncPNG(label, file)
    dim shared as IMAGE ptr label
    scope
        incfile (p##label, file)
        label = PNG_Load_Mem(p##label, p##label##_len)
    end scope 
#endmacro



...for use with PNG_Load by yetifoot, that's basically what i was trying to do =)
to make it work with the macros above, just remove 'shared' from IncFileEx.. ;)
marinedalek
Posts: 124
Joined: Aug 24, 2005 1:55
Contact:

Postby marinedalek » Jun 07, 2007 22:15

Looks cool if it's what I think it is (I'm still a bit of a stick in the mud when it comes to FB-specific stuff!) but how is it used? Does it store a file in a resource at compile time and let it be used at run time?
ShelledGhost
Posts: 10
Joined: Jan 20, 2007 0:43

Postby ShelledGhost » Jun 07, 2007 23:49

Can I BLOAD BMPs stored in this way?
D.J.Peters
Posts: 7191
Joined: May 28, 2005 3:28

Postby D.J.Peters » Jun 08, 2007 0:01

marinedalek wrote:Looks cool if it's what I think it is (I'm still a bit of a stick in the mud when it comes to FB-specific stuff!) but how is it used? Does it store a file in a resource at compile time and let it be used at run time?
yes with the assembler command .incbin you can include any file in your compiled binary exe or library on windows and linux too.
i use it longer without any trouble.
incany.bi

Code: Select all

#ifndef __incany_bi__
#define __incany_bi__

#macro incany(strFile,lpAny)
  asm
  .section .text
  jmp .end_##lpAny
  .section .data
  .align 16
  .start_##lpAny:
  .incbin ##strFile
  .section .text
  .align 16
  .end_##lpAny:
  lea eax, .start_##lpAny
  mov dword ptr [lpAny], eax
  end asm
#endmacro
#endif ' __incany_bi__

playwave.bas

Code: Select all

#include "incany.bi"
declare function PlayMemory alias "sndPlaySoundA" (byval as any ptr, byval flag as uinteger=4) as integer
#inclib "winmm"

dim lpWave as any ptr
incany("c:/windows/media/tada.wav",lpWave)
PlayMemory(lpWave)
sleep

now it plays the wavefile inside the exe (loaded in memory)
NOTE:
if you include any media in your exe (fonts, sounds, gfx)
it should be free of any copyrights so here the file is only an example :-)

Joshy
v1ctor
Site Admin
Posts: 3795
Joined: May 27, 2005 8:08
Location: SP / Bra[s]il
Contact:

Postby v1ctor » Jun 08, 2007 0:37

Sup Voodoo :), nice trick. Maybe we could add that to fb, but i can't think of any syntax that wouldn't look ugly - and oh yeah, it shouldn't trash the global namespace.

Maybe

Code: Select all

some_label:
#include binary "foo.bar"
restore some_label
read a, b, c


But then FB should allow taking the address of labels to pass them to functions etc.. you wouldn't also know the size of the data unless "var size = (@label_after - @label_before) + 1" were used.
D.J.Peters
Posts: 7191
Joined: May 28, 2005 3:28

Postby D.J.Peters » Jun 08, 2007 0:52

ShelledGhost wrote:Can I BLOAD BMPs stored in this way?

not directly but you can create an raw file and then include it in your exe.
saveraw.bas

Code: Select all

const bmpfile = "C:\WINDOWS\system32\setup.bmp"
screenres 800,600,32
dim as ubyte ptr lpBMP=ImageCreate(800,600)
bload bmpfile,lpBMP
dim as integer hFile=FreeFile()
if open("test.raw" for binary access write as #hFile)<>0 then
  print "error: can't open raw file !"
  beep:sleep:end
end if
put #hFile,,*lpBMP,32+800*600*4
close #hFile
ImageDestroy lpBMP
sleep
end

incbmp.bas

Code: Select all

#include "incany.bi"
dim lpBMP as any ptr
incany("test.raw",lpBMP)
screenres 800,600,32
put (0,0),lpBMP,pset
sleep
end

we saved the bmp to raw with the imageheader included.
that is we don't need ImageCreate() again.

Joshy
D.J.Peters
Posts: 7191
Joined: May 28, 2005 3:28

Postby D.J.Peters » Jun 08, 2007 2:02

ShelledGhost@
if you don't know how large your bmp's will be
you can read it from bmp file first
then create an equel imagebuffer
load it in target bits gfx mode
and save it as raw file.

Joshy
bmp2raw.bas

Code: Select all

#include "fbgfx.bi"
using FB
const BMPFILE = "C:\WINDOWS\system32\setup.bmp"
const RAWFILE = "test.raw"
const BITS    = 32 ' any target bits 8,16,32

dim as integer   hFile,w,h,c
dim as short     bpp
dim as byte ptr  lpBMP
dim as image ptr lpImg

' try to open any bmp file
hFile=FreeFile()
if open(BMPFILE for binary access read as #hFile)<>0 then
  print "error: can't open [" & BMPFILE & "] file !"
  beep:sleep:end
end if

' get dimension w x h bpp and compression from bmp file
get #hFile,19,w
get #hFile,23,h
get #hFile,29,bpp
get #hFile,31,c
close #hFile
if (c<>0) then
  print "error: libfbgfx has no support for compressed bitmaps!"
  beep:sleep:end 1
end if

' now we have w,h
screenres w,h,BITS

lpBMP=ImageCreate(w,h)
' cast byte ptr to image ptr
lpIMG=cptr(image ptr,lpBMP)
bload BMPFILE,lpBMP

' save bmp as raw file
hFile=FreeFile()
if open(RAWFILE for binary access write as #hFile)<>0 then
  print "error: can't open/create [" & RAWFILE & "] !"
  beep:sleep:end
end if
' save header (32 bytes) + pixels (h*pitch bytes)
put #hFile,,*lpBMP,32+lpImg->pitch*h
close #hFile
ImageDestroy lpBMP
end

incbmp.bas

Code: Select all

#include "incany.bi"
const BITS=32 ' bits per pixel
dim lpBMP as any ptr
incany("test.raw",lpBMP)

screenres 800,600,BITS
put (0,0),lpBMP,pset
sleep
end
1000101
Posts: 2556
Joined: Jun 13, 2005 23:14
Location: SK, Canada

Postby 1000101 » Jun 08, 2007 2:26

Hurray for 100M executables!

Data is stored in separate files for a reason.
voodooattack
Posts: 605
Joined: Feb 18, 2006 13:30
Location: Alexandria / Egypt
Contact:

Postby voodooattack » Jun 08, 2007 10:32

marinedalek wrote:Looks cool if it's what I think it is (I'm still a bit of a stick in the mud when it comes to FB-specific stuff!) but how is it used? Does it store a file in a resource at compile time and let it be used at run time?


Sorry i forgot to provide an example, here:

incfiletest.bas

Code: Select all

   #include "incfile.bi"
   
   
   incfile (source, "incfiletest.bas")
   
   print "Listing the source code of this program: "
   print
   print *cast(zstring ptr, source)
   
   sleep


in the example above:
'source' becomes a ubyte pointer pointing to the first byte of your file.
a uinteger variable called 'source_len' is created and it holds the total length of your file in memory.

ShelledGhost wrote:Can I BLOAD BMPs stored in this way?


I think D.J.Peters pretty much answered any questions on the subject, although he used his own macro, this can be done in the same way with IncFile().. :)

v1ctor wrote:Sup Voodoo :), nice trick. Maybe we could add that to fb, but i can't think of any syntax that wouldn't look ugly - and oh yeah, it shouldn't trash the global namespace.

Maybe
... code..

But then FB should allow taking the address of labels to pass them to functions etc.. you wouldn't also know the size of the data unless "var size = (@label_after - @label_before) + 1" were used.


hey v1ctor, that'd be a great feature :)

I'd suggest something like:

Code: Select all

dim myarray() as ubyte => { #binary("somefile") }

..or something along the lines =P

I've already tried messing around with RTLib's FBARRAY structures on my own, although the calls could work, i couldn't get the address of the actual array descriptor from within FB code, So I couldn't use that method through my macro (define a new array and point it to that memory block) :(

1000101 wrote:Hurray for 100M executables!

Data is stored in separate files for a reason.


heh, I don't see any reason why not.. self-extractors exceed that limit.. I've seen 750 MB exe files... *looks at M$ service packs* ;)
oh, and btw, self-extractors mostly just APPEND the data to the end of the executable, so it's not even worthy, and it can be tampered with much easier...

@D.J.Peters: although your code functions mostly the same, it doesn't provide the length of the file.. which is sometimes needed, and there's an extra (not needed jump)

Code: Select all

.section .text
jmp .end_t '''''' <<< NOT NEEDED
.section .data ''''' < this switch is out of execution context
.align 16
.start_t:
.incbin "incanytest.bas"
.section .text
.align 16
.end_t:  '''' << this label is in the code section, not at the end of the included file... so if you try to calculate the length of the file this way, you'll get wrong results ;)
lea eax, .start_t
mov dword Ptr [ebp-8], eax


good work though =)
1000101
Posts: 2556
Joined: Jun 13, 2005 23:14
Location: SK, Canada

Postby 1000101 » Jun 08, 2007 12:42

voodooattack wrote:
1000101 wrote:Hurray for 100M executables!

Data is stored in separate files for a reason.


heh, I don't see any reason why not.. self-extractors exceed that limit.. I've seen 750 MB exe files... *looks at M$ service packs* ;)
oh, and btw, self-extractors mostly just APPEND the data to the end of the executable, so it's not even worthy, and it can be tampered with much easier...


Ahh, but installers never have bug updates for them. As soon as you update your executable for some bug release, you have to release a huge update.
voodooattack
Posts: 605
Joined: Feb 18, 2006 13:30
Location: Alexandria / Egypt
Contact:

Postby voodooattack » Jun 08, 2007 13:50

1000101 wrote:Ahh, but installers never have bug updates for them. As soon as you update your executable for some bug release, you have to release a huge update.


Or just release your updates through a selective patcher, might write one of those soon.. ;)
Anyways, I'm not encouraging packed applications, but this could be useful for situations where you need the data to be packed along, possibly 64k demos, self-extractors, installers, software patches, satellite DLLs, and so on..
Tigra
Posts: 155
Joined: Jan 07, 2007 17:21

Postby Tigra » Jun 11, 2007 12:22

I've started to use voodooattack's macro and I have found an incompatibility with the -g compiler switch.

I use the SciTE editor and I've configured it to compile and run .bas modules when the F5 key is hit. To aid in my debugging I have these default switches:
-g -exx -maxerr inf -s console -w pedantic

When I try to compile the test module, I get these errors:

incfiletest.asm:85: Error: can't resolve `.Data' {.Data section} - `_main' {.text section}
incfiletest.asm:90: Error: can't resolve `.Data' {.Data section} - `_main' {.text section}
incfiletest.asm:95: Error: can't resolve `.Data' {.Data section} - `_main' {.text section}
incfiletest.asm:100: Error: can't resolve `.Data' {.Data section} - `_main' {.text section}
incfiletest.asm:105: Error: can't resolve `.Data' {.Data section} - `_main' {.text section}

Here is the section of the .asm file that the macro expands to (I've marked those five lines with their line number):

##
## #include "includefile.bi"
##
## IncludeFile (source, "incfiletest.bas") [Macro Expansion: IncludeFileEx(source, "incfiletest.bas", Data)Dim Shared source As Ubyte Ptr
## Dim Shared source_len As Uinteger
##asm ]
.section .Data
## asm .section .Data
.stabn 68,0,3,.Lt_000C-_main
.Lt_000D:
##
.LT_START_OF_FILE_source:
## .LT_START_OF_FILE_source#:
.stabn 68,0,3,.Lt_000D-_main <-- 85
.Lt_000E:
##
.incbin "incfiletest.bas"
## .incbin "incfiletest.bas"
.stabn 68,0,3,.Lt_000E-_main <-- 90
.Lt_000F:
##
.LT_END_OF_FILE_source:
## .LT_END_OF_FILE_source:
.stabn 68,0,3,.Lt_000F-_main <-- 95
.Lt_0010:
##
.LONG 0
## .LONG 0
.stabn 68,0,3,.Lt_0010-_main <-- 100
.Lt_0011:
##
.section .text
## .section .text
.stabn 68,0,3,.Lt_0011-_main <-- 105
.Lt_0012:
.balign 16
## .balign 16
.stabn 68,0,3,.Lt_0012-_main
.Lt_0013:
##
lea eax, .LT_START_OF_FILE_source
## lea eax, .LT_START_OF_FILE_source#
.stabn 68,0,3,.Lt_0013-_main
.Lt_0014:
##
mov dword Ptr [_SOURCE], eax
## mov dword Ptr [source], eax
.stabn 68,0,3,.Lt_0014-_main
.Lt_0015:
##
lea ebx, .LT_END_OF_FILE_source
## lea ebx, .LT_END_OF_FILE_source
.stabn 68,0,3,.Lt_0015-_main
.Lt_0016:
##
Sub ebx, eax
## Sub ebx, eax
.stabn 68,0,3,.Lt_0016-_main
.Lt_0017:
##
mov dword Ptr [_SOURCE_LEN], ebx
## mov dword Ptr [source_len], ebx
.stabn 68,0,3,.Lt_0017-_main
.Lt_0018:
## End

When I removed the -g compiler option there are no errors.

Return to “Tips and Tricks”

Who is online

Users browsing this forum: No registered users and 0 guests