IncFile() and IncArray() macros [Updated 22-1-2009]

[voodooattack]

IncFile() and IncArray() are macros that allow you, as the names suggest, to embed files into your executable at compile-time.
The embedded files can then be accessed just like regular memory pointers (or as arrays, with IncArray).

incfile.bi

Code: Select all

#IFNDEF INCFILE_BI
#DEFINE INCFILE_BI

#MACRO IncFileEx(label, file, sectionName, attr)
#if __FUNCTION__ <> "__FB_MAINPROC__"
   
    Dim label As Const Ubyte Ptr = Any
    Dim label##_len As Uinteger = Any
   
    #if __FB_DEBUG__
        asm jmp .LT_END_OF_FILE_##label##_DEBUG_JMP
    #else
        ' Switch to/Create the specified section
        #if attr = ""
            asm .section sectionName
        #else
            asm .section sectionName, attr
        #endif
    #endif
   
    ' Assign a label to the beginning of the file
    asm .LT_START_OF_FILE_##label#:
    asm __##label##__start = .
    ' Include the file
    asm .incbin ##file
    ' Mark the end of the the file
    asm __##label##__len = . - __##label##__start
    asm .LT_END_OF_FILE_##label:
    ' Pad it with a NULL Integer (harmless, yet useful for text files)
    asm .LONG 0
#if __FB_DEBUG__
    asm .LT_END_OF_FILE_##label##_DEBUG_JMP:
#else
    ' Switch back to the .text (code) section               
    asm .section .text
    asm .balign 16
#endif
    asm .LT_SKIP_FILE_##label:
    asm mov dword ptr [label], offset .LT_START_OF_FILE_##label#
    asm mov dword ptr [label##_len], offset __##label##__len
#else

    Extern "c"
    Extern label As Ubyte Ptr
    Extern label##_len As Uinteger
    End Extern
   
    #if __FB_DEBUG__
        asm jmp .LT_END_OF_FILE_##label##_DEBUG_JMP
    #else
        ' Switch to/create the specified section
        #if attr = ""
            asm .section sectionName
        #else
            asm .section sectionName, attr
        #endif
    #endif
   
    ' Assign a label to the beginning of the file
    asm .LT_START_OF_FILE_##label#:
    asm __##label##__start = .
    ' Include the file
    asm .incbin ##file
    ' Mark the end of the the file
    asm __##label##__len = . - __##label##__start
    asm .LT_END_OF_FILE_##label:
    ' Pad it with a NULL Integer (harmless, yet useful for text files)
    asm .LONG 0
    asm label:
    asm .int .LT_START_OF_FILE_##label#
    asm label##_len:
    asm .int __##label##__len
#if __FB_DEBUG__
    asm .LT_END_OF_FILE_##label##_DEBUG_JMP:
#else
    ' Switch back to the .text (code) section               
    asm .section .text
    asm .balign 16
#endif
    asm .LT_SKIP_FILE_##label:
#endif       
#endmacro

#macro IncFile(label, file)
    IncFileEx(label, file, .data, "")        'Use the .data (storage) section (SHARED)
#endmacro

#endif

Usage:

IncFile(label, filename)
creates a variable named "label" that points to the file contents, stores the file in the .data section, the length of the contents is stored in another variable called "label_len".

IncFileEx(label, filename, section_name, attr)
same as the above, but lets you specify a name for the section to store data in and the attributes of the section (can be 0 for default).

I hope some one finds it useful :)

Change log:
* Fixed some glitches and replaced LEA instruction with MOV (OFFSET).
* Now works inside functions once more.
* removed all x86 run-time initialization code (all except the jump for debug mode), replaced by compile-time assembler directives. [ If used at module leve ]
* removed the SHARE parameter altogether..
* IncFile Automatically decides whether to declare the variable as SHARED or not, based on the context. (__FB_MAINPROC__ trick)
* Support for debug builds.
* Now skips updating the variables if not needed.
* IncFileEx now allows you to specify section attributes.
* Added the 'share' parameter.
* fixed a small glitch with the "attr" parameter.


[22-1-2009] IncArray() macro

Code: Select all

#IFNDEF __INCARRAY_BI__
#DEFINE __INCARRAY_BI__

#macro IncArray(name, filename)
#if __FUNCTION__ <> "__FB_MAINPROC__"
    #error "IncArray() cannot be used within a function."
    ' just to avoid extra pointless errors and keep things cleaner
    dim label() as ubyte
#else   
    extern "c"
    extern name() alias #name as ubyte
    end extern
   
    asm
    #IF __FB_DEBUG__
        jmp .LT_END_OF_FILE_##name##_DEBUG_JMP
    #ELSE       
        .section .data
    #ENDIF
        .balign 1
        __##name##__start = .
        .incbin filename
        __##name##__len = . - __##name##__start
        .long 0                 ' add one null int as padding
        .globl ##name
        .balign 4
        ##name:
        .int __##name##__start  'data
        .int __##name##__start  'ptr
        .int __##name##__len    'size
        .int 1                  'element_len
        .int 1                  'dimensions
       
        ' dimTB
        .int  __##name##__len   'elements
        .int 0                  'lbound
        .int  __##name##__len-1 'ubound
        .LT_END_OF_FILE_##name##_DEBUG_JMP:
        .section .text
        .balign 16
    End asm
   
    #undef Name##size
#endif
#endmacro

#ENDIF        '__INCARRAY_BI__

Notes:

• the macro builds an array descriptor using assembler directives, then imports the generated array using the "extern" statement.
• due to a bug in the FB preprocessor, the array's name must begin with an underscore (_test) [ FIXED, Not needed anymore ]
• this will not work with debug builds. (maybe if you build an object file separately) [ Read below ]
• you can use IncArrayEx() to specify the data-type of your array elements, the file will be divided into logical UDTs in memory just like normal arrays (variable-length strings are not supported)
• you cannot redim the generated array
• 0 instructions used, no executable code added to your application.
• [UPDATE] This should work in debug builds now, once this bug gets fixed.
• fixed some glitches with data-type alignment, now works as expected.
• [22-1-2009] support for datatypes other than byte has been removed, assembler started spewing errors recently, probably due to a recent change in the emitter.

and here's a small example:

Code: Select all

#include "incarray.bi"
IncArray(myFile, "test.bas")

print "File size: " & ubound(myFile) - lbound(myFile) + 1
print "Listing file contents:"

for i as integer = lbound(myFile) to ubound(myFile)
	print chr(myFile(i));
next

[redcrab]

Whaoo impressive ... Does it works for all platform (Linux, Win, DOs) ?
Not yet tested...
I thinks this is cool to include static resource without windows resource file stuff... So if this way is cross platform , sure i will adopt your way
there is another way to inclue binary resource in file (staticaly initialized array) but you've to use a program to generate the code ... your way is more direct... each time you change the resource file you do not have to generate a special code , you may use the resource directly without transformation.

That's cool

[cha0s]

Yeah, this is great for the people who want everything in their exe but don't want to phuck with Windows resources and stuff. Really great (and greater if like redcrab asked, if it's cross-platofmr ;)

[voodooattack]

Thanks redcrab and cha0s, as far as I know, it should be cross-platform, since GAS, the GNU assembler does most of the work. :)

EDIT:

Here's another useful macro, based on the original:

Code: Select all

#macro IncPNG(label, file)
    dim shared as IMAGE ptr label
    scope
        incfile (p##label, file)
        label = PNG_Load_Mem(p##label, p##label##_len) 
    end scope  
#endmacro

...for use with PNG_Load by yetifoot, that's basically what i was trying to do =)
to make it work with the macros above, just remove 'shared' from IncFileEx.. ;)

[marinedalek]

Looks cool if it's what I think it is (I'm still a bit of a stick in the mud when it comes to FB-specific stuff!) but how is it used? Does it store a file in a resource at compile time and let it be used at run time?

[ShelledGhost]

Can I BLOAD BMPs stored in this way?

[D.J.Peters]

Looks cool if it's what I think it is (I'm still a bit of a stick in the mud when it comes to FB-specific stuff!) but how is it used? Does it store a file in a resource at compile time and let it be used at run time?

yes with the assembler command .incbin you can include any file in your compiled binary exe or library on windows and linux too.
i use it longer without any trouble.
incany.bi

Code: Select all

#ifndef __incany_bi__
#define __incany_bi__

#macro incany(strFile,lpAny)
  asm
  .section .text
  jmp .end_##lpAny
  .section .data
  .align 16
  .start_##lpAny:
  .incbin ##strFile
  .section .text
  .align 16
  .end_##lpAny:
  lea eax, .start_##lpAny
  mov dword ptr [lpAny], eax
  end asm
#endmacro
#endif ' __incany_bi__

playwave.bas

Code: Select all

#include "incany.bi"
declare function PlayMemory alias "sndPlaySoundA" (byval as any ptr, byval flag as uinteger=4) as integer
#inclib "winmm"

dim lpWave as any ptr
incany("c:/windows/media/tada.wav",lpWave)
PlayMemory(lpWave)
sleep

now it plays the wavefile inside the exe (loaded in memory)
NOTE:
if you include any media in your exe (fonts, sounds, gfx)
it should be free of any copyrights so here the file is only an example :-)

Joshy

[v1ctor]

Sup Voodoo :), nice trick. Maybe we could add that to fb, but i can't think of any syntax that wouldn't look ugly - and oh yeah, it shouldn't trash the global namespace.

Maybe

Code: Select all

some_label:
#include binary "foo.bar"
restore some_label
read a, b, c

But then FB should allow taking the address of labels to pass them to functions etc.. you wouldn't also know the size of the data unless "var size = (@label_after - @label_before) + 1" were used.

[D.J.Peters]

Can I BLOAD BMPs stored in this way?

not directly but you can create an raw file and then include it in your exe.
saveraw.bas

Code: Select all

const bmpfile = "C:\WINDOWS\system32\setup.bmp"
screenres 800,600,32
dim as ubyte ptr lpBMP=ImageCreate(800,600)
bload bmpfile,lpBMP
dim as integer hFile=FreeFile()
if open("test.raw" for binary access write as #hFile)<>0 then
  print "error: can't open raw file !"
  beep:sleep:end
end if
put #hFile,,*lpBMP,32+800*600*4
close #hFile
ImageDestroy lpBMP
sleep
end

incbmp.bas

Code: Select all

#include "incany.bi"
dim lpBMP as any ptr
incany("test.raw",lpBMP)
screenres 800,600,32
put (0,0),lpBMP,pset
sleep
end

we saved the bmp to raw with the imageheader included.
that is we don't need ImageCreate() again.

Joshy

[D.J.Peters]

ShelledGhost@
if you don't know how large your bmp's will be
you can read it from bmp file first
then create an equel imagebuffer
load it in target bits gfx mode
and save it as raw file.

Joshy
bmp2raw.bas

Code: Select all

#include "fbgfx.bi"
using FB
const BMPFILE = "C:\WINDOWS\system32\setup.bmp"
const RAWFILE = "test.raw"
const BITS    = 32 ' any target bits 8,16,32

dim as integer   hFile,w,h,c
dim as short     bpp
dim as byte ptr  lpBMP
dim as image ptr lpImg

' try to open any bmp file
hFile=FreeFile()
if open(BMPFILE for binary access read as #hFile)<>0 then
  print "error: can't open [" & BMPFILE & "] file !"
  beep:sleep:end
end if

' get dimension w x h bpp and compression from bmp file
get #hFile,19,w
get #hFile,23,h
get #hFile,29,bpp
get #hFile,31,c
close #hFile
if (c<>0) then
  print "error: libfbgfx has no support for compressed bitmaps!"
  beep:sleep:end 1
end if

' now we have w,h
screenres w,h,BITS

lpBMP=ImageCreate(w,h)
' cast byte ptr to image ptr
lpIMG=cptr(image ptr,lpBMP)
bload BMPFILE,lpBMP

' save bmp as raw file 
hFile=FreeFile()
if open(RAWFILE for binary access write as #hFile)<>0 then
  print "error: can't open/create [" & RAWFILE & "] !"
  beep:sleep:end
end if
' save header (32 bytes) + pixels (h*pitch bytes)
put #hFile,,*lpBMP,32+lpImg->pitch*h
close #hFile
ImageDestroy lpBMP
end

incbmp.bas

Code: Select all

#include "incany.bi"
const BITS=32 ' bits per pixel
dim lpBMP as any ptr
incany("test.raw",lpBMP)

screenres 800,600,BITS
put (0,0),lpBMP,pset
sleep
end

[1000101]

Hurray for 100M executables!

Data is stored in separate files for a reason.

[voodooattack]

Looks cool if it's what I think it is (I'm still a bit of a stick in the mud when it comes to FB-specific stuff!) but how is it used? Does it store a file in a resource at compile time and let it be used at run time?

Sorry i forgot to provide an example, here:

incfiletest.bas

Code: Select all

	#include "incfile.bi"
	
	
	incfile (source, "incfiletest.bas")
	
	print "Listing the source code of this program: "
	print
	print *cast(zstring ptr, source)
	
	sleep

in the example above:
'source' becomes a ubyte pointer pointing to the first byte of your file.
a uinteger variable called 'source_len' is created and it holds the total length of your file in memory.

Can I BLOAD BMPs stored in this way?

I think D.J.Peters pretty much answered any questions on the subject, although he used his own macro, this can be done in the same way with IncFile().. :)

Sup Voodoo :), nice trick. Maybe we could add that to fb, but i can't think of any syntax that wouldn't look ugly - and oh yeah, it shouldn't trash the global namespace.

Maybe
... code..

But then FB should allow taking the address of labels to pass them to functions etc.. you wouldn't also know the size of the data unless "var size = (@label_after - @label_before) + 1" were used.

hey v1ctor, that'd be a great feature :)

I'd suggest something like:

Code: Select all

dim myarray() as ubyte => { #binary("somefile") }

..or something along the lines =P

I've already tried messing around with RTLib's FBARRAY structures on my own, although the calls could work, i couldn't get the address of the actual array descriptor from within FB code, So I couldn't use that method through my macro (define a new array and point it to that memory block) :(

Hurray for 100M executables!

Data is stored in separate files for a reason.

heh, I don't see any reason why not.. self-extractors exceed that limit.. I've seen 750 MB exe files... *looks at M$ service packs* ;)
oh, and btw, self-extractors mostly just APPEND the data to the end of the executable, so it's not even worthy, and it can be tampered with much easier...

@D.J.Peters: although your code functions mostly the same, it doesn't provide the length of the file.. which is sometimes needed, and there's an extra (not needed jump)

Code: Select all

.section .text
jmp .end_t '''''' <<< NOT NEEDED
.section .data ''''' < this switch is out of execution context
.align 16
.start_t:
.incbin "incanytest.bas"
.section .text
.align 16
.end_t:  '''' << this label is in the code section, not at the end of the included file... so if you try to calculate the length of the file this way, you'll get wrong results ;)
lea eax, .start_t
mov dword Ptr [ebp-8], eax

good work though =)

[1000101]

Hurray for 100M executables!

Data is stored in separate files for a reason.

heh, I don't see any reason why not.. self-extractors exceed that limit.. I've seen 750 MB exe files... *looks at M$ service packs* ;)
oh, and btw, self-extractors mostly just APPEND the data to the end of the executable, so it's not even worthy, and it can be tampered with much easier...

Ahh, but installers never have bug updates for them. As soon as you update your executable for some bug release, you have to release a huge update.

[voodooattack]

Ahh, but installers never have bug updates for them. As soon as you update your executable for some bug release, you have to release a huge update.

Or just release your updates through a selective patcher, might write one of those soon.. ;)
Anyways, I'm not encouraging packed applications, but this could be useful for situations where you need the data to be packed along, possibly 64k demos, self-extractors, installers, software patches, satellite DLLs, and so on..

[Tigra]

I've started to use voodooattack's macro and I have found an incompatibility with the -g compiler switch.

I use the SciTE editor and I've configured it to compile and run .bas modules when the F5 key is hit. To aid in my debugging I have these default switches:
-g -exx -maxerr inf -s console -w pedantic

When I try to compile the test module, I get these errors:

incfiletest.asm:85: Error: can't resolve `.Data' {.Data section} - `_main' {.text section}
incfiletest.asm:90: Error: can't resolve `.Data' {.Data section} - `_main' {.text section}
incfiletest.asm:95: Error: can't resolve `.Data' {.Data section} - `_main' {.text section}
incfiletest.asm:100: Error: can't resolve `.Data' {.Data section} - `_main' {.text section}
incfiletest.asm:105: Error: can't resolve `.Data' {.Data section} - `_main' {.text section}

Here is the section of the .asm file that the macro expands to (I've marked those five lines with their line number):

##
## #include "includefile.bi"
##
## IncludeFile (source, "incfiletest.bas") [Macro Expansion: IncludeFileEx(source, "incfiletest.bas", Data)Dim Shared source As Ubyte Ptr
## Dim Shared source_len As Uinteger
##asm ]
.section .Data
## asm .section .Data
.stabn 68,0,3,.Lt_000C-_main
.Lt_000D:
##
.LT_START_OF_FILE_source:
## .LT_START_OF_FILE_source#:
.stabn 68,0,3,.Lt_000D-_main <-- 85
.Lt_000E:
##
.incbin "incfiletest.bas"
## .incbin "incfiletest.bas"
.stabn 68,0,3,.Lt_000E-_main <-- 90
.Lt_000F:
##
.LT_END_OF_FILE_source:
## .LT_END_OF_FILE_source:
.stabn 68,0,3,.Lt_000F-_main <-- 95
.Lt_0010:
##
.LONG 0
## .LONG 0
.stabn 68,0,3,.Lt_0010-_main <-- 100
.Lt_0011:
##
.section .text
## .section .text
.stabn 68,0,3,.Lt_0011-_main <-- 105
.Lt_0012:
.balign 16
## .balign 16
.stabn 68,0,3,.Lt_0012-_main
.Lt_0013:
##
lea eax, .LT_START_OF_FILE_source
## lea eax, .LT_START_OF_FILE_source#
.stabn 68,0,3,.Lt_0013-_main
.Lt_0014:
##
mov dword Ptr [_SOURCE], eax
## mov dword Ptr [source], eax
.stabn 68,0,3,.Lt_0014-_main
.Lt_0015:
##
lea ebx, .LT_END_OF_FILE_source
## lea ebx, .LT_END_OF_FILE_source
.stabn 68,0,3,.Lt_0015-_main
.Lt_0016:
##
Sub ebx, eax
## Sub ebx, eax
.stabn 68,0,3,.Lt_0016-_main
.Lt_0017:
##
mov dword Ptr [_SOURCE_LEN], ebx
## mov dword Ptr [source_len], ebx
.stabn 68,0,3,.Lt_0017-_main
.Lt_0018:
## End

When I removed the -g compiler option there are no errors.