NTDLL buffer compression vs ZLIB

MichaelW · Post by **MichaelW** » Jun 17, 2006 10:52

These are updated versions of two quick-and-dirty command-line programs that I used to test the undocumented NTDLL.DLL buffer compression functions, and compare them to the zlib functions. Both programs use memory-mapped files.

Code: Select all

#include once "windows.bi"
#include "ntdll.bi"
option explicit

#define NL chr$(10)

'*******************************************************************************************

function NTCompressFile( uncompressedFileName as zstring ptr, _
                         compressedFileName as zstring ptr, _
                         engine as uint ) _
                         as uint

    dim hFileU as HANDLE, lofU as uint, hMMFU as HANDLE, lpMemU as PVOID
    dim hFileC as HANDLE, lofC as uint, hMMFC as HANDLE, lpMemC as PVOID
    dim workSpaceSize as uint, lpWorkSpace as PVOID
    dim finalSize as uint, rVal as uint, junk as uint
    dim start as double, finish as double

    start = timer

    '' Open the uncompressed file.
    ''
    hFileU = CreateFile( uncompressedFileName, _
                         GENERIC_READ or GENERIC_WRITE, _
                         FILE_SHARE_READ or FILE_SHARE_WRITE, _
                         null, _
                         OPEN_EXISTING, _
                         FILE_ATTRIBUTE_NORMAL, _
                         null )

    if hFileU = INVALID_HANDLE_VALUE then
        print "Error opening source file";NL
        function = -1
        exit function
    endif

    '' Create an unnamed file mapping object for the uncompressed file.
    ''
    hMMFU = CreateFileMapping( hFileU, null, PAGE_READWRITE, 0, 0, null )

    '' Map a view of the uncompressed file into our address space.
    ''
    lpMemU = MapViewOfFile( hMMFU, FILE_MAP_WRITE, 0, 0, 0 )

    '' Open the compressed file.
    ''
    hFileC = CreateFile( compressedFileName, _
                         GENERIC_READ or GENERIC_WRITE, _
                         FILE_SHARE_READ or FILE_SHARE_WRITE, _
                         null, _
                         OPEN_ALWAYS, _
                         FILE_ATTRIBUTE_NORMAL, _
                         null )

    if hFileC = INVALID_HANDLE_VALUE then
        print "Error opening destination file";NL
        return -1
    endif

    '' Get the size of uncompressed file (low-order dword only).
    ''
    lofU = GetFileSize( hFileU, null )

    '' Calc a probable maximum size for the compressed file.
    ''
    lofC = 1.13 * lofU + 4

    '' Create an unnamed file mapping object for the compressed
    '' file, specifying the calculated maximum size.
    ''
    hMMFC = CreateFileMapping( hFileC, null, PAGE_READWRITE, 0, lofC, null )

    ' Map a view of the compressed file into our address space.
    '
    lpMemC = MapViewOfFile( hMMFC, FILE_MAP_WRITE, 0, 0, 0 )

    '' Create a compression workspace (last parameter not needed).
    ''
    RtlGetCompressionWorkSpaceSize( COMPRESSION_FORMAT_LZNT1 or engine, _
                                    @workSpaceSize, _
                                    @junk )
    lpWorkSpace = HeapAlloc( GetProcessHeap( ), 0, workSpaceSize )

    '' Compress the file.
    ''
    rVal = RtlCompressBuffer( COMPRESSION_FORMAT_LZNT1 or engine, _
                              lpMemU, _
                              lofU, _
                              lpMemC, _
                              lofC, _
                              0, _
                              @finalSize, _
                              lpWorkSpace )

    '' Unmap mapped views and close the file mapping object and
    '' uncompressed file handles.
    ''
    UnmapViewOfFile( lpMemU )
    UnmapViewOfFile( lpMemC )
    CloseHandle( hMMFU )
    CloseHandle( hMMFC )
    CloseHandle( hFileU )

    '' Free the allocated compression workspace.
    ''
    HeapFree( GetProcessHeap( ), 0, lpWorkSpace )

    '' Set the final length of the compressed file and close the handle.
    ''
    SetFilePointer( hFileC, finalSize, 0, 0 )
    SetEndOfFile( hFileC )
    CloseHandle( hFileC )

    '' Return whatever RtlCompressBuffer returned.
    ''
    function = rVal

    '' Display statistics.
    ''
    finish = timer
    print "uncompressed size = ";lofU;" bytes"
    print "compressed size = ";finalSize;" bytes"
    print "ratio ="; 100 - int(finalSize / lofU * 100); "%"
    print "elapsed time =";int((finish - start) * 1000);"ms";NL

end function

'*******************************************************************************************

function NTDecompressFile( compressedFileName as zstring ptr, _
                           uncompressedFileName as zstring ptr, _
                           bufferSize as uint) _
                           as uint

    dim hFileC as HANDLE, lofC as uint, hMMFC as HANDLE, lpMemC as PVOID
    dim hFileU as HANDLE, lofU as uint, hMMFU as HANDLE, lpMemU as PVOID
    dim finalSize as uint, rVal as uint
    dim start as double, finish as double

    start = timer

    '' Open the compressed file.
    ''
    hFileC = CreateFile( compressedFileName, _
                         GENERIC_READ or GENERIC_WRITE, _
                         FILE_SHARE_READ or FILE_SHARE_WRITE, _
                         null, _
                         OPEN_EXISTING, _
                         FILE_ATTRIBUTE_NORMAL, _
                         null )

    if hFileC = INVALID_HANDLE_VALUE then
        print "Error opening source file";NL
        function = -1
        exit function
    endif

    '' Create an unnamed file mapping object for the compressed file.
    ''
    hMMFC = CreateFileMapping( hFileC, null, PAGE_READWRITE, 0, 0, null )

    '' Map a view of the compressed file into our address space.
    ''
    lpMemC = MapViewOfFile( hMMFC, FILE_MAP_WRITE, 0, 0, 0 )

    '' Open the uncompressed file.
    ''
    hFileU = CreateFile( uncompressedFileName, _
                         GENERIC_READ or GENERIC_WRITE, _
                         FILE_SHARE_READ or FILE_SHARE_WRITE, _
                         null, _
                         OPEN_ALWAYS, _
                         FILE_ATTRIBUTE_NORMAL, _
                         null )

    if hFileU = INVALID_HANDLE_VALUE then
        print "Error opening destination file";NL
        return -1
    endif

    '' Get the size of compressed file (low-order dword only).
    ''
    lofC = GetFileSize( hFileC, null )


    '' Unless overridden by a non-zero bufferSize parameter, calculate
    '' the maximum size of the uncompressed file based on a 95% ratio.
    ''
    if bufferSize then
      lofU = bufferSize
    else
      lofU = lofC * 20
    endif

    '' Create an unnamed file mapping object for the uncompressed
    '' file, specifying the calculated maximum size.
    ''
    hMMFU = CreateFileMapping( hFileU, null, PAGE_READWRITE, 0, lofU, null )

    '' Map a view of the uncompressed file into our address space.
    ''
    lpMemU = MapViewOfFile( hMMFU, FILE_MAP_WRITE, 0, 0, 0 )

    '' Decompress the file.
    ''
    rVal = RtlDecompressBuffer( COMPRESSION_FORMAT_LZNT1, _
                                lpMemU, _
                                lofU, _
                                lpMemC, _
                                lofC, _
                                @finalSize )

    '' Unmap mapped views and close the file mapping object and
    '' compressed file handles.
    ''
    UnmapViewOfFile( lpMemC )
    UnmapViewOfFile( lpMemU )
    CloseHandle( hMMFC )
    CloseHandle( hMMFU )
    CloseHandle( hFileC )

    '' Set the final length of the uncompressed file and close the handle.
    ''
    SetFilePointer( hFileU, finalSize, 0, 0 )
    SetEndOfFile( hFileU )
    CloseHandle( hFileU )

    '' Return whatever RtlDecompressBuffer returned.
    ''
    function = rVal

    '' Display statistics.
    ''
    finish = timer
    print "compressed size = ";lofC;" bytes"
    print "uncompressed size = ";finalSize;" bytes"
    print "elapsed time =";int((finish - start) * 1000);"ms";NL

end function

'*******************************************************************************************

sub ShowStatus( rVal as uint )
    select case rVal
        case STATUS_SUCCESS
            print "STATUS_SUCCESS"
        case STATUS_UNSUPPORTED_COMPRESSION
            print "STATUS_UNSUPPORTED_COMPRESSION"
        case STATUS_INVALID_PARAMETER
            print "STATUS_INVALID_PARAMETER"
        case STATUS_BUFFER_ALL_ZEROS
            print "STATUS_BUFFER_ALL_ZEROS"
        case STATUS_NOT_SUPPORTED
            print "STATUS_NOT_SUPPORTED"
        case STATUS_BUFFER_TOO_SMALL
            print "STATUS_BUFFER_TOO_SMALL"
        case STATUS_BAD_COMPRESSION_BUFFER
            print "STATUS_BAD_COMPRESSION_BUFFER"
        case else
            print "Unexpected Error ";hex$(rVal);"h"
    end select
end sub

'*******************************************************************************************

dim as uint i,decompress,bufferSize,engine,rVal

i = 1
engine = COMPRESSION_ENGINE_STANDARD

if ucase(left(command$(i),2)) = "/D" then
    decompress = true
    if instr(command$(i),":") then
      bufferSize = val( mid( command$(i), instr(command$(i),":") + 1 ) )
    endif
    i += 1
endif

if ucase(command$(i)) = "/M" then
    engine = COMPRESSION_ENGINE_MAXIMUM
    i += 1
endif

if len(command$(i)) = 0 or len(command$(i+1)) = 0 then
    print NL;"NTCOMP [/D[:buffersize]] [/M] source destination";NL
    print "  /D             Perform a decompress, instead of the default compress."
    print "  [:buffersize]  Specifies a decompress buffer size that overrides the default."
    print "  /M             Use the maximum compression engine.";NL
    print "  The default decompress buffer is sized for a 95% compression ratio.";NL
    print "Press any key to exit..."
    sleep
    end
endif

if decompress then
  print "bufferSize:";bufferSize
  print NL;"Decompressing ";command$(i); " to "; command$(i+1);NL
  rVal = NTDecompressFile( command$(i), command$(i+1), bufferSize )
  ShowStatus rVal
else
  print NL;"Compressing ";command$(i); " to "; command$(i+1);NL
  rVal = NTCompressFile( command$(i), command$(i+1), engine )
  ShowStatus rVal
endif

print NL;"Press any key to exit..."
sleep

Code: Select all

#include once "windows.bi"
#include "zlib.bi"
option explicit

#define NL chr$(10)

'*******************************************************************************************

function ZCompressFile( uncompressedFileName as zstring ptr, _
                        compressedFileName as zstring ptr, _
                        level as integer ) _
                        as integer

    dim hFileU as HANDLE, lofU as uint, hMMFU as HANDLE, lpMemU as PVOID
    dim hFileC as HANDLE, lofC as uint, hMMFC as HANDLE, lpMemC as PVOID
    dim rVal as uint, start as double, finish as double

    start = timer

    '' Open the uncompressed file.
    ''
    hFileU = CreateFile( uncompressedFileName, _
                         GENERIC_READ or GENERIC_WRITE, _
                         FILE_SHARE_READ or FILE_SHARE_WRITE, _
                         null, _
                         OPEN_EXISTING, _
                         FILE_ATTRIBUTE_NORMAL, _
                         null )

    if hFileU = INVALID_HANDLE_VALUE then
        print "Error opening source file";NL
        function = -1
        exit function
    endif

    '' Create an unnamed file mapping object for the uncompressed file.
    ''
    hMMFU = CreateFileMapping( hFileU, null, PAGE_READWRITE, 0, 0, null )

    '' Map a view of the uncompressed file into our address space.
    ''
    lpMemU = MapViewOfFile( hMMFU, FILE_MAP_WRITE, 0, 0, 0 )

    '' Open the compressed file.
    ''
    hFileC = CreateFile( compressedFileName, _
                         GENERIC_READ or GENERIC_WRITE, _
                         FILE_SHARE_READ or FILE_SHARE_WRITE, _
                         null, _
                         OPEN_ALWAYS, _
                         FILE_ATTRIBUTE_NORMAL, _
                         null )

    if hFileC = INVALID_HANDLE_VALUE then
        print "Error opening destination file";NL
        function = -1
        exit function
    endif

    '' Get the size of uncompressed file (low-order dword only).
    ''
    lofU = GetFileSize( hFileU, null )

    '' Calc a probable maximum size for the compressed file.
    ''
    lofC = 1.001 * lofU + 12

    '' Create an unnamed file mapping object for the compressed
    '' file, specifying the calculated maximum size.
    ''
    hMMFC = CreateFileMapping( hFileC, null, PAGE_READWRITE, 0, lofC, null )

    '' Map a view of the compressed file into our address space.
    ''
    lpMemC = MapViewOfFile( hMMFC, FILE_MAP_WRITE, 0, 0, 0 )

    '' Compress the file.
    ''
    rVal = compress2( lpMemC, @lofC, lpMemU, lofU, level )

    '' Unmap the mapped views and close the file mapping object and
    '' uncompressed file handles.
    ''
    UnmapViewOfFile( lpMemU )
    UnmapViewOfFile( lpMemC )
    CloseHandle( hMMFU )
    CloseHandle( hMMFC )
    CloseHandle( hFileU )

    '' Set the final length of the compressed file and close the handle.
    ''
    SetFilePointer( hFileC, lofC, 0, 0 )
    SetEndOfFile( hFileC )
    CloseHandle( hFileC )

    '' Return whatever compress2 returned.
    ''
    function = rVal

    '' Display statistics.
    ''
    finish = timer
    print "uncompressed size = ";lofU;" bytes"
    print "compressed size = ";lofC;" bytes"
    print "ratio ="; 100 - int(lofC / lofU * 100); "%"
    print "elapsed time =";int((finish - start) * 1000);"ms";NL

end function

'*******************************************************************************************

function ZUncompressFile( compressedFileName as zstring ptr, _
                          uncompressedFileName as zstring ptr, _
                          bufferSize as uint) _
                          as integer

    dim hFileC as HANDLE, lofC as uint, hMMFC as HANDLE, lpMemC as PVOID
    dim hFileU as HANDLE, lofU as uint, hMMFU as HANDLE, lpMemU as PVOID
    dim rVal as uint, start as double, finish as double

    start = timer

    '' Open the compressed file.
    ''
    hFileC = CreateFile( compressedFileName, _
                         GENERIC_READ or GENERIC_WRITE, _
                         FILE_SHARE_READ or FILE_SHARE_WRITE, _
                         null, _
                         OPEN_EXISTING, _
                         FILE_ATTRIBUTE_NORMAL, _
                         null )

    if hFileC = INVALID_HANDLE_VALUE then
        print "Error opening source file";NL
        function = -1
        exit function
    endif

    '' Create an unnamed file mapping object for the compressed file.
    ''
    hMMFC = CreateFileMapping( hFileC, null, PAGE_READWRITE, 0, 0, null )

    '' Map a view of the compressed file into our address space.
    ''
    lpMemC = MapViewOfFile( hMMFC, FILE_MAP_WRITE, 0, 0, 0 )

    '' Open the uncompressed file.
    ''
    hFileU = CreateFile( uncompressedFileName, _
                         GENERIC_READ or GENERIC_WRITE, _
                         FILE_SHARE_READ or FILE_SHARE_WRITE, _
                         null, _
                         OPEN_ALWAYS, _
                         FILE_ATTRIBUTE_NORMAL, _
                         null )

    if hFileU = INVALID_HANDLE_VALUE then
        print "Error opening destination file";NL
        function = -1
        exit function
    endif

    '' Get the size of compressed file (low-order dword only).
    ''
    lofC = GetFileSize( hFileC, null )

    '' Unless overridden by a non-zero bufferSize parameter, calculate
    '' the maximum size of the uncompressed file based on a 99% ratio.
    ''
    if bufferSize then
      lofU = bufferSize
    else
      lofU = lofC * 100
    endif

    '' Create an unnamed file mapping object for the uncompressed
    '' file, specifying the calculated maximum size.
    ''
    hMMFU = CreateFileMapping( hFileU, null, PAGE_READWRITE, 0, lofU, null )

    '' Map a view of the uncompressed file into our address space.
    ''
    lpMemU = MapViewOfFile( hMMFU, FILE_MAP_WRITE, 0, 0, 0 )

    '' Decompress the file.
    ''
    rVal = uncompress( lpMemU, @lofU, lpMemC, lofC )

    '' Unmap mapped views and close the file mapping object and
    '' compressed file handles.
    ''
    UnmapViewOfFile( lpMemC )
    UnmapViewOfFile( lpMemU )
    CloseHandle( hMMFC )
    CloseHandle( hMMFU )
    CloseHandle( hFileC )

    '' Set the final length of the uncompressed file and close the handle.
    ''
    SetFilePointer( hFileU, lofU, 0, 0 )
    SetEndOfFile( hFileU )
    CloseHandle( hFileU )

    '' Return whatever uncompress returned.
    ''
    function = rVal

    '' Display statistics.
    ''
    finish = timer
    print "compressed size = ";lofC;" bytes"
    print "uncompressed size = ";lofU;" bytes"
    print "elapsed time =";int((finish - start) * 1000);"ms";NL

end function

'*******************************************************************************************

sub ShowStatus( rVal as uint )
    select case rVal
        case Z_OK
            print "Z_OK"
        case Z_STREAM_ERROR
            print "Z_STREAM_ERROR"
        case Z_DATA_ERROR
            print "Z_DATA_ERROR"
        case Z_MEM_ERROR
            print "Z_MEM_ERROR"
        case Z_BUF_ERROR
            print "Z_BUF_ERROR"
        case else
            print "Unexpected Error ";hex$(rVal);"h"
    end select
end sub

'*******************************************************************************************

dim as uint i,decompress,bufferSize,rVal
dim level as integer

i = 1

if ucase(left(command$(i),2)) = "/D" then
    decompress = true
    if instr(command$(i),":") then
      bufferSize = val( mid( command$(i), instr(command$(i),":") + 1 ) )
    endif
    i += 1
endif

if ucase(left(command$(i),2)) = "/L" then
    level = val( mid( command$(i), instr(command$(i),":") + 1 ) )
    i += 1
endif

if len(command$(i)) = 0 or len(command$(i+1)) = 0 then
    print NL;"ZCOMP [/D[:buffersize]] [/L:level] source destination";NL
    print "  /D             Perform a decompress, instead of the default compress."
    print "  [:buffersize]  Specifies a decompress buffer size that overrides the default."
    print "  /L:level       Specifies compression level (0-9).";NL
    print "  The default decompress buffer is sized for a 99% compression ratio.";NL
    print "Press any key to exit..."
    sleep
    end
endif

if decompress then
  print "bufferSize:";bufferSize
  print NL;"Decompressing ";command$(i); " to "; command$(i+1);NL
  rVal = ZUncompressFile( command$(i), command$(i+1), bufferSize )
  ShowStatus rVal
else
  print NL;"Compressing ";command$(i); " to "; command$(i+1);NL
  if level > 0 then
    rVal = ZCompressFile( command$(i), command$(i+1), level )
  else
    rVal = ZCompressFile( command$(i), command$(i+1), Z_DEFAULT_COMPRESSION )
  endif
  ShowStatus rVal
endif

print NL;"Press any key to exit..."
sleep

The test results:

Code: Select all

Using as a test file a 3442497 byte text file created by combining
all of the include files in the inc\win directory (v0.15b stable).

NTComp, standard compression engine:
    compressed size = 1040211 bytes
    ratio = 70%
    compression time = 274ms
    decompression time = 72ms

NTComp, maximum compression engine:
    compressed size = 936522 bytes
    ratio = 73%
    compression time = 25102ms
    decompression time = 69ms

ZComp, default compression (level 6):
    compressed size = 527619 bytes
    ratio = 85%
    compression time = 998ms
    decompression time = 101ms

ZComp, level 1 (best speed):
    compressed size = 653851 bytes
    ratio = 82%
    compression time = 381ms
    decompression time = 108ms

ZComp, level 9 (best compression):
    compressed size = 519310 bytes
    ratio = 85%
    compression time  = 2573ms
    decompression time = 100ms

The DEF and batch files that I used to create the NTDLL import library:

Code: Select all

;
; NTDLL.DLL.DEF
;
; Module definition file for creating the import library ntdll.dll.a.
;
; This file defines only 3 of the 1000+ functions in NTDLL.DLL.
;

LIBRARY NTDLL.DLL
EXPORTS
RtlGetCompressionWorkSpaceSize@12
RtlCompressBuffer@32
RtlDecompressBuffer@24

Code: Select all

:
: MAKELIB.BAT
:
: This file creates the import library ntdll.dll.a.
:
: dlltool.exe and as.exe must be in the current directory.
:

dlltool -k -d ntdll.dll.def -l ntdll.dll.a

pause

And the matching include file:

Code: Select all

''
'' NTDLL.BI
''
'' This file defines a *tiny* subset of the NTDLL functions
'' and related data types and constants, specifically the
'' *minimum* components required to use the native buffer
'' compression functions.
''
#inclib "ntdll"

'' These already defined in winnt.bi:
''
''#define COMPRESSION_FORMAT_LZNT1    2 ' the only format currently supported
''#define COMPRESSION_ENGINE_STANDARD 0 ' the only 2 engines currently supported
''#define COMPRESSION_ENGINE_MAXIMUM  &H100

type NTSTATUS as UINT

#define STATUS_SUCCESS                  &h00000000
#define STATUS_UNSUPPORTED_COMPRESSION  &hC000025F
#define STATUS_INVALID_PARAMETER        &hC000000D
#define STATUS_BUFFER_ALL_ZEROS         &h00000117
#define STATUS_NOT_SUPPORTED            &hC00000BB
#define STATUS_BUFFER_TOO_SMALL         &hC0000023
#define STATUS_BAD_COMPRESSION_BUFFER   &hC0000242

declare function RtlGetCompressionWorkSpaceSize alias "RtlGetCompressionWorkSpaceSize"( _
                    byval dwCompressionFormatAndEngine as DWORD,_
                    byval lpdwCompressBufferWorkSpaceSize as DWORD ptr, _
                    byval lpdwCompressFragmentWorkSpaceSize as DWORD ptr ) _
                    as NTSTATUS

declare function RtlCompressBuffer alias "RtlCompressBuffer"( _
                    byval dwCompressionFormatAndEngine as DWORD, _
                    byval lpUnCompressedBuffer as LPCVOID, _
                    byval dwUnCompressedBufferSize As DWORD, _
                    byval lpCompressedBuffer as LPCVOID, _
                    byval dwCompressedBufferSize As DWORD, _
                    byval dwUnCompressedChunkSize As DWORD, _
                    byval lpdwFinalCompressedSize as DWORD ptr, _
                    byval lpCompressBufferWorkspace as LPCVOID ) _
                    as NTSTATUS

Declare Function RtlDecompressBuffer alias "RtlDecompressBuffer"( _
                    byval dwCompressionFormat as DWORD, _
                    byval lpUnCompressedBuffer as LPCVOID, _
                    byval dwUnCompressedBufferSize As DWORD, _
                    byval lpCompressedBuffer as LPCVOID, _
                    byval dwCompressedBufferSize As DWORD, _
                    byval lpdwFinalDecompressedSize as DWORD ptr ) _
                    as NTSTATUS

yetifoot · Post by **yetifoot** » Jun 17, 2006 14:07

Do you know what versions of windows it runs on? I took a look but obviously being undocumented i couldn't find much about it.

I'm always a bit scared to use undocumented functions after reading the blog by Raymond Chen, and all his talk of shimming to deal with people who used undocumented code. (Old New Thing)

http://blogs.msdn.com/oldnewthing/archi ... 55296.aspx

Do microsoft use these functions? I read that since the anti-trust stuff, if they use a function in one of their apps then it MUST be documented or they will get into trouble.

http://blogs.msdn.com/calvin_hsia/archi ... 61033.aspx

MichaelW · Post by **MichaelW** » Jun 17, 2006 21:43

NTDLL.DLL should be present for NT/2000/XP. You should be able to find a lot of information on it, and most of the functions have been partially documented, but not by Microsoft. I would be hesitant to use an undocumented function in a commercial app, but I know that this was done quite a lot starting back in the DOS days, and I’m sure it’s continuing today. I don’t know if NTDLL.DLL is present and/or functional in the Vista betas, but sooner or later it will probably disappear. Microsoft could also remove or cripple it with a SP, but I think that this is very unlikely to happen. For Windows 2000 I know that Microsoft made heavy use of NTDLL.DLL. For example, the kernel32.dll on my system imports ~310 functions from NTDLL.DLL, and forwards 12 of its exports to NTDLL.DLL (even common functions like HeapAlloc, HeapFree, HeapReAlloc, and HeapSize). I can’t check XP because I currently don’t have it installed on anything, but I suspect that it to makes heavy use of NTDLL.DLL.

voodooattack · Post by **voodooattack** » Jun 17, 2006 23:24

NTDLL is the NT back-end...

the dll does nothing but forward the params of any call to the kernel subsystems...

for that, it uses some sort of interrupt call (i can't remember which)...

most of the functions' parameter verification code relays in higher level dlls (i.e kernel32, user32, etc..), so by linking to it directly, you're skipping hecks of checks (speeding up your applications up to 10x some times)

and btw, the compressions functions here are used by the system for NTFS file compression too..

i personally have some experience linking to the dll, for example, linking CRT functions to their low level equivalents in NTDLL instead (sin, cos, memset, and more)...

I've tested that with many small FB games and it gave a HUGE fps increase...

that's the good side...

on the other side, i must say that it's really hazardous linking to the dll directly, as of that M$ is not responsible for any changes that may happen to the dll, they can add,remove, rename functions without notice, they can change the functions' parameters, calling convention at any time.. and yet we don't have the right to question...

i must say that you should not link directly to the dll, try loading the DLL dynamically using LoadLibrary (nothing is loaded though, since the DLL would essentially be there.. already loaded in your application's address space) and calling the functions only if you find them (that's why i was asking about "delay load modules/lazy binding" in a previous thread...

however, if you're looking for any documentation about the dll.. here's a good unofficial reference:

http://undocumented.ntinternals.net/

and they offer a CHM version too :)

MichaelW · Post by **MichaelW** » Jun 18, 2006 2:26

What “hazard”, and why load dynamically? Microsoft is not likely to break these functions in any way in an existing version of Windows, and if the functions are not available, Windows will let you know when you attempt to start the app.

voodooattack · Post by **voodooattack** » Jun 18, 2006 4:06

MichaelW wrote:What “hazard”

the hazard is that your applications may crash.. see below..

MichaelW wrote:, and why load dynamically?

to prevent your applications from crashing, and get the chance to recover if the functions are not found..

MichaelW wrote:Microsoft is not likely to break these functions in any way in an existing version of Windows, and if the functions are not available, Windows will let you know when you attempt to start the app.

Microsoft is more likely to break them (and i'm not talking about the standard functions, like the CRT equivalents)..

for example, microsoft implemented only one native compression engine so far, LZNT1... they might add new engines anytime, and since they are NOT obligated to keep their functions the way they are now, they might decide it's better to do some changes, and may simply do that at any moment...

even the slightest change in the functions' prototypes, will crash your application instantly...

i think the upcoming vista is a good example..

what i'm saying here.. is that using the native APIs is cool... but not really suitable for a production environment...

and btw, did you try LZMA?

it beats the dust out of LZNT1 & ZLIB :P

i didn't finish translating the headers yet though :-/

http://www.7-zip.org/sdk.html

MichaelW · Post by **MichaelW** » Jun 18, 2006 6:14

If you are going to criticize, at least formulate criticisms that have some substance. Right up at the top of my initial post I included the word “test”, and in the second post “I would be hesitant to use an undocumented function in a commercial app…”, so what is this about a production environment? For a test app, or even a small-time utility, crashing is not a hazard. And your argument regarding the incomplete state of these functions is weak, given that these functions have been in this incomplete state since 1999, at least.

voodooattack · Post by **voodooattack** » Jun 18, 2006 6:55

MichaelW wrote:If you are going to criticize, at least formulate criticisms that have some substance. Right up at the top of my initial post I included the word “test”, and in the second post “I would be hesitant to use an undocumented function in a commercial app…”, so what is this about a production environment? For a test app, or even a small-time utility, crashing is not a hazard. And your argument regarding the incomplete state of these functions is weak, given that these functions have been in this incomplete state since 1999, at least.

chill out mate.. i wasn't trying to argue here ;)

i personally *love* to mess around with NT internals, and i did use that compression API long ago, what i'm talking about is.. its rather non-practical to use in real life applications...

the dynamic linkage stuff above.. was to make sure if anyone decides its cool enough to implement, they should be aware of the consequences of using it, and how to be prepared if they decide to use it anyway :)

(i did use them in apps before, and i have some experience with the subject)

given that these functions have been in this incomplete state since 1999, at least.

that's another reason why m$ may alter their internal APIs..
5 years have passed since the release of Windows XP, now windows Vista is almost out, and it's being heavily developed for five years (as of what m$'ve been blabbing around everywhere) and you don't expect its internal system calls/structures to be altered in any way? ;)

nimdays · Post by **nimdays** » Jun 12, 2015 0:31

Interisting , i want to try to create native app with NtProcessStartup and display a message during startup
A small example

marpon · Post by **marpon** » Aug 20, 2017 11:39

Reactivating that old post...

here is an adaptation to use the ntdll compression feature with multiples files compressed into single archive file and reverse.

the interresting point is it's free, no extra dll to add, very fast , but not as efficient as other compression tools.

the code

Code: Select all

#include once "windows.bi"
#include once "crt/string.bi"  ' needed for strtok


#define _DEBUG_FB		 'show debug info if uncommented  else nothing shown

#ifndef _DEBUG_FB_
	#define _DEBUG_FB_
	#MACRO DEBUG_FB(st)
		#ifdef _DEBUG_FB
			print(st)
		#ENDIF	
	#ENDMACRO
#ENDIF

' These are already defined in winnt.bi:
'
'#define COMPRESSION_FORMAT_LZNT1 2 ' the only format currently supported
'#define COMPRESSION_ENGINE_STANDARD 0 ' the first of 2 engines currently supported
'#define COMPRESSION_ENGINE_MAXIMUM &H100 ' and the second supported


' These for ntdll compression usage
#define _SUCCESS_NTDLL &h00000000
#define _INVALID_PARAMETER_NTDLL &hC000000D
#define _NOT_SUPPORTED_NTDLL &hC00000BB
#define _UNSUPPORTED_COMPRESSION_NTDLL &hC000025F
#define _BUFFER_ALL_ZEROS_NTDLL &h00000117
#define _NOT_SUPPORTED_NTDLL &hC00000BB
#define _BUFFER_TOO_SMALL_NTDLL &hC0000023
#define _BAD_COMPRESSION_BUFFER_NTDLL &hC0000242

' These for more error info
#define _SOURCE_FILE_ERROR &h00000001
#define _DESTINATION_FILE_ERROR &h00000002

#define _MAX_BUFFER_SIZE_NTDLL &h0FFFFFFF        ' can be modified up to ULONG Max

#define _FILE_SEPARATOR_ 	"|"


type my_info_ntdll                               ' gCollect
   as long uncomp_size
   as long comp_size
   as long elapsed
   as long comp_dec
	as long retval
END TYPE

dim shared as my_info_ntdll gCollect             ' to collect the different info


Extern "Windows" Lib "ntdll"
   declare function RtlGetCompressionWorkSpaceSize alias "RtlGetCompressionWorkSpaceSize"( _
         byval dwCompressionFormatAndEngine as Ulong , _
         byval lpdwCompressBufferWorkSpaceSize as Ulong ptr , _
         byval lpdwCompressFragmentWorkSpaceSize as Ulong ptr) _
         as ULong
   
   declare function RtlCompressBuffer alias "RtlCompressBuffer"( _
         byval dwCompressionFormatAndEngine as Ulong , _
         byval lpUnCompressedBuffer as any Ptr , _
         byval dwUnCompressedBufferSize As Ulong , _
         byval lpCompressedBuffer as any Ptr , _
         byval dwCompressedBufferSize As Ulong , _
         byval dwUnCompressedChunkSize As Ulong , _
         byval lpdwFinalCompressedSize as Ulong ptr , _
         byval lpCompressBufferWorkspace as any Ptr) _
         as ULong
   
   Declare Function RtlDecompressBuffer alias "RtlDecompressBuffer"( _
         byval dwCompressionFormat as Ulong , _
         byval lpUnCompressedBuffer as any Ptr , _
         byval dwUnCompressedBufferSize As Ulong , _
         byval lpCompressedBuffer as any Ptr , _
         byval dwCompressedBufferSize As Ulong , _
         byval lpdwFinalDecompressedSize as Ulong ptr) _
         as ULong
end extern

'*******************************************************************************************

private sub clean_global()
   gCollect.uncomp_size = 0
   gCollect.comp_size = 0
   gCollect.elapsed = 0
	gCollect.retval = 0
end sub

private function NTCompressFile(Byval uncompressedFileName as zstring ptr , _
         Byval compressedFileName as zstring ptr , _
         Byval engine as ULong = COMPRESSION_ENGINE_STANDARD) _
         as ULong
   
   dim hFileU            as HANDLE
   dim lofU              as ULong
   dim hMMFU             as HANDLE
   dim lpMemU            as any Ptr
   dim hFileC            as HANDLE
   dim lofC              as ULong
   dim hMMFC             as HANDLE
   dim lpMemC            as any Ptr
   dim workSpaceSize     as ULong
   dim lpWorkSpace       as any Ptr
   dim finalSize         as ULong
   dim rVal              as ULong
   dim junk              as ULong
   dim start             as double
   dim finish            as double
   
   clean_global()
   gCollect.comp_dec = 1
   start = timer
   
   ' Open the uncompressed file.
   hFileU = CreateFile(uncompressedFileName , _
         GENERIC_READ or GENERIC_WRITE , _
         FILE_SHARE_READ or FILE_SHARE_WRITE , _
         null , _
         OPEN_EXISTING , _
         FILE_ATTRIBUTE_NORMAL , _
         null)
   
   if hFileU = INVALID_HANDLE_VALUE then
      function = _SOURCE_FILE_ERROR
      exit function
   end if
   
   ' Get the size of uncompressed file (low-order Ulong only).
   lofU = GetFileSize(hFileU , null)
   
   '? "size = ";lofU
   
   ' Open the compressed file.
   hFileC = CreateFile(compressedFileName , _
         GENERIC_READ or GENERIC_WRITE , _
         FILE_SHARE_READ or FILE_SHARE_WRITE , _
         null , _
         OPEN_ALWAYS , _
         FILE_ATTRIBUTE_NORMAL , _
         null)
   
   if hFileC = INVALID_HANDLE_VALUE then
      return _DESTINATION_FILE_ERROR
   end if
   
   if lofU = 0 THEN
      CloseHandle(hFileU)
      CloseHandle(hFileC)
      function = 0
      exit function
   END IF
   
   ' Create an unnamed file mapping object for the uncompressed file.
   hMMFU = CreateFileMapping(hFileU , null , PAGE_READWRITE , 0 , 0 , null)
   
   ' Map a view of the uncompressed file into our address space.
   lpMemU = MapViewOfFile(hMMFU , FILE_MAP_WRITE , 0 , 0 , 0)
   
   
   ' reserve a maximum size for the compressed file. (it can be bigger than uncompressed)
   lofC = (1.25 * lofU) + 2048
   if lofC > _MAX_BUFFER_SIZE_NTDLL THEN lofC = _MAX_BUFFER_SIZE_NTDLL
   
   ' Create an unnamed file mapping object for the compressed
   hMMFC = CreateFileMapping(hFileC , null , PAGE_READWRITE , 0 , lofC , null)
   
   ' Map a view of the compressed file into our address space.
   lpMemC = MapViewOfFile(hMMFC , FILE_MAP_WRITE , 0 , 0 , 0)
   
   ' Create a compression workspace (last parameter not needed).
   RtlGetCompressionWorkSpaceSize(COMPRESSION_FORMAT_LZNT1 or engine , _
         @workSpaceSize , _
         @junk)
   lpWorkSpace = HeapAlloc(GetProcessHeap() , 0 , workSpaceSize)
   
   ' Compress the file.
   rVal = RtlCompressBuffer(COMPRESSION_FORMAT_LZNT1 or engine , _
         lpMemU , _
         lofU , _
         lpMemC , _
         lofC , _
         4096 , _
         @finalSize , _
         lpWorkSpace)
   
   ' Unmap mapped views and close the file mapping object and
   ' uncompressed file handles.
   UnmapViewOfFile(lpMemU)
   UnmapViewOfFile(lpMemC)
   CloseHandle(hMMFU)
   CloseHandle(hMMFC)
   CloseHandle(hFileU)
   
   ' Free the allocated compression workspace.
   HeapFree(GetProcessHeap() , 0 , lpWorkSpace)
   
   ' Set the final length of the compressed file and close the handle.
   SetFilePointer(hFileC , finalSize , 0 , 0)
   SetEndOfFile(hFileC)
   CloseHandle(hFileC)
   
   ' Return whatever RtlCompressBuffer returned.
   function = rVal
   finish = timer
   
   gCollect.uncomp_size = lofU
   gCollect.comp_size = finalSize
   gCollect.elapsed = int((finish - start) *1000)
   gCollect.retval = rVal
end function

'*******************************************************************************************

private function NTDecompressFile(Byval compressedFileName as zstring ptr , _
         Byval uncompressedFileName as zstring ptr , _
         Byval bufferSize as Ulong = 0) _
         as ULong
   
   dim hFileC            as HANDLE
   dim lofC              as ULong
   dim hMMFC             as HANDLE
   dim lpMemC            as any Ptr
   dim hFileU            as HANDLE
   dim lofU              as ULong
   dim hMMFU             as HANDLE
   dim lpMemU            as any Ptr
   dim finalSize         as ULong
   dim rVal              as ULong
   dim start             as double
   dim finish            as double
   
   'dim ncycles as ULong
   
   clean_global()
   gCollect.comp_dec = 0
   start = timer
   
   ' Open the compressed file.
   hFileC = CreateFile(compressedFileName , _
         GENERIC_READ or GENERIC_WRITE , _
         FILE_SHARE_READ or FILE_SHARE_WRITE , _
         null , _
         OPEN_EXISTING , _
         FILE_ATTRIBUTE_NORMAL , _
         null)
   
   if hFileC = INVALID_HANDLE_VALUE then
      function = _SOURCE_FILE_ERROR
      exit function
   end if
   
   ' Get the size of compressed file (low-order Ulong only).
   lofC = GetFileSize(hFileC , null)
   '? "size = ";lofC
   
   ' Open the uncompressed file.
   hFileU = CreateFile(uncompressedFileName , _
         GENERIC_READ or GENERIC_WRITE , _
         FILE_SHARE_READ or FILE_SHARE_WRITE , _
         null , _
         OPEN_ALWAYS , _
         FILE_ATTRIBUTE_NORMAL , _
         null)
   
   if hFileU = INVALID_HANDLE_VALUE then
      return _DESTINATION_FILE_ERROR
   end if
   
   if lofC = 0 THEN
      CloseHandle(hFileU)
      CloseHandle(hFileC)
      function = 0
      exit function
   END IF
   
   ' Create an unnamed file mapping object for the compressed file.
   hMMFC = CreateFileMapping(hFileC , null , PAGE_READWRITE , 0 , 0 , null)
   
   ' Map a view of the compressed file into our address space.
   lpMemC = MapViewOfFile(hMMFC , FILE_MAP_WRITE , 0 , 0 , 0)
   
   'test first with "minimal" buffer or bufferSize
   lofU = (lofC * 4) + 2048
   if bufferSize > _MAX_BUFFER_SIZE_NTDLL THEN
      lofU = _MAX_BUFFER_SIZE_NTDLL
   elseif bufferSize > lofU THEN
      lofU = bufferSize
   elseif lofU > _MAX_BUFFER_SIZE_NTDLL then
      lofU = _MAX_BUFFER_SIZE_NTDLL
   end if
   
   ' to insure loop while get _BAD_COMPRESSION_BUFFER_NTDLL (in fact buffer too small)
   do
      
      'ncycles += 1
      
      ' Create an unnamed file mapping object for the uncompressed
      ' file, specifying the calculated maximum size.
      hMMFU = CreateFileMapping(hFileU , null , PAGE_READWRITE , 0 , lofU , null)
      
      ' Map a view of the uncompressed file into our address space.
      lpMemU = MapViewOfFile(hMMFU , FILE_MAP_WRITE , 0 , 0 , 0)
      
      ' Decompress the file.
      rVal = RtlDecompressBuffer(COMPRESSION_FORMAT_LZNT1 , _
            lpMemU , _
            lofU , _
            lpMemC , _
            lofC , _
            @finalSize)
      
      ' Unmap mapped views and close the file mapping object and
      ' compressed file handles.
      UnmapViewOfFile(lpMemU)
      CloseHandle(hMMFU)
      lofU *= 2                                  ' increase buffer *2 each loop
      if lofU >= _MAX_BUFFER_SIZE_NTDLL THEN exit do
   loop while rVal = _BAD_COMPRESSION_BUFFER_NTDLL
   '? "ncycles = " ;ncycles
   
   UnmapViewOfFile(lpMemC)
   CloseHandle(hMMFC)
   CloseHandle(hFileC)
   
   ' Set the final length of the uncompressed file and close the handle.
   SetFilePointer(hFileU , finalSize , 0 , 0)
   SetEndOfFile(hFileU)
   CloseHandle(hFileU)
   
   ' Return whatever RtlDecompressBuffer returned.
   function = rVal
   finish = timer
   
   gCollect.uncomp_size = finalSize
   gCollect.comp_size = lofC
   gCollect.elapsed = int((finish - start) *1000)
   gCollect.retval = rVal
end function

'*******************************************************************************************

private sub ShowStatus()
	dim rVal as ULong = gCollect.retval
   Dim as string info = "COMPRESSION"
   if gCollect.comp_dec = 0 THEN info = "DECOMPRESSION"
   select case rVal
      case _SUCCESS_NTDLL
         print "SUCCESS_" & info & "_NTDLL"
      case _UNSUPPORTED_COMPRESSION_NTDLL
         print "UNSUPPORTED_" & info & "_NTDLL"
      case _INVALID_PARAMETER_NTDLL
         print "INVALID_PARAMETER_NTDLL"
      case _BUFFER_ALL_ZEROS_NTDLL
         print "BUFFER_ALL_ZEROS_NTDLL"
      case _NOT_SUPPORTED_NTDLL
         print "NOT_SUPPORTED_NTDLL"
      case _BUFFER_TOO_SMALL_NTDLL
         print "BUFFER_TOO_SMALL_NTDLL"
      case _BAD_COMPRESSION_BUFFER_NTDLL
         print "BAD_" & info & "_BUFFER_NTDLL"
      case _SOURCE_FILE_ERROR
         print "SOURCE_FILE_ERROR"
      case _DESTINATION_FILE_ERROR
         print "DESTINATION_FILE_ERROR"
      case else
         print "Unexpected Error " ; hex(rVal) ; "h"
   end select
   if rVal = 0 THEN
      print "Uncompressed size = " ; gCollect.uncomp_size ; " bytes"
      print "Compressed size = " ; gCollect.comp_size ; " bytes"
      if gCollect.uncomp_size THEN
         if gCollect.comp_size < gCollect.uncomp_size THEN
            print "Deflated compressed : " ; 100 - int(gCollect.comp_size / gCollect.uncomp_size * 100) ; " %"
         else
            print "Inflated compressed : " ; int(gCollect.comp_size / gCollect.uncomp_size * 100) ; " %"
         END IF
      else
         print "Warning : empty files"
      END IF
      
      print "Elapsed time = " ; gCollect.elapsed ; " ms"
   END IF
end sub


private sub MsgStatus()
	dim rVal as ULong = gCollect.retval
   Dim as string info = "COMPRESSION"
   Dim         as string collect_info
   
   if gCollect.comp_dec = 0 THEN info = "DECOMPRESSION"
   select case rVal
      case _SUCCESS_NTDLL
         collect_info += "SUCCESS_" & info & "_NTDLL"
      case _UNSUPPORTED_COMPRESSION_NTDLL
         collect_info += "UNSUPPORTED_" & info & "_NTDLL"
      case _INVALID_PARAMETER_NTDLL
         collect_info += "INVALID_PARAMETER_NTDLL"
      case _BUFFER_ALL_ZEROS_NTDLL
         collect_info += "BUFFER_ALL_ZEROS_NTDLL"
      case _NOT_SUPPORTED_NTDLL
         collect_info += "NOT_SUPPORTED_NTDLL"
      case _BUFFER_TOO_SMALL_NTDLL
         collect_info += "BUFFER_TOO_SMALL_NTDLL"
      case _BAD_COMPRESSION_BUFFER_NTDLL
         collect_info += "BAD_" & info & "_BUFFER_NTDLL"
      case _SOURCE_FILE_ERROR
         collect_info += "SOURCE_FILE_ERROR"
      case _DESTINATION_FILE_ERROR
         collect_info += "DESTINATION_FILE_ERROR"
      case else
         collect_info += "Unexpected Error " & hex(rVal) & "h"
   end select
   if rVal = 0 THEN
      collect_info += (chr(10 , 10) & "Uncompressed size = " & gCollect.uncomp_size & " bytes" & chr(10))
      collect_info += ( "Compressed size = " & gCollect.comp_size & " bytes" & chr(10))
      if gCollect.uncomp_size THEN
         if gCollect.comp_size < gCollect.uncomp_size THEN
            collect_info += ( "Deflated compressed : " & str(100 - int(gCollect.comp_size / gCollect.uncomp_size * 100)) _
                  & " %" & chr(10))
         else
            collect_info += ( "Inflated compressed : " & int(gCollect.comp_size / gCollect.uncomp_size * 100) _
                  & " %" & chr(10))
         END IF
      else
         collect_info += ( "Warning : empty files" & chr(10))
      END IF
      
      collect_info += ( "Elapsed time = " & gCollect.elapsed & " ms" & chr(10))
      messagebox(0 , collect_info , "Status : " & info , MB_ICONINFORMATION)
   else
      messagebox(0 , collect_info , "Status : " & info , MB_ICONERROR)
   END IF
   
end sub

'*******************************************************************************************

private Function Parse_names(ByRef sText As String , _
         ByRef sDelimiter As String , _
         ByVal nPosition As Integer _
         ) As String  
   Dim sTemp As String = sText
   Dim pch               As ZString Ptr
   Dim i                 As Integer  
   If Len(sText) = 0 Then return ""
   pch = strtok(StrPtr(sTemp) , sDelimiter)
   For i = 1 To nPosition
      If (pch <> 0) And (i = nPosition) Then Return *pch
      pch = strtok(0 , sDelimiter)
   Next  
   return ""  
End Function

private Sub Kill_files(ByRef filespec As String , Byref path as string)
   Dim As String filename = Dir(path & filespec) ' Start a file search with the specified filespec/attrib *AND* get the first filename.
   Do While Len(filename) > 0                    ' If len(filename) is 0, exit the loop: no more filenames are left to be read.
      DEBUG_FB("killing ... " & path & filename)
      kill path & filename
      filename = Dir()
   Loop
End Sub

Private Function ZgetTempPath() as String
   Dim         as Long bufflen
   Dim         as Long res
   Dim         as String tpath
   bufflen = GetTempPath(0 , StrPtr(tpath))
   tpath = Space(bufflen)
   res = GetTempPath(bufflen , StrPtr(tpath))
   tpath = RTrim(tpath)
   Return tpath
End Function

private function temp_dir_manage(byval flag as long = 0) as string
   Dim as String tpath = ZgetTempPath() & "temp_ntdll"
	DEBUG_FB(tpath)
   if flag = 0 THEN
      if Dir(tpath , &h10) <> "" THEN
         kill_files( "*.*" , tpath & "\")
      else
         mkdir tpath
      END IF
   elseif flag = 1 THEN
      if Dir(tpath , &h10) <> "" THEN
         kill_files( "*.*" , tpath & "\")
			chdir exepath()
         rmdir tpath & "\" 
      END IF
		tpath = ""
   END IF
   return tpath
end function


Private Function get_FileName (ByRef Src As String) As String
    Dim x As Long
    x = InStrrev( Src, Any ":/\")
    If x Then
        Function = Mid(Src, x + 1)
    Else
        Function = Src
    End If
End Function


Private Function File_Ubyte(byref Nom_File as String, Byref fileData as UByte Ptr) as long
    Dim  as Long Filesize, result
    Dim  as Long myHandle
    Dim as String cont
	 
    myHandle = Freefile()
	 Function = 0
    result = Open (Nom_File For Binary as #myHandle )
    If result <> 0 Then Exit Function
    Filesize = LOF(myHandle)
    If Filesize=0 Then
       Close #myHandle
       Exit Function
    End If
    fileData = Allocate(Filesize)
    Get #myHandle, 0, *fileData, Filesize
    Close #myHandle
	function = Filesize
End Function


Private sub Ubyte_File(Byref fileData as UByte Ptr, Byval Filesize as Long, byref Nom_File as String)
    Dim  as Long result
    Dim  as Integer myHandle
    Dim as String cont

	 myHandle = Freefile()
    result = Open (Nom_File For Binary Access write as #myHandle )
    If result <> 0 Then Exit sub
	 If Filesize = 0 Then
		Close #myHandle 
		exit sub
	 end if
    put #myHandle, , *fileData, Filesize
    Close #myHandle
End sub

private function getStrings( ub() as ubyte, byval nb as long)as string
	dim as string st1 
	dim x as long

	for x = 0 to nb
		if ub(x)= 0 THEN exit for
		st1 &= chr(ub(x))
	NEXT
	return st1
END FUNCTION

private function my_UnZip( byref source as string, byref dest as string) as long
	dim as ULong rVal
	dim as Ulong index
	dim as ushort ipos
	dim as ULong icount
	dim as long ff1
	dim fileData as UByte Ptr
	dim as ubyte ilen
	dim as string stemp
	dim as ubyte ub2()
	
	dim as string st1 = temp_dir_manage(0) & "\"
	function = -1
	if source = "" THEN exit function
	if dest <> ""  and right(dest,1) <> "\" and right(dest,1) <> "/" THEN dest &= "\"
	ff1 = freefile
	If Open(source For Binary Access read As #ff1) = 0 Then
		DEBUG_FB( "Successfully opened file to decompress")
		redim ub2(6) 
		icount = 1
		get #ff1,icount, ub2()
		stemp = getStrings(ub2(), 6)
		if stemp <> "MYPAR" THEN 
			Close #ff1
			 st1 = temp_dir_manage(1)
			exit function
      END IF
		DEBUG_FB(stemp)
		icount += 6
		get #ff1,icount,ipos
		DEBUG_FB(ipos)
		dim as ulong comp_size(ipos)
		dim as ulong uncomp_size(ipos)
		dim as string files(ipos)
		icount +=2
		for index = 0 to ipos -1 
			get #ff1,icount, comp_size(index)
			DEBUG_FB(comp_size(index))
			icount +=4
			get #ff1, icount, uncomp_size(index)
			DEBUG_FB(uncomp_size(index))
			icount +=4
			get #ff1,icount, ilen
			DEBUG_FB(ilen)
			icount +=1
			redim ub2(ilen)
			get #ff1,icount, ub2()
			DEBUG_FB("icount = " & icount)
			files(index)= getStrings(ub2(), ilen)
			DEBUG_FB("<" & files(index)& ">")
			icount += ilen
      NEXT
		for index = 0 to ipos -1
			if comp_size(index) THEN
				filedata = allocate(comp_size(index))
				get #ff1,icount, *filedata, comp_size(index) 
				icount += comp_size(index)
				Ubyte_File(fileData, comp_size(index), st1 & files(index) & ".LZN")
				deallocate(filedata)
				filedata = 0
				rVal = NTDecompressFile(st1 & files(index) & ".LZN" , dest & files(index), uncomp_size(index)+20)
				if rVal THEN
					ipos = 0
					exit for
            END IF
			else
				Ubyte_File(0, 0, dest & files(index) )	
         END IF
		next
		Close #ff1
	Else
		ipos = 0
	End If

	DEBUG_FB("nb files uncompressed = " & ipos)
	st1 = temp_dir_manage(1)
	return ipos	
end function

private function my_Zip(byref liste as string, byref dest as string, _
								Byval engine as ULong = COMPRESSION_ENGINE_STANDARD) as long
	if liste = "" or dest = ""  THEN return 0
	kill dest
	dim as string fname
	dim as string file
	dim as long ipos
	dim as string st1
	dim as ULong rVal
	redim as ulong comp_size(8)
	redim as ulong uncomp_size(8)
	redim as string files(8)
	dim as long index = 8
	dim as long ff1
	dim fileData as UByte Ptr
	dim as long ilen
	dim as ubyte ub1 = 0
	do
		fname = Parse_names(liste , _FILE_SEPARATOR_ , ipos + 1)
		if ipos = 0 and fname <> "" THEN st1 = temp_dir_manage() & "\"
		if fname <> "" THEN 
			DEBUG_FB("fname = " & fname)
			file = get_FileName(fname)
			DEBUG_FB(st1 & file & ".LZN")
			rVal = NTCompressFile(fname , st1 & file & ".LZN", engine)
			'MsgStatus() 
			if rVal THEN
				st1 = temp_dir_manage(1)
				return 0
         END IF
			if ipos = index THEN 
				index *= 2
				redim preserve comp_size(index)
				redim preserve uncomp_size(index)
				redim preserve files(index)
         END IF
			comp_size(ipos) = gCollect.comp_size	
			uncomp_size(ipos) = gCollect.uncomp_size
			files(ipos) = file 
			ipos +=1
		end if
	loop while fname <> ""
	ff1 = freefile
	If Open(dest For Binary Access Write As #ff1) = 0 Then
		DEBUG_FB( "Successfully opened file")
		put #ff1,,"MYPAR"
		put #ff1,,ub1 
		put #ff1,,cast(ushort,ipos)
		
		for index = 0 to ipos -1
			put #ff1,,comp_size(index)
			put #ff1,,uncomp_size(index)
			ilen = len(files(index))+1
			put #ff1,,cast(ubyte,ilen)
			put #ff1,,files(index)
			put #ff1,,ub1 			
      NEXT
		for index = 0 to ipos -1
			ilen = File_Ubyte(st1 & files(index) & ".LZN", fileData)
			if fileData THEN
				put #ff1,,*fileData, ilen
				deallocate(filedata)
				filedata = 0
         END IF
		next
		Close #ff1
	Else
		ipos = 0
	End If
	st1 = temp_dir_manage(1)
	DEBUG_FB("nb files compressed = " & ipos)
	return ipos	
end function



'*******************************************************************************************
'test code  if you want multiple files use a |  as separator between each file
'       the files can be from different path  you only have to give the full path of each file
'       if you use file to compress on the same folder, just use name.ext

 my_Zip("dlltool64.exe|dlltool32.exe|def-editor.exe|gendef.exe|as32.exe|as64.exe", "my_full_zip.par")
 
 'to unzip, give the path/name of the file to expand and the folder where to put the extracted files

my_UnZip("my_full_zip.par", exepath() & "\get_info\")

the info stored in the compressed file is obviouly the compressed info
+ some little header including size compressed, size uncompressed, name of file

thats all

hope that can help some of you

NTDLL buffer compression vs ZLIB

NTDLL buffer compression vs ZLIB

Re: NTDLL buffer compression vs ZLIB

Re: NTDLL buffer compression