Routines to write syntax highlighted code

[Munair]

Thank you for the test sancho3. Otherwise I would have done it myself.

[paul doe]

Thats 10x slower for the instr method.

With a more optimized implementation the difference is about 3x:

Code: Select all

function tokenizeString( _
	byref inputString as string, _
	byref delimiters as string, _
	tokens() as string, _
	byval returnDelimiters as boolean = true ) as uinteger
	
	/'
		Tokenizes a string based on specified delimiter characters. Returns
		the token count, and the spliced tokens in the tokens() array 
	'/
	'' trivial reject
	if( len( inputString ) = 0 ) then
		return( 0 )
	end if
	
	dim as uinteger strLen = len( inputString )
	dim as boolean tally( 1 to strLen )
	dim as uinteger delimiterCount, nonDelimiterCount, count
	
	dim as uinteger position = 1
	dim as uinteger p
	
	'' first pass: tally the delimiters	
	do
		p = instr( position, inputString, any delimiters )
		
		if( p > 0 ) then
			tally( p ) = true
			
			delimiterCount += 1
			
			if( p > 1 ) then
				'' not a delimiter
				if( tally( p - 1 ) = false ) then
					nonDelimiterCount += 1
				end if
			end if
					
			position = p + 1
		else
			exit do
		end if
	loop
		
	'' if the last token wasn't a delimiter, add one to the tokens count
	'' ( to account for the last token )
	if( tally( strLen ) = false ) then
		nonDelimitercount += 1
	end if
	
	if( returnDelimiters = true ) then
		count = delimiterCount + nonDelimiterCount
	else
		count = nonDelimiterCount
	end if
	
	'' allocate space for returned tokens
	redim tokens( 0 to count - 1 )
	
	dim as uinteger current = 0
	position = 1
	
	'' second pass: splice the tokens
	if( returnDelimiters = true ) then
		'' return delimiters
		for i as uinteger = 1 to strLen
			if( tally( i ) = true ) then
				if( i > 1 ) then
					if( tally( i - 1 ) = false ) then
						'' splice the token
						tokens( current ) = mid( inputString, position, i - position )
						current += 1
					end if
				end if
				
				'' splice the delimiter
				tokens( current ) =	chr( inputString[ i - 1 ] )
				current += 1
			
				position = i + 1
			end if
		next
	else
		'' don't return delimiters
		for i as uinteger = 1 to strLen
			if( tally( i ) = true ) then
				if( i > 1 ) then
					if( tally( i - 1 ) = false ) then
						'' splice the token
						tokens( current ) = mid( inputString, position, i - position )
						current += 1
					end if
				end if
			
				position = i + 1
			end if
		next
	end if
	
	'' if the last char wasn't a delimiter, splice the last token too
	if( position < strLen ) then
		tokens( current ) =	mid( inputString, position, strLen - position + 1 )
	end if
	
	return( count )
end function

dim as string commands = "abs,access,acos,alias,allocate,alpha,andalso,and,any,append,asc,as,asin,asm,assertwarn,assert,atan2,atn,base,beep,binary,bin,bitset,bitreset,bit,bload,bsave,byref,byte,byval,callocate,call,case,cast,cbyte,cdbl,cdecl,chain,chdir,chr,cint,circle," + _ 
"class,clear,clngint,clng,close,cls,color,,command,common,condbroadcast,condcreate,conddestroy,condsignal,condwait,constructor,const,cons,continue,cos,cptr,cshort,csign,csng,csrlin,cubyte,cuint,culngint,culng,cunsg,curdir,cushort,custom,cvd,cvi,cvl,cvlongint," + _ 
"cvs,cvshort,data,datevalue,dateadd,datediff,datepart,dateserial,date,day,deallocate,declare,defbyte,defdbl,defint,deflngint,deflng,defshort,defsng,defstr,defubyte,defuint,defulngint,defushort,delete,destructor,dim,dir,do,double,draw,dylibfree,dylibload,dylibsymbol," + _ 
"dynamic,elseif,else,encoding,endif,end,enum,environ,eof,eqv,erase,erfn,erl,ermn,err,error,escape,exec,exepath,exit,exp,explicit,export,extends,extern,false,fboolean,field,fileattr,filecopy,filedatetime,fileexists,filelen,fix,flip,format,for,frac,freefile," + _ 
"fre,function,getmouse,getjoystick,getkey,get,gosub,goto,hex,hibyte,hiword,hour,if,iif,imageconvertrow,imagecreate,imagedestroy,import,imp,inkey,input,input,inp,instrrev,instr,integer,int,interface,is,isdate,kill,lbound,lcase,left,len,let,lib,line,lobyte,loc," + _ 
"local,locate,lock,lof,log,long,longint,loop,loword,lpos,lprint,lpt,lset,ltrim,mid,minute,mkd,mkdir,mki,mkl,mklongint,mks,mkshort,mod,month,monthname,multikey,mutexcreate,mutexdestroy,mutexlock,mutexunlock,name,namespace,new,next,nokeyword,not,now,object,oct," + _ 
"offsetof,once,on,open,operator,option,orelse,or,output,out,overload,paint,palette,pascal,pcopy,peek,pipe,pmap,pointer,point,poke,pos,preserve,preset,print,private,procptr,property,protected,pset,ptr,public,put,randomize,random,read,reallocate,redim,rem,reset," + _ 
"restore,resume,return,rgba,rgb,right,rmdir,rnd,rset,rtrim,run,sadd,scope,screenunlock,screencontrol,screencopy,screenevent,screenglproc,screeninfo,screenlist,screenlock,screenptr,screenres,screenset,screensync,screen,scrn,second,seek,select,setdate,setenviron," + _ 
"setmouse,settime,sgn,shared,shell,shl,short,shr,sin,single,sizeof,sleep,space,spc,sqr,static,stdcall,step,stop,string,strptr,str,sub,swap,system,tab,tan,then,this,threadcreate,threadwait,timer,time,timeserial,timevalue,to,trans,trim,true,type,ubound,ubyte," + _ 
"ucase,uinteger,ulongint,ulong,union,unlock,unsigned,until,ushort,using,va,arg,va,first,valulng,valuint,vallng,valint,val,va,next,varptr,var,view,virtual,wait,wbin,wchr,weekdayname,weekday,wend,whex,while,width,windowtitle,window,winput,with,woct,write,wspace" + _
"#assert,#defined,#define,#elseif,#else,#endif,#endmacro,#error,#ifndef,#ifdef,#if,#inclib,#include,#lang,#libpath,#line,#macro,#once,#pragma,#print"

dim as integer numTokens = 414
dim as string tokens()
dim as string tokens2( 0 to numTokens - 1 )
dim as integer time1, time2

dim as double t, sum
dim as integer count = 100000

? "with tokenization"
for i as integer = 0 to count - 1
	t = timer()
	tokenizeString( commands, ",", tokens(), false )
	t = timer() - t
		
	sum += t
next

time1 = int( 1 / ( sum / count ) )
? time1

sum = 0.0

? "with 'data'"
for i as integer = 0 to count - 1
	t = timer()
	restore commandList
	
	for j as integer = 0 to numTokens - 1
		read tokens2( j )
	next
	t = timer() - t
	
	sum += t
next

time2 = int( 1 / ( sum / count ) ) 
? time2

?
? "tokenization:"

for i as integer = 0 to numTokens - 1
	? tokens( i ); " ";
next

?
?
? "data:"

for i as integer = 0 to numTokens - 1
	? tokens2( i ); " ";
next

?
? 
? "difference: "; time2 / time1

sleep()

commandList:
data "abs","access","acos","alias","allocate","alpha","andalso","and","any","append","asc","as","asin","asm","assertwarn","assert","atan2","atn","base","beep","binary","bin","bitset","bitreset","bit","bload","bsave","byref","byte","byval","callocate","call"
data "case","cast","cbyte","cdbl","cdecl","chain","chdir","chr","cint","circle","class","clear","clngint","clng","close","cls","color","","command","common","condbroadcast","condcreate","conddestroy","condsignal","condwait","constructor","const","cons","continue"
data "cos","cptr","cshort","csign","csng","csrlin","cubyte","cuint","culngint","culng","cunsg","curdir","cushort","custom","cvd","cvi","cvl","cvlongint","cvs","cvshort","data","datevalue","dateadd","datediff","datepart","dateserial","date","day","deallocate"
data "declare","defbyte","defdbl","defint","deflngint","deflng","defshort","defsng","defstr","defubyte","defuint","defulngint","defushort","delete","destructor","dim","dir","do","double","draw","dylibfree","dylibload","dylibsymbol","dynamic","elseif","else"
data "encoding","endif","end","enum","environ","eof","eqv","erase","erfn","erl","ermn","err","error","escape","exec","exepath","exit","exp","explicit","export","extends","extern","false","fboolean","field","fileattr","filecopy","filedatetime","fileexists"
data "filelen","fix","flip","format","for","frac","freefile","fre","function","getmouse","getjoystick","getkey","get","gosub","goto","hex","hibyte","hiword","hour","if","iif","imageconvertrow","imagecreate","imagedestroy","import","imp","inkey","input","input"
data "inp","instrrev","instr","integer","int","interface","is","isdate","kill","lbound","lcase","left","len","let","lib","line","lobyte","loc","local","locate","lock","lof","log","long","longint","loop","loword","lpos","lprint","lpt","lset","ltrim","mid","minute"
data "mkd","mkdir","mki","mkl","mklongint","mks","mkshort","mod","month","monthname","multikey","mutexcreate","mutexdestroy","mutexlock","mutexunlock","name","namespace","new","next","nokeyword","not","now","object","oct","offsetof","once","on","open","operator"
data "option","orelse","or","output","out","overload","paint","palette","pascal","pcopy","peek","pipe","pmap","pointer","point","poke","pos","preserve","preset","print","private","procptr","property","protected","pset","ptr","public","put","randomize","random"
data "read","reallocate","redim","rem","reset","restore","resume","return","rgba","rgb","right","rmdir","rnd","rset","rtrim","run","sadd","scope","screenunlock","screencontrol","screencopy","screenevent","screenglproc","screeninfo","screenlist","screenlock"
data "screenptr","screenres","screenset","screensync","screen","scrn","second","seek","select","setdate","setenviron","setmouse","settime","sgn","shared","shell","shl","short","shr","sin","single","sizeof","sleep","space","spc","sqr","static","stdcall","step"
data "stop","string","strptr","str","sub","swap","system","tab","tan","then","this","threadcreate","threadwait","timer","time","timeserial","timevalue","to","trans","trim","true","type","ubound","ubyte","ucase","uinteger","ulongint","ulong","union","unlock"
data "unsigned","until","ushort","using","va","arg","va","first","valulng","valuint","vallng","valint","val","va","next","varptr","var","view","virtual","wait","wbin","wchr","weekdayname","weekday","wend","whex","while","width","windowtitle","window","winput"
data "with","woct","write","wspace","wstring","wstr","xor","year","zstring","#assert","#defined","#define","#elseif","#else","#endif","#endmacro","#error","#ifndef","#ifdef","#if","#inclib","#include","#lang","#libpath","#line","#macro","#once","#pragma","#print"

Still, speed alone shouldn't be the only concern in every scenario. For this particular one, it's far better to define a 'language file' and pass that to the syntax coloring routine. That way you could modify/add keywords if you forgot them, or add them if the language changes, or even support other languages, without changing a single line of code. That's how most editors, like FBEdit and Jellyfish, manage this sort of thing.

@IchMagBier: nice work, mate.

PS: DATA? Seriously?

[Munair]

Quite seriously. Not just because it's faster, but because it is also easy to maintain when stored as include files, which only requires a recompilation, which happens all the time anyway because of fixes and improvements. Loading separate files is slower and also more prone to corruption.

However, both are legitimate ways and it is generally a programmer's preference. So yes, seriously.

[paul doe]

Faster still:

Code: Select all

dim as string tokens( ... ) = _
{ _
"abs","access","acos","alias","allocate","alpha","andalso","and","any","append","asc","as","asin","asm","assertwarn","assert","atan2","atn","base","beep","binary","bin","bitset","bitreset","bit","bload","bsave","byref","byte","byval","callocate","call" _
"case","cast","cbyte","cdbl","cdecl","chain","chdir","chr","cint","circle","class","clear","clngint","clng","close","cls","color","","command","common","condbroadcast","condcreate","conddestroy","condsignal","condwait","constructor","const","cons","continue" _
"cos","cptr","cshort","csign","csng","csrlin","cubyte","cuint","culngint","culng","cunsg","curdir","cushort","custom","cvd","cvi","cvl","cvlongint","cvs","cvshort","data","datevalue","dateadd","datediff","datepart","dateserial","date","day","deallocate" _
"declare","defbyte","defdbl","defint","deflngint","deflng","defshort","defsng","defstr","defubyte","defuint","defulngint","defushort","delete","destructor","dim","dir","do","double","draw","dylibfree","dylibload","dylibsymbol","dynamic","elseif","else" _
"encoding","endif","end","enum","environ","eof","eqv","erase","erfn","erl","ermn","err","error","escape","exec","exepath","exit","exp","explicit","export","extends","extern","false","fboolean","field","fileattr","filecopy","filedatetime","fileexists" _
"filelen","fix","flip","format","for","frac","freefile","fre","function","getmouse","getjoystick","getkey","get","gosub","goto","hex","hibyte","hiword","hour","if","iif","imageconvertrow","imagecreate","imagedestroy","import","imp","inkey","input","input" _
"inp","instrrev","instr","integer","int","interface","is","isdate","kill","lbound","lcase","left","len","let","lib","line","lobyte","loc","local","locate","lock","lof","log","long","longint","loop","loword","lpos","lprint","lpt","lset","ltrim","mid","minute" _
"mkd","mkdir","mki","mkl","mklongint","mks","mkshort","mod","month","monthname","multikey","mutexcreate","mutexdestroy","mutexlock","mutexunlock","name","namespace","new","next","nokeyword","not","now","object","oct","offsetof","once","on","open","operator" _
"option","orelse","or","output","out","overload","paint","palette","pascal","pcopy","peek","pipe","pmap","pointer","point","poke","pos","preserve","preset","print","private","procptr","property","protected","pset","ptr","public","put","randomize","random" _
"read","reallocate","redim","rem","reset","restore","resume","return","rgba","rgb","right","rmdir","rnd","rset","rtrim","run","sadd","scope","screenunlock","screencontrol","screencopy","screenevent","screenglproc","screeninfo","screenlist","screenlock" _
"screenptr","screenres","screenset","screensync","screen","scrn","second","seek","select","setdate","setenviron","setmouse","settime","sgn","shared","shell","shl","short","shr","sin","single","sizeof","sleep","space","spc","sqr","static","stdcall","step" _
"stop","string","strptr","str","sub","swap","system","tab","tan","then","this","threadcreate","threadwait","timer","time","timeserial","timevalue","to","trans","trim","true","type","ubound","ubyte","ucase","uinteger","ulongint","ulong","union","unlock" _
"unsigned","until","ushort","using","va","arg","va","first","valulng","valuint","vallng","valint","val","va","next","varptr","var","view","virtual","wait","wbin","wchr","weekdayname","weekday","wend","whex","while","width","windowtitle","window","winput" _
"with","woct","write","wspace","wstring","wstr","xor","year","zstring","#assert","#defined","#define","#elseif","#else","#endif","#endmacro","#error","#ifndef","#ifdef","#if","#inclib","#include","#lang","#libpath","#line","#macro","#once","#pragma","#print" _
}

for i as integer = 0 to ubound( tokens )
	? tokens( i ); " ";
next

sleep()

But, as I already stated, it's inflexible. The OP's original intent was to highlight syntax for a variety of languages, and if you do it like this, you need to recompile the program every time you want to add another language's lexicon. Nothing wrong with it if you're debugging the app, but it shouldn't become stuck in debug mode forever.

[sancho3]

A language/keyword file is the way to go. You get to take advantage of the many editing tools available in a text editor. You get portability. You don't have to release your code.
Just as a side note the keyword lists used here are incomplete. I notice that none of the boolean keywords are in it.
I think the one I use in my project is incomplete as well.
IIRC you can find the complete list of keywords in the source code files for FB.

Faster still:

Its not faster.

Code: Select all

dim as double atime, btime, ctime, dtime
dim as string array()

atime = timer
redim array(9)
for n as integer = 1 to 1000
	for x as integer = 0 to 9
		read array(x)
	next
next
btime = timer - atime

ctime = timer 
for x as integer = 1 to 1000
	dim as string v(...) = {"1","2","3", "4", "5", "6", "7", "8", "9", "0"}
next
dtime = timer - ctime

print btime , dtime

data "1", "2", "3", "4", "5", "6", "7", "8", "9", "0"

I tried it on xp and on ubuntu and in opposite order and it never won once.

Code: Select all

0.0003439908614382148       0.0009779528481885791

[IchMagBier]

I took the keyword-list out of geany's file for FreeBasic. Notice how some of the OOP-stuff is missing aswell ("Virtual" or "Abstract" for example).

[paul doe]

Its not faster.

Perhaps. It's about 1.6 times slower to populate the array, according to my tests. But it's faster and simpler to 'set up' than the restore-read crap (it's also more elegant, but that's just my preference). However, be aware that you aren't really doing fair comparisons. You're also measuring the creation and destruction of the array (since it's included in a for-next loop and, thus, into it's own scope), which is incorrect:

Code: Select all

'ctime = timer
dim as double sum, t
for x as integer = 1 to 1000
   t = timer()
   dim as string v(...) = {"1","2","3", "4", "5", "6", "7", "8", "9", "0"}
   t = timer() - t
   sum += t
next
t = sum / 1000
'dtime = timer - ctime

Is more like it.

The very fastest way would simply be:

Code: Select all

dim as integer numTokens = 414
dim as string tokens( 0 to numTokens - 1 ) 
dim as string tokens2( 0 to numTokens - 1 )
dim as integer time1, time2

dim as double t, sum
dim as integer count = 100000

? "with initialization"
for i as integer = 0 to count - 1
	t = timer()
	tokens(0)="abs"
	tokens(1)="access"
	tokens(2)="acos"
	tokens(3)="alias"
	tokens(4)="allocate"
	tokens(5)="alpha"
	tokens(6)="andalso"
	tokens(7)="and"
	tokens(8)="any"
	tokens(9)="append"
	tokens(10)="asc"
	tokens(11)="as"
	tokens(12)="asin"
	tokens(13)="asm"
	tokens(14)="assertwarn"
	tokens(15)="assert"
	tokens(16)="atan2"
	tokens(17)="atn"
	tokens(18)="base"
	tokens(19)="beep"
	tokens(20)="binary"
	tokens(21)="bin"
	tokens(22)="bitset"
	tokens(23)="bitreset"
	tokens(24)="bit"
	tokens(25)="bload"
	tokens(26)="bsave"
	tokens(27)="byref"
	tokens(28)="byte"
	tokens(29)="byval"
	tokens(30)="callocate"
	tokens(31)="call"
	tokens(32)="case"
	tokens(33)="cast"
	tokens(34)="cbyte"
	tokens(35)="cdbl"
	tokens(36)="cdecl"
	tokens(37)="chain"
	tokens(38)="chdir"
	tokens(39)="chr"
	tokens(40)="cint"
	tokens(41)="circle"
	tokens(42)="class"
	tokens(43)="clear"
	tokens(44)="clngint"
	tokens(45)="clng"
	tokens(46)="close"
	tokens(47)="cls"
	tokens(48)="color"
	tokens(50)="command"
	tokens(51)="common"
	tokens(52)="condbroadcast"
	tokens(53)="condcreate"
	tokens(54)="conddestroy"
	tokens(55)="condsignal"
	tokens(56)="condwait"
	tokens(57)="constructor"
	tokens(58)="const"
	tokens(59)="cons"
	tokens(60)="continue"
	tokens(61)="cos"
	tokens(62)="cptr"
	tokens(63)="cshort"
	tokens(64)="csign"
	tokens(65)="csng"
	tokens(66)="csrlin"
	tokens(67)="cubyte"
	tokens(68)="cuint"
	tokens(69)="culngint"
	tokens(70)="culng"
	tokens(71)="cunsg"
	tokens(72)="curdir"
	tokens(73)="cushort"
	tokens(74)="custom"
	tokens(75)="cvd"
	tokens(76)="cvi"
	tokens(77)="cvl"
	tokens(78)="cvlongint"
	tokens(79)="cvs"
	tokens(80)="cvshort"
	tokens(81)="data"
	tokens(82)="datevalue"
	tokens(83)="dateadd"
	tokens(84)="datediff"
	tokens(85)="datepart"
	tokens(86)="dateserial"
	tokens(87)="date"
	tokens(88)="day"
	tokens(89)="deallocate"
	tokens(90)="declare"
	tokens(91)="defbyte"
	tokens(92)="defdbl"
	tokens(93)="defint"
	tokens(94)="deflngint"
	tokens(95)="deflng"
	tokens(96)="defshort"
	tokens(97)="defsng"
	tokens(98)="defstr"
	tokens(99)="defubyte"
	tokens(100)="defuint"
	tokens(101)="defulngint"
	tokens(102)="defushort"
	tokens(103)="delete"
	tokens(104)="destructor"
	tokens(105)="dim"
	tokens(106)="dir"
	tokens(107)="do"
	tokens(108)="double"
	tokens(109)="draw"
	tokens(110)="dylibfree"
	tokens(111)="dylibload"
	tokens(112)="dylibsymbol"
	tokens(113)="dynamic"
	tokens(114)="elseif"
	tokens(115)="else"
	tokens(116)="encoding"
	tokens(117)="endif"
	tokens(118)="end"
	tokens(119)="enum"
	tokens(120)="environ"
	tokens(121)="eof"
	tokens(122)="eqv"
	tokens(123)="erase"
	tokens(124)="erfn"
	tokens(125)="erl"
	tokens(126)="ermn"
	tokens(127)="err"
	tokens(128)="error"
	tokens(129)="escape"
	tokens(130)="exec"
	tokens(131)="exepath"
	tokens(132)="exit"
	tokens(133)="exp"
	tokens(134)="explicit"
	tokens(135)="export"
	tokens(136)="extends"
	tokens(137)="extern"
	tokens(138)="false"
	tokens(139)="fboolean"
	tokens(140)="field"
	tokens(141)="fileattr"
	tokens(142)="filecopy"
	tokens(143)="filedatetime"
	tokens(144)="fileexists"
	tokens(145)="filelen"
	tokens(146)="fix"
	tokens(147)="flip"
	tokens(148)="format"
	tokens(149)="for"
	tokens(150)="frac"
	tokens(151)="freefile"
	tokens(152)="fre"
	tokens(153)="function"
	tokens(154)="getmouse"
	tokens(155)="getjoystick"
	tokens(156)="getkey"
	tokens(157)="get"
	tokens(158)="gosub"
	tokens(159)="goto"
	tokens(160)="hex"
	tokens(161)="hibyte"
	tokens(162)="hiword"
	tokens(163)="hour"
	tokens(164)="if"
	tokens(165)="iif"
	tokens(166)="imageconvertrow"
	tokens(167)="imagecreate"
	tokens(168)="imagedestroy"
	tokens(169)="import"
	tokens(170)="imp"
	tokens(171)="inkey"
	tokens(172)="input"
	tokens(173)="input"
	tokens(174)="inp"
	tokens(175)="instrrev"
	tokens(176)="instr"
	tokens(177)="integer"
	tokens(178)="int"
	tokens(179)="interface"
	tokens(180)="is"
	tokens(181)="isdate"
	tokens(182)="kill"
	tokens(183)="lbound"
	tokens(184)="lcase"
	tokens(185)="left"
	tokens(186)="len"
	tokens(187)="let"
	tokens(188)="lib"
	tokens(189)="line"
	tokens(190)="lobyte"
	tokens(191)="loc"
	tokens(192)="local"
	tokens(193)="locate"
	tokens(194)="lock"
	tokens(195)="lof"
	tokens(196)="log"
	tokens(197)="long"
	tokens(198)="longint"
	tokens(199)="loop"
	tokens(200)="loword"
	tokens(201)="lpos"
	tokens(202)="lprint"
	tokens(203)="lpt"
	tokens(204)="lset"
	tokens(205)="ltrim"
	tokens(206)="mid"
	tokens(207)="minute"
	tokens(208)="mkd"
	tokens(209)="mkdir"
	tokens(210)="mki"
	tokens(211)="mkl"
	tokens(212)="mklongint"
	tokens(213)="mks"
	tokens(214)="mkshort"
	tokens(215)="mod"
	tokens(216)="month"
	tokens(217)="monthname"
	tokens(218)="multikey"
	tokens(219)="mutexcreate"
	tokens(220)="mutexdestroy"
	tokens(221)="mutexlock"
	tokens(222)="mutexunlock"
	tokens(223)="name"
	tokens(224)="namespace"
	tokens(225)="new"
	tokens(226)="next"
	tokens(227)="nokeyword"
	tokens(228)="not"
	tokens(229)="now"
	tokens(230)="object"
	tokens(231)="oct"
	tokens(232)="offsetof"
	tokens(233)="once"
	tokens(234)="on"
	tokens(235)="open"
	tokens(236)="operator"
	tokens(237)="option"
	tokens(238)="orelse"
	tokens(239)="or"
	tokens(240)="output"
	tokens(241)="out"
	tokens(242)="overload"
	tokens(243)="paint"
	tokens(244)="palette"
	tokens(245)="pascal"
	tokens(246)="pcopy"
	tokens(247)="peek"
	tokens(248)="pipe"
	tokens(249)="pmap"
	tokens(250)="pointer"
	tokens(251)="point"
	tokens(252)="poke"
	tokens(253)="pos"
	tokens(254)="preserve"
	tokens(255)="preset"
	tokens(256)="print"
	tokens(257)="private"
	tokens(258)="procptr"
	tokens(259)="property"
	tokens(260)="protected"
	tokens(261)="pset"
	tokens(262)="ptr"
	tokens(263)="public"
	tokens(264)="put"
	tokens(265)="randomize"
	tokens(266)="random"
	tokens(267)="read"
	tokens(268)="reallocate"
	tokens(269)="redim"
	tokens(270)="rem"
	tokens(271)="reset"
	tokens(272)="restore"
	tokens(273)="resume"
	tokens(274)="return"
	tokens(275)="rgba"
	tokens(276)="rgb"
	tokens(277)="right"
	tokens(278)="rmdir"
	tokens(279)="rnd"
	tokens(280)="rset"
	tokens(281)="rtrim"
	tokens(282)="run"
	tokens(283)="sadd"
	tokens(284)="scope"
	tokens(285)="screenunlock"
	tokens(286)="screencontrol"
	tokens(287)="screencopy"
	tokens(288)="screenevent"
	tokens(289)="screenglproc"
	tokens(290)="screeninfo"
	tokens(291)="screenlist"
	tokens(292)="screenlock"
	tokens(293)="screenptr"
	tokens(294)="screenres"
	tokens(295)="screenset"
	tokens(296)="screensync"
	tokens(297)="screen"
	tokens(298)="scrn"
	tokens(299)="second"
	tokens(300)="seek"
	tokens(301)="select"
	tokens(302)="setdate"
	tokens(303)="setenviron"
	tokens(304)="setmouse"
	tokens(305)="settime"
	tokens(306)="sgn"
	tokens(307)="shared"
	tokens(308)="shell"
	tokens(309)="shl"
	tokens(310)="short"
	tokens(311)="shr"
	tokens(312)="sin"
	tokens(313)="single"
	tokens(314)="sizeof"
	tokens(315)="sleep"
	tokens(316)="space"
	tokens(317)="spc"
	tokens(318)="sqr"
	tokens(319)="static"
	tokens(320)="stdcall"
	tokens(321)="step"
	tokens(322)="stop"
	tokens(323)="string"
	tokens(324)="strptr"
	tokens(325)="str"
	tokens(326)="sub"
	tokens(327)="swap"
	tokens(328)="system"
	tokens(329)="tab"
	tokens(330)="tan"
	tokens(331)="then"
	tokens(332)="this"
	tokens(333)="threadcreate"
	tokens(334)="threadwait"
	tokens(335)="timer"
	tokens(336)="time"
	tokens(337)="timeserial"
	tokens(338)="timevalue"
	tokens(339)="to"
	tokens(340)="trans"
	tokens(341)="trim"
	tokens(342)="true"
	tokens(343)="type"
	tokens(344)="ubound"
	tokens(345)="ubyte"
	tokens(346)="ucase"
	tokens(347)="uinteger"
	tokens(348)="ulongint"
	tokens(349)="ulong"
	tokens(350)="union"
	tokens(351)="unlock"
	tokens(352)="unsigned"
	tokens(353)="until"
	tokens(354)="ushort"
	tokens(355)="using"
	tokens(356)="va"
	tokens(357)="arg"
	tokens(358)="va"
	tokens(359)="first"
	tokens(360)="valulng"
	tokens(361)="valuint"
	tokens(362)="vallng"
	tokens(363)="valint"
	tokens(364)="val"
	tokens(365)="va"
	tokens(366)="next"
	tokens(367)="varptr"
	tokens(368)="var"
	tokens(369)="view"
	tokens(370)="virtual"
	tokens(371)="wait"
	tokens(372)="wbin"
	tokens(373)="wchr"
	tokens(374)="weekdayname"
	tokens(375)="weekday"
	tokens(376)="wend"
	tokens(377)="whex"
	tokens(378)="while"
	tokens(379)="width"
	tokens(380)="windowtitle"
	tokens(381)="window"
	tokens(382)="winput"
	tokens(383)="with"
	tokens(384)="woct"
	tokens(385)="write"
	tokens(386)="wspace"
	tokens(387)="wstring"
	tokens(388)="wstr"
	tokens(389)="xor"
	tokens(390)="year"
	tokens(391)="zstring"
	tokens(392)="#assert"
	tokens(393)="#defined"
	tokens(394)="#define"
	tokens(395)="#elseif"
	tokens(396)="#else"
	tokens(397)="#endif"
	tokens(398)="#endmacro"
	tokens(399)="#error"
	tokens(400)="#ifndef"
	tokens(401)="#ifdef"
	tokens(402)="#if"
	tokens(403)="#inclib"
	tokens(404)="#include"
	tokens(405)="#lang"
	tokens(406)="#libpath"
	tokens(407)="#line"
	tokens(408)="#macro"
	tokens(409)="#once"
	tokens(410)="#pragma"
	tokens(411)="#print"
	tokens(412)="#typeof"
	tokens(413)="#undef"

	t = timer() - t
		
	sum += t
next

time1 = int( 1 / ( sum / count ) )
? time1

sum = 0.0

? "with 'data'"
for i as integer = 0 to count - 1
	restore commandList
	
	t = timer()
	
	for j as integer = 0 to numTokens - 1
		read tokens2( j )
	next
	
	t = timer() - t
	
	sum += t
next

time2 = int( 1 / ( sum / count ) ) 
? time2

?
? 
? "difference: "; time2 / time1

sleep()

commandList:
data "abs","access","acos","alias","allocate","alpha","andalso","and","any","append","asc","as","asin","asm","assertwarn","assert","atan2","atn","base","beep","binary","bin","bitset","bitreset","bit","bload","bsave","byref","byte","byval","callocate","call"
data "case","cast","cbyte","cdbl","cdecl","chain","chdir","chr","cint","circle","class","clear","clngint","clng","close","cls","color","","command","common","condbroadcast","condcreate","conddestroy","condsignal","condwait","constructor","const","cons","continue"
data "cos","cptr","cshort","csign","csng","csrlin","cubyte","cuint","culngint","culng","cunsg","curdir","cushort","custom","cvd","cvi","cvl","cvlongint","cvs","cvshort","data","datevalue","dateadd","datediff","datepart","dateserial","date","day","deallocate"
data "declare","defbyte","defdbl","defint","deflngint","deflng","defshort","defsng","defstr","defubyte","defuint","defulngint","defushort","delete","destructor","dim","dir","do","double","draw","dylibfree","dylibload","dylibsymbol","dynamic","elseif","else"
data "encoding","endif","end","enum","environ","eof","eqv","erase","erfn","erl","ermn","err","error","escape","exec","exepath","exit","exp","explicit","export","extends","extern","false","fboolean","field","fileattr","filecopy","filedatetime","fileexists"
data "filelen","fix","flip","format","for","frac","freefile","fre","function","getmouse","getjoystick","getkey","get","gosub","goto","hex","hibyte","hiword","hour","if","iif","imageconvertrow","imagecreate","imagedestroy","import","imp","inkey","input","input"
data "inp","instrrev","instr","integer","int","interface","is","isdate","kill","lbound","lcase","left","len","let","lib","line","lobyte","loc","local","locate","lock","lof","log","long","longint","loop","loword","lpos","lprint","lpt","lset","ltrim","mid","minute"
data "mkd","mkdir","mki","mkl","mklongint","mks","mkshort","mod","month","monthname","multikey","mutexcreate","mutexdestroy","mutexlock","mutexunlock","name","namespace","new","next","nokeyword","not","now","object","oct","offsetof","once","on","open","operator"
data "option","orelse","or","output","out","overload","paint","palette","pascal","pcopy","peek","pipe","pmap","pointer","point","poke","pos","preserve","preset","print","private","procptr","property","protected","pset","ptr","public","put","randomize","random"
data "read","reallocate","redim","rem","reset","restore","resume","return","rgba","rgb","right","rmdir","rnd","rset","rtrim","run","sadd","scope","screenunlock","screencontrol","screencopy","screenevent","screenglproc","screeninfo","screenlist","screenlock"
data "screenptr","screenres","screenset","screensync","screen","scrn","second","seek","select","setdate","setenviron","setmouse","settime","sgn","shared","shell","shl","short","shr","sin","single","sizeof","sleep","space","spc","sqr","static","stdcall","step"
data "stop","string","strptr","str","sub","swap","system","tab","tan","then","this","threadcreate","threadwait","timer","time","timeserial","timevalue","to","trans","trim","true","type","ubound","ubyte","ucase","uinteger","ulongint","ulong","union","unlock"
data "unsigned","until","ushort","using","va","arg","va","first","valulng","valuint","vallng","valint","val","va","next","varptr","var","view","virtual","wait","wbin","wchr","weekdayname","weekday","wend","whex","while","width","windowtitle","window","winput"
data "with","woct","write","wspace","wstring","wstr","xor","year","zstring","#assert","#defined","#define","#elseif","#else","#endif","#endmacro","#error","#ifndef","#ifdef","#if","#inclib","#include","#lang","#libpath","#line","#macro","#once","#pragma","#print"

But I wouldn't go bragging about it. It's almost as ugly as the read-restore thing.

[Munair]

I took the keyword-list out of geany's file for FreeBasic. Notice how some of the OOP-stuff is missing aswell ("Virtual" or "Abstract" for example).

Geany's list is incomplete. If you want the latest you can always go over the keyword list in the manual.

[jj2007]

It's about 1.6 times slower to populate the array, according to my tests

According to my tests, reading an array from a textfile takes about ONE millisecond. Per 30,000 strings, that is. The whole debate is a bit strange ;-)

[sancho3]

I took the keyword-list out of geany's file for FreeBasic. Notice how some of the OOP-stuff is missing aswell ("Virtual" or "Abstract" for example).

I have uploaded the complete (I think) list of keywords here.
They are in the format you are using:

Code: Select all

.Command(0) = "ABS":.Command(1) = "ABSTRACT":.Command(2) = "ACCESS"
.Command(4) = "ALIAS":.Command(5) = "AND":.Command(6) = "ANDALSO"

Edited to fix incomplete list.

[paul doe]

According to my tests, reading an array from a textfile takes about ONE millisecond. Per 30,000 strings, that is. The whole debate is a bit strange ;-)

And irrelevant for being boilerplate code, indeed =D

[IchMagBier]

@sancho
It's important that 'ABS' comes after 'ABSTRACT' in the list, because the routine would first find 'ABS' in the code and then completely ignores 'ABSTRACT'. Like this: ABSTRACT SUB

I updated the first post and added the following keywords to the list:

Code: Select all

abstract virtual boolean implemets naked override add threaddetach cbool deflongint defulongint imageinfo nogosub stick strig threadcall val64

[sancho3]

There might be a few more then.
AND ANDALSO
AS ASC ASM
CLNG CLNGINT
CONST CONSTRUCTOR
CULNG CULNGINT
CVS CVSSHORT
DO DOUBLE
ELSE ELSEIF
END ENDIF
ERR ERROR
IMP IMPLEMENTS IMPORT - implements isn't used but it is reserved
INSTR INSTRREV
LONG LONGINT
MKL MKLONGINT
NAME NAMESPACE
OR ORELSE
POINT POINTER
SIN SINGLE
STR STRING STRPTR
TYPE TYPEOF
VAR VARPTR
WSTR WSTRING

ERROR NOTICE:
Sorry my friend. I messed up in my code formatting and cut out lots of commands.
I updated the link in my previous post with the complete list of keywords. And above are the corrections needed to be made to work as you want.

[lizard]

@sancho
It's important that 'ABS' comes after 'ABSTRACT' in the list, because the routine would first find 'ABS' in the code and then completely ignores 'ABSTRACT'. Like this: ABSTRACT SUB

"AS" should change place with "ASIN"
"CULNG" should change place with "CULNGINT"
"DO" should change place with "DOUBLE"
"END" should change place with "ENDIF"
"ELSE" should change place with "ELSEIF"
"ERR" should change place with "ERROR"
"EXP" should change place with "EXPLICIT"

And so on. Or maybe better to search the table in reverse order Z - A.

[dodicat]

Here is the FBWiki : CatPgFullIndex today if it is any help.

Code: Select all

FBWiki : CatPgFullIndex



__DATE__
__DATE_ISO__
__FB_64BIT__
__FB_ARGC__
__FB_ARGV__
__FB_ARM__
__FB_ASM__
__FB_BACKEND__
__FB_BIGENDIAN__
__FB_BUILD_DATE__
__FB_CYGWIN__
__FB_DARWIN__
__FB_DEBUG__
__FB_DOS__
__FB_ERR__
__FB_FPMODE__
__FB_FPU__
__FB_FREEBSD__
__FB_GCC__
__FB_LANG__
__FB_LINUX__
__FB_MAIN__
__FB_MIN_VERSION__
__FB_MT__
__FB_NETBSD__
__FB_OPENBSD__
__FB_OPTION_BYVAL__
__FB_OPTION_DYNAMIC__
__FB_OPTION_ESCAPE__
__FB_OPTION_EXPLICIT__
__FB_OPTION_GOSUB__
__FB_OPTION_PRIVATE__
__FB_OUT_DLL__
__FB_OUT_EXE__
__FB_OUT_LIB__
__FB_OUT_OBJ__
__FB_PCOS__
__FB_SIGNATURE__
__FB_SSE__
__FB_UNIX__
__FB_VECTORIZE__
__FB_VER_MAJOR__
__FB_VER_MINOR__
__FB_VER_PATCH__
__FB_VERSION__
__FB_WIN32__
__FB_XBOX__
__FILE__
__FILE_NQ__
__FUNCTION__
__FUNCTION_NQ__
__LINE__
__PATH__
__TIME__

#ASSERT
#DEFINE
#ELSE
#ELSEIF
#ENDIF
#ENDMACRO
#ERROR
#IF
#IFDEF
#IFNDEF
#INCLIB
#INCLUDE
#LANG
#LIBPATH
#LINE
#MACRO
#PRAGMA
#PRINT
#UNDEF

$DYNAMIC
$INCLUDE
$LANG
$STATIC

ABS
ABSTRACT (member)
ACCESS
ACOS
ADD (Graphics PUT)
ALIAS
ALLOCATE
ALPHA (Graphics PUT)
AND
ANDALSO
AND (GRAPHICS PUT)
ANY
APPEND
AS
ASC
ASIN
ASM
ASSERT
ASSERTWARN
ATAN2
ATN

BASE (initialization)
BASE (member access)
BEEP
BIN
BINARY
BIT
BITRESET
BITSET
BLOAD
BOOLEAN
BSAVE
BYREF (parameters)
BYREF (function results)
BYREF (variables)
BYTE
BYVAL

CALL
CALLOCATE
CASE
CAST
CBOOL
CBYTE
CDBL
CDECL
CHAIN
CHDIR
CHR
CINT
CIRCLE
CLASS
CLEAR
CLNG
CLNGINT
CLOSE
CLS
COLOR
COMMAND
COMMON
CONDBROADCAST
CONDCREATE
CONDDESTROY
CONDSIGNAL
CONDWAIT
CONST
CONST (Member)
CONST (Qualifier)
CONSTRUCTOR
CONSTRUCTOR (Module)
CONTINUE
COS
CPTR
CSHORT
CSIGN
CSNG
CSRLIN
CUBYTE
CUINT
CULNG
CULNGINT
CUNSG
CURDIR
CUSHORT
CUSTOM (Graphics PUT)
CVD
CVI
CVL
CVLONGINT
CVS
CVSHORT

DATA
DATE
DATEADD
DATEDIFF
DATEPART
DATESERIAL
DATEVALUE
DAY
DEALLOCATE
DECLARE
DEFBYTE
DEFDBL
DEFINED
DEFINT
DEFLNG
DEFLONGINT
DEFSHORT
DEFSNG
DEFSTR
DEFUBYTE
DEFUINT
DEFULONGINT
DEFUSHORT
DELETE
DESTRUCTOR
DESTRUCTOR (Module)
DIM
DIR
DO
DO...LOOP
DOUBLE
DRAW
DRAW STRING
DYLIBFREE
DYLIBLOAD
DYLIBSYMBOL

ELSE
ELSEIF
ENCODING
END (Block)
END (Statement)
END IF
ENUM
ENVIRON statement
ENVIRON
EOF
EQV
ERASE
ERFN
ERL
ERMN
ERR
ERROR
EVENT (message data from ScreenEvent)
EXEC
EXEPATH
EXIT
EXP
EXPORT
EXTENDS
EXTERN
EXTERN...END EXTERN

FALSE
FIELD
FILEATTR
FILECOPY
FILEDATETIME
FILEEXISTS
FILELEN
FIX
FLIP
FOR
FOR...NEXT
FORMAT
FRAC
FRE
FREEFILE
FUNCTION
FUNCTION (Member)

GET (Graphics)
GET # (File I/O)
GETJOYSTICK
GETKEY
GETMOUSE
GOSUB
GOTO

HEX
HIBYTE
HIWORD
HOUR

IF...THEN
IIF
IMAGECONVERTROW
IMAGECREATE
IMAGEDESTROY
IMAGEINFO
IMP
IMPLEMENTS
IMPORT
INKEY
INP
INPUT (Statement)
INPUT (File I/O)
INPUT #
INPUT$
INSTR
INSTRREV
INT
INTEGER
IS (SELECT CASE)
IS (Run-time type information operator)
ISDATE
ISREDIRECTED


KILL

LBOUND
LCASE
LEFT
LEN
LET
LIB
LINE
LINE INPUT
LINE INPUT #
LOBYTE
LOC
LOCAL
LOCATE
LOCK
LOF
LOG
LONG
LONGINT
LOOP
LOWORD
LPOS
LPRINT
LSET
LTRIM

MID (Statement)
MID (Function)
MINUTE
MKD
MKDIR
MKI
MKL
MKLONGINT
MKS
MKSHORT
MOD
MONTH
MONTHNAME
MULTIKEY
MUTEXCREATE
MUTEXDESTROY
MUTEXLOCK
MUTEXUNLOCK

NAKED
NAME
NAMESPACE
NEW
NEW (Placement)
NEXT
NEXT (RESUME)
NOT
NOW

OBJECT
OCT
OFFSETOF
ON ERROR
ON...GOSUB
ON...GOTO
ONCE
OPEN
OPEN COM
OPEN CONS
OPEN ERR
OPEN LPT
OPEN PIPE
OPEN SCRN
OPERATOR
OPTION()
OPTION BASE
OPTION BYVAL
OPTION DYNAMIC
OPTION ESCAPE
OPTION EXPLICIT
OPTION GOSUB
OPTION NOGOSUB
OPTION NOKEYWORD
OPTION PRIVATE
OPTION STATIC
OR
OR (GRAPHICS PUT)
ORELSE
OUT
OUTPUT
OVERLOAD
OVERRIDE

PAINT
PALETTE
PASCAL
PCOPY
PEEK
PMAP
POINT
POINTCOORD
POINTER
POKE
POS
PRESERVE
PRESET
PRINT
?
PRINT #
? #
PRINT USING
? USING
PRIVATE
PRIVATE: (Access Control)
PROCPTR
PROPERTY
PROTECTED: (Access Control)
PSET (Statement)
PSET (Graphics PUT)
PTR
PUBLIC
PUBLIC: (Access Control)
PUT (Graphics)
PUT # (File I/O)

RANDOM
RANDOMIZE
READ
READ (File Access)
READ WRITE (File Access)
REALLOCATE
REDIM
REM
RESET
RESTORE
RESUME
RESUME NEXT
RETURN
RGB
RGBA
RIGHT
RMDIR
RND
RSET
RTRIM
RUN

SADD
SCOPE
SCREEN
SCREEN (Console)
SCREENCOPY
SCREENCONTROL
SCREENEVENT
SCREENGLPROC
SCREENINFO
SCREENLIST
SCREENLOCK
SCREENPTR
SCREENRES
SCREENSET
SCREENSYNC
SCREENUNLOCK
SECOND
SEEK (Statement)
SEEK (Function)
SELECT CASE
SETDATE
SETENVIRON
SETMOUSE
SETTIME
SGN
SHARED
SHELL
SHL
SHORT
SHR
SIN
SINGLE
SIZEOF
SLEEP
SPACE
SPC
SQR
STATIC
STATIC (Member)
STDCALL
STEP
STICK
STOP
STR
STRIG
STRING (Function)
STRING
STRPTR
SUB
SUB (Member)
SWAP
SYSTEM

TAB
TAN
THEN
THIS
THREADCALL
THREADCREATE
THREADDETACH
THREADWAIT
TIME
TIMER
TIMESERIAL
TIMEVALUE
TO
TRANS (Graphics PUT)
TRIM
TRUE
TYPE (Alias)
TYPE (Temporary)
TYPE (UDT)
TYPEOF

UBOUND
UBYTE
UCASE
UINTEGER
ULONG
ULONGINT
UNION
UNLOCK
UNSIGNED
UNTIL
USHORT
USING (PRINT)
USING (Namespaces)

VA_ARG
VA_FIRST
VA_NEXT
VAL
VALLNG
VALINT
VALUINT
VALULNG
VAR
VARPTR
VIEW PRINT
VIEW (Graphics)
VIRTUAL (member)

WAIT
WBIN
WCHR
WEEKDAY
WEEKDAYNAME
WEND
WHILE
WHILE...WEND
WHEX
WIDTH
WINDOW
WINDOWTITLE
WINPUT
WITH
WOCT
WRITE
WRITE #
WRITE (File Access)
WSPACE
WSTR
WSTRING (Data Type)
WSTRING (Function)

XOR
XOR (GRAPHICS PUT)

YEAR

ZSTRING







[Source]
[History]
2017-08-16 15:58:39
Owner: SysOp
Your hostname is ...

Valid XHTML :: Valid CSS: :: Powered by WikkaWiki