Routines to write syntax highlighted code
Re: Routines to write syntax highlighted code
Thank you for the test sancho3. Otherwise I would have done it myself.
Re: Routines to write syntax highlighted code
With a more optimized implementation the difference is about 3x:sancho3 wrote:Thats 10x slower for the instr method.
Code: Select all
function tokenizeString( _
byref inputString as string, _
byref delimiters as string, _
tokens() as string, _
byval returnDelimiters as boolean = true ) as uinteger
/'
Tokenizes a string based on specified delimiter characters. Returns
the token count, and the spliced tokens in the tokens() array
'/
'' trivial reject
if( len( inputString ) = 0 ) then
return( 0 )
end if
dim as uinteger strLen = len( inputString )
dim as boolean tally( 1 to strLen )
dim as uinteger delimiterCount, nonDelimiterCount, count
dim as uinteger position = 1
dim as uinteger p
'' first pass: tally the delimiters
do
p = instr( position, inputString, any delimiters )
if( p > 0 ) then
tally( p ) = true
delimiterCount += 1
if( p > 1 ) then
'' not a delimiter
if( tally( p - 1 ) = false ) then
nonDelimiterCount += 1
end if
end if
position = p + 1
else
exit do
end if
loop
'' if the last token wasn't a delimiter, add one to the tokens count
'' ( to account for the last token )
if( tally( strLen ) = false ) then
nonDelimitercount += 1
end if
if( returnDelimiters = true ) then
count = delimiterCount + nonDelimiterCount
else
count = nonDelimiterCount
end if
'' allocate space for returned tokens
redim tokens( 0 to count - 1 )
dim as uinteger current = 0
position = 1
'' second pass: splice the tokens
if( returnDelimiters = true ) then
'' return delimiters
for i as uinteger = 1 to strLen
if( tally( i ) = true ) then
if( i > 1 ) then
if( tally( i - 1 ) = false ) then
'' splice the token
tokens( current ) = mid( inputString, position, i - position )
current += 1
end if
end if
'' splice the delimiter
tokens( current ) = chr( inputString[ i - 1 ] )
current += 1
position = i + 1
end if
next
else
'' don't return delimiters
for i as uinteger = 1 to strLen
if( tally( i ) = true ) then
if( i > 1 ) then
if( tally( i - 1 ) = false ) then
'' splice the token
tokens( current ) = mid( inputString, position, i - position )
current += 1
end if
end if
position = i + 1
end if
next
end if
'' if the last char wasn't a delimiter, splice the last token too
if( position < strLen ) then
tokens( current ) = mid( inputString, position, strLen - position + 1 )
end if
return( count )
end function
dim as string commands = "abs,access,acos,alias,allocate,alpha,andalso,and,any,append,asc,as,asin,asm,assertwarn,assert,atan2,atn,base,beep,binary,bin,bitset,bitreset,bit,bload,bsave,byref,byte,byval,callocate,call,case,cast,cbyte,cdbl,cdecl,chain,chdir,chr,cint,circle," + _
"class,clear,clngint,clng,close,cls,color,,command,common,condbroadcast,condcreate,conddestroy,condsignal,condwait,constructor,const,cons,continue,cos,cptr,cshort,csign,csng,csrlin,cubyte,cuint,culngint,culng,cunsg,curdir,cushort,custom,cvd,cvi,cvl,cvlongint," + _
"cvs,cvshort,data,datevalue,dateadd,datediff,datepart,dateserial,date,day,deallocate,declare,defbyte,defdbl,defint,deflngint,deflng,defshort,defsng,defstr,defubyte,defuint,defulngint,defushort,delete,destructor,dim,dir,do,double,draw,dylibfree,dylibload,dylibsymbol," + _
"dynamic,elseif,else,encoding,endif,end,enum,environ,eof,eqv,erase,erfn,erl,ermn,err,error,escape,exec,exepath,exit,exp,explicit,export,extends,extern,false,fboolean,field,fileattr,filecopy,filedatetime,fileexists,filelen,fix,flip,format,for,frac,freefile," + _
"fre,function,getmouse,getjoystick,getkey,get,gosub,goto,hex,hibyte,hiword,hour,if,iif,imageconvertrow,imagecreate,imagedestroy,import,imp,inkey,input,input,inp,instrrev,instr,integer,int,interface,is,isdate,kill,lbound,lcase,left,len,let,lib,line,lobyte,loc," + _
"local,locate,lock,lof,log,long,longint,loop,loword,lpos,lprint,lpt,lset,ltrim,mid,minute,mkd,mkdir,mki,mkl,mklongint,mks,mkshort,mod,month,monthname,multikey,mutexcreate,mutexdestroy,mutexlock,mutexunlock,name,namespace,new,next,nokeyword,not,now,object,oct," + _
"offsetof,once,on,open,operator,option,orelse,or,output,out,overload,paint,palette,pascal,pcopy,peek,pipe,pmap,pointer,point,poke,pos,preserve,preset,print,private,procptr,property,protected,pset,ptr,public,put,randomize,random,read,reallocate,redim,rem,reset," + _
"restore,resume,return,rgba,rgb,right,rmdir,rnd,rset,rtrim,run,sadd,scope,screenunlock,screencontrol,screencopy,screenevent,screenglproc,screeninfo,screenlist,screenlock,screenptr,screenres,screenset,screensync,screen,scrn,second,seek,select,setdate,setenviron," + _
"setmouse,settime,sgn,shared,shell,shl,short,shr,sin,single,sizeof,sleep,space,spc,sqr,static,stdcall,step,stop,string,strptr,str,sub,swap,system,tab,tan,then,this,threadcreate,threadwait,timer,time,timeserial,timevalue,to,trans,trim,true,type,ubound,ubyte," + _
"ucase,uinteger,ulongint,ulong,union,unlock,unsigned,until,ushort,using,va,arg,va,first,valulng,valuint,vallng,valint,val,va,next,varptr,var,view,virtual,wait,wbin,wchr,weekdayname,weekday,wend,whex,while,width,windowtitle,window,winput,with,woct,write,wspace" + _
"#assert,#defined,#define,#elseif,#else,#endif,#endmacro,#error,#ifndef,#ifdef,#if,#inclib,#include,#lang,#libpath,#line,#macro,#once,#pragma,#print"
dim as integer numTokens = 414
dim as string tokens()
dim as string tokens2( 0 to numTokens - 1 )
dim as integer time1, time2
dim as double t, sum
dim as integer count = 100000
? "with tokenization"
for i as integer = 0 to count - 1
t = timer()
tokenizeString( commands, ",", tokens(), false )
t = timer() - t
sum += t
next
time1 = int( 1 / ( sum / count ) )
? time1
sum = 0.0
? "with 'data'"
for i as integer = 0 to count - 1
t = timer()
restore commandList
for j as integer = 0 to numTokens - 1
read tokens2( j )
next
t = timer() - t
sum += t
next
time2 = int( 1 / ( sum / count ) )
? time2
?
? "tokenization:"
for i as integer = 0 to numTokens - 1
? tokens( i ); " ";
next
?
?
? "data:"
for i as integer = 0 to numTokens - 1
? tokens2( i ); " ";
next
?
?
? "difference: "; time2 / time1
sleep()
commandList:
data "abs","access","acos","alias","allocate","alpha","andalso","and","any","append","asc","as","asin","asm","assertwarn","assert","atan2","atn","base","beep","binary","bin","bitset","bitreset","bit","bload","bsave","byref","byte","byval","callocate","call"
data "case","cast","cbyte","cdbl","cdecl","chain","chdir","chr","cint","circle","class","clear","clngint","clng","close","cls","color","","command","common","condbroadcast","condcreate","conddestroy","condsignal","condwait","constructor","const","cons","continue"
data "cos","cptr","cshort","csign","csng","csrlin","cubyte","cuint","culngint","culng","cunsg","curdir","cushort","custom","cvd","cvi","cvl","cvlongint","cvs","cvshort","data","datevalue","dateadd","datediff","datepart","dateserial","date","day","deallocate"
data "declare","defbyte","defdbl","defint","deflngint","deflng","defshort","defsng","defstr","defubyte","defuint","defulngint","defushort","delete","destructor","dim","dir","do","double","draw","dylibfree","dylibload","dylibsymbol","dynamic","elseif","else"
data "encoding","endif","end","enum","environ","eof","eqv","erase","erfn","erl","ermn","err","error","escape","exec","exepath","exit","exp","explicit","export","extends","extern","false","fboolean","field","fileattr","filecopy","filedatetime","fileexists"
data "filelen","fix","flip","format","for","frac","freefile","fre","function","getmouse","getjoystick","getkey","get","gosub","goto","hex","hibyte","hiword","hour","if","iif","imageconvertrow","imagecreate","imagedestroy","import","imp","inkey","input","input"
data "inp","instrrev","instr","integer","int","interface","is","isdate","kill","lbound","lcase","left","len","let","lib","line","lobyte","loc","local","locate","lock","lof","log","long","longint","loop","loword","lpos","lprint","lpt","lset","ltrim","mid","minute"
data "mkd","mkdir","mki","mkl","mklongint","mks","mkshort","mod","month","monthname","multikey","mutexcreate","mutexdestroy","mutexlock","mutexunlock","name","namespace","new","next","nokeyword","not","now","object","oct","offsetof","once","on","open","operator"
data "option","orelse","or","output","out","overload","paint","palette","pascal","pcopy","peek","pipe","pmap","pointer","point","poke","pos","preserve","preset","print","private","procptr","property","protected","pset","ptr","public","put","randomize","random"
data "read","reallocate","redim","rem","reset","restore","resume","return","rgba","rgb","right","rmdir","rnd","rset","rtrim","run","sadd","scope","screenunlock","screencontrol","screencopy","screenevent","screenglproc","screeninfo","screenlist","screenlock"
data "screenptr","screenres","screenset","screensync","screen","scrn","second","seek","select","setdate","setenviron","setmouse","settime","sgn","shared","shell","shl","short","shr","sin","single","sizeof","sleep","space","spc","sqr","static","stdcall","step"
data "stop","string","strptr","str","sub","swap","system","tab","tan","then","this","threadcreate","threadwait","timer","time","timeserial","timevalue","to","trans","trim","true","type","ubound","ubyte","ucase","uinteger","ulongint","ulong","union","unlock"
data "unsigned","until","ushort","using","va","arg","va","first","valulng","valuint","vallng","valint","val","va","next","varptr","var","view","virtual","wait","wbin","wchr","weekdayname","weekday","wend","whex","while","width","windowtitle","window","winput"
data "with","woct","write","wspace","wstring","wstr","xor","year","zstring","#assert","#defined","#define","#elseif","#else","#endif","#endmacro","#error","#ifndef","#ifdef","#if","#inclib","#include","#lang","#libpath","#line","#macro","#once","#pragma","#print"
@IchMagBier: nice work, mate.
PS: DATA? Seriously?
Re: Routines to write syntax highlighted code
Quite seriously. Not just because it's faster, but because it is also easy to maintain when stored as include files, which only requires a recompilation, which happens all the time anyway because of fixes and improvements. Loading separate files is slower and also more prone to corruption.
However, both are legitimate ways and it is generally a programmer's preference. So yes, seriously.
However, both are legitimate ways and it is generally a programmer's preference. So yes, seriously.
Re: Routines to write syntax highlighted code
Faster still:
But, as I already stated, it's inflexible. The OP's original intent was to highlight syntax for a variety of languages, and if you do it like this, you need to recompile the program every time you want to add another language's lexicon. Nothing wrong with it if you're debugging the app, but it shouldn't become stuck in debug mode forever.
Code: Select all
dim as string tokens( ... ) = _
{ _
"abs","access","acos","alias","allocate","alpha","andalso","and","any","append","asc","as","asin","asm","assertwarn","assert","atan2","atn","base","beep","binary","bin","bitset","bitreset","bit","bload","bsave","byref","byte","byval","callocate","call" _
"case","cast","cbyte","cdbl","cdecl","chain","chdir","chr","cint","circle","class","clear","clngint","clng","close","cls","color","","command","common","condbroadcast","condcreate","conddestroy","condsignal","condwait","constructor","const","cons","continue" _
"cos","cptr","cshort","csign","csng","csrlin","cubyte","cuint","culngint","culng","cunsg","curdir","cushort","custom","cvd","cvi","cvl","cvlongint","cvs","cvshort","data","datevalue","dateadd","datediff","datepart","dateserial","date","day","deallocate" _
"declare","defbyte","defdbl","defint","deflngint","deflng","defshort","defsng","defstr","defubyte","defuint","defulngint","defushort","delete","destructor","dim","dir","do","double","draw","dylibfree","dylibload","dylibsymbol","dynamic","elseif","else" _
"encoding","endif","end","enum","environ","eof","eqv","erase","erfn","erl","ermn","err","error","escape","exec","exepath","exit","exp","explicit","export","extends","extern","false","fboolean","field","fileattr","filecopy","filedatetime","fileexists" _
"filelen","fix","flip","format","for","frac","freefile","fre","function","getmouse","getjoystick","getkey","get","gosub","goto","hex","hibyte","hiword","hour","if","iif","imageconvertrow","imagecreate","imagedestroy","import","imp","inkey","input","input" _
"inp","instrrev","instr","integer","int","interface","is","isdate","kill","lbound","lcase","left","len","let","lib","line","lobyte","loc","local","locate","lock","lof","log","long","longint","loop","loword","lpos","lprint","lpt","lset","ltrim","mid","minute" _
"mkd","mkdir","mki","mkl","mklongint","mks","mkshort","mod","month","monthname","multikey","mutexcreate","mutexdestroy","mutexlock","mutexunlock","name","namespace","new","next","nokeyword","not","now","object","oct","offsetof","once","on","open","operator" _
"option","orelse","or","output","out","overload","paint","palette","pascal","pcopy","peek","pipe","pmap","pointer","point","poke","pos","preserve","preset","print","private","procptr","property","protected","pset","ptr","public","put","randomize","random" _
"read","reallocate","redim","rem","reset","restore","resume","return","rgba","rgb","right","rmdir","rnd","rset","rtrim","run","sadd","scope","screenunlock","screencontrol","screencopy","screenevent","screenglproc","screeninfo","screenlist","screenlock" _
"screenptr","screenres","screenset","screensync","screen","scrn","second","seek","select","setdate","setenviron","setmouse","settime","sgn","shared","shell","shl","short","shr","sin","single","sizeof","sleep","space","spc","sqr","static","stdcall","step" _
"stop","string","strptr","str","sub","swap","system","tab","tan","then","this","threadcreate","threadwait","timer","time","timeserial","timevalue","to","trans","trim","true","type","ubound","ubyte","ucase","uinteger","ulongint","ulong","union","unlock" _
"unsigned","until","ushort","using","va","arg","va","first","valulng","valuint","vallng","valint","val","va","next","varptr","var","view","virtual","wait","wbin","wchr","weekdayname","weekday","wend","whex","while","width","windowtitle","window","winput" _
"with","woct","write","wspace","wstring","wstr","xor","year","zstring","#assert","#defined","#define","#elseif","#else","#endif","#endmacro","#error","#ifndef","#ifdef","#if","#inclib","#include","#lang","#libpath","#line","#macro","#once","#pragma","#print" _
}
for i as integer = 0 to ubound( tokens )
? tokens( i ); " ";
next
sleep()
Re: Routines to write syntax highlighted code
A language/keyword file is the way to go. You get to take advantage of the many editing tools available in a text editor. You get portability. You don't have to release your code.
Just as a side note the keyword lists used here are incomplete. I notice that none of the boolean keywords are in it.
I think the one I use in my project is incomplete as well.
IIRC you can find the complete list of keywords in the source code files for FB.
Its not faster.
I tried it on xp and on ubuntu and in opposite order and it never won once.
Just as a side note the keyword lists used here are incomplete. I notice that none of the boolean keywords are in it.
I think the one I use in my project is incomplete as well.
IIRC you can find the complete list of keywords in the source code files for FB.
paul doe wrote:Faster still:
Its not faster.
Code: Select all
dim as double atime, btime, ctime, dtime
dim as string array()
atime = timer
redim array(9)
for n as integer = 1 to 1000
for x as integer = 0 to 9
read array(x)
next
next
btime = timer - atime
ctime = timer
for x as integer = 1 to 1000
dim as string v(...) = {"1","2","3", "4", "5", "6", "7", "8", "9", "0"}
next
dtime = timer - ctime
print btime , dtime
data "1", "2", "3", "4", "5", "6", "7", "8", "9", "0"
Code: Select all
0.0003439908614382148 0.0009779528481885791
Last edited by sancho3 on Jan 15, 2018 17:16, edited 2 times in total.
-
- Posts: 52
- Joined: Jan 13, 2018 8:47
- Location: Germany
- Contact:
Re: Routines to write syntax highlighted code
I took the keyword-list out of geany's file for FreeBasic. Notice how some of the OOP-stuff is missing aswell ("Virtual" or "Abstract" for example).
Re: Routines to write syntax highlighted code
Perhaps. It's about 1.6 times slower to populate the array, according to my tests. But it's faster and simpler to 'set up' than the restore-read crap (it's also more elegant, but that's just my preference). However, be aware that you aren't really doing fair comparisons. You're also measuring the creation and destruction of the array (since it's included in a for-next loop and, thus, into it's own scope), which is incorrect:sancho3 wrote:Its not faster.
Code: Select all
'ctime = timer
dim as double sum, t
for x as integer = 1 to 1000
t = timer()
dim as string v(...) = {"1","2","3", "4", "5", "6", "7", "8", "9", "0"}
t = timer() - t
sum += t
next
t = sum / 1000
'dtime = timer - ctime
The very fastest way would simply be:
Code: Select all
dim as integer numTokens = 414
dim as string tokens( 0 to numTokens - 1 )
dim as string tokens2( 0 to numTokens - 1 )
dim as integer time1, time2
dim as double t, sum
dim as integer count = 100000
? "with initialization"
for i as integer = 0 to count - 1
t = timer()
tokens(0)="abs"
tokens(1)="access"
tokens(2)="acos"
tokens(3)="alias"
tokens(4)="allocate"
tokens(5)="alpha"
tokens(6)="andalso"
tokens(7)="and"
tokens(8)="any"
tokens(9)="append"
tokens(10)="asc"
tokens(11)="as"
tokens(12)="asin"
tokens(13)="asm"
tokens(14)="assertwarn"
tokens(15)="assert"
tokens(16)="atan2"
tokens(17)="atn"
tokens(18)="base"
tokens(19)="beep"
tokens(20)="binary"
tokens(21)="bin"
tokens(22)="bitset"
tokens(23)="bitreset"
tokens(24)="bit"
tokens(25)="bload"
tokens(26)="bsave"
tokens(27)="byref"
tokens(28)="byte"
tokens(29)="byval"
tokens(30)="callocate"
tokens(31)="call"
tokens(32)="case"
tokens(33)="cast"
tokens(34)="cbyte"
tokens(35)="cdbl"
tokens(36)="cdecl"
tokens(37)="chain"
tokens(38)="chdir"
tokens(39)="chr"
tokens(40)="cint"
tokens(41)="circle"
tokens(42)="class"
tokens(43)="clear"
tokens(44)="clngint"
tokens(45)="clng"
tokens(46)="close"
tokens(47)="cls"
tokens(48)="color"
tokens(50)="command"
tokens(51)="common"
tokens(52)="condbroadcast"
tokens(53)="condcreate"
tokens(54)="conddestroy"
tokens(55)="condsignal"
tokens(56)="condwait"
tokens(57)="constructor"
tokens(58)="const"
tokens(59)="cons"
tokens(60)="continue"
tokens(61)="cos"
tokens(62)="cptr"
tokens(63)="cshort"
tokens(64)="csign"
tokens(65)="csng"
tokens(66)="csrlin"
tokens(67)="cubyte"
tokens(68)="cuint"
tokens(69)="culngint"
tokens(70)="culng"
tokens(71)="cunsg"
tokens(72)="curdir"
tokens(73)="cushort"
tokens(74)="custom"
tokens(75)="cvd"
tokens(76)="cvi"
tokens(77)="cvl"
tokens(78)="cvlongint"
tokens(79)="cvs"
tokens(80)="cvshort"
tokens(81)="data"
tokens(82)="datevalue"
tokens(83)="dateadd"
tokens(84)="datediff"
tokens(85)="datepart"
tokens(86)="dateserial"
tokens(87)="date"
tokens(88)="day"
tokens(89)="deallocate"
tokens(90)="declare"
tokens(91)="defbyte"
tokens(92)="defdbl"
tokens(93)="defint"
tokens(94)="deflngint"
tokens(95)="deflng"
tokens(96)="defshort"
tokens(97)="defsng"
tokens(98)="defstr"
tokens(99)="defubyte"
tokens(100)="defuint"
tokens(101)="defulngint"
tokens(102)="defushort"
tokens(103)="delete"
tokens(104)="destructor"
tokens(105)="dim"
tokens(106)="dir"
tokens(107)="do"
tokens(108)="double"
tokens(109)="draw"
tokens(110)="dylibfree"
tokens(111)="dylibload"
tokens(112)="dylibsymbol"
tokens(113)="dynamic"
tokens(114)="elseif"
tokens(115)="else"
tokens(116)="encoding"
tokens(117)="endif"
tokens(118)="end"
tokens(119)="enum"
tokens(120)="environ"
tokens(121)="eof"
tokens(122)="eqv"
tokens(123)="erase"
tokens(124)="erfn"
tokens(125)="erl"
tokens(126)="ermn"
tokens(127)="err"
tokens(128)="error"
tokens(129)="escape"
tokens(130)="exec"
tokens(131)="exepath"
tokens(132)="exit"
tokens(133)="exp"
tokens(134)="explicit"
tokens(135)="export"
tokens(136)="extends"
tokens(137)="extern"
tokens(138)="false"
tokens(139)="fboolean"
tokens(140)="field"
tokens(141)="fileattr"
tokens(142)="filecopy"
tokens(143)="filedatetime"
tokens(144)="fileexists"
tokens(145)="filelen"
tokens(146)="fix"
tokens(147)="flip"
tokens(148)="format"
tokens(149)="for"
tokens(150)="frac"
tokens(151)="freefile"
tokens(152)="fre"
tokens(153)="function"
tokens(154)="getmouse"
tokens(155)="getjoystick"
tokens(156)="getkey"
tokens(157)="get"
tokens(158)="gosub"
tokens(159)="goto"
tokens(160)="hex"
tokens(161)="hibyte"
tokens(162)="hiword"
tokens(163)="hour"
tokens(164)="if"
tokens(165)="iif"
tokens(166)="imageconvertrow"
tokens(167)="imagecreate"
tokens(168)="imagedestroy"
tokens(169)="import"
tokens(170)="imp"
tokens(171)="inkey"
tokens(172)="input"
tokens(173)="input"
tokens(174)="inp"
tokens(175)="instrrev"
tokens(176)="instr"
tokens(177)="integer"
tokens(178)="int"
tokens(179)="interface"
tokens(180)="is"
tokens(181)="isdate"
tokens(182)="kill"
tokens(183)="lbound"
tokens(184)="lcase"
tokens(185)="left"
tokens(186)="len"
tokens(187)="let"
tokens(188)="lib"
tokens(189)="line"
tokens(190)="lobyte"
tokens(191)="loc"
tokens(192)="local"
tokens(193)="locate"
tokens(194)="lock"
tokens(195)="lof"
tokens(196)="log"
tokens(197)="long"
tokens(198)="longint"
tokens(199)="loop"
tokens(200)="loword"
tokens(201)="lpos"
tokens(202)="lprint"
tokens(203)="lpt"
tokens(204)="lset"
tokens(205)="ltrim"
tokens(206)="mid"
tokens(207)="minute"
tokens(208)="mkd"
tokens(209)="mkdir"
tokens(210)="mki"
tokens(211)="mkl"
tokens(212)="mklongint"
tokens(213)="mks"
tokens(214)="mkshort"
tokens(215)="mod"
tokens(216)="month"
tokens(217)="monthname"
tokens(218)="multikey"
tokens(219)="mutexcreate"
tokens(220)="mutexdestroy"
tokens(221)="mutexlock"
tokens(222)="mutexunlock"
tokens(223)="name"
tokens(224)="namespace"
tokens(225)="new"
tokens(226)="next"
tokens(227)="nokeyword"
tokens(228)="not"
tokens(229)="now"
tokens(230)="object"
tokens(231)="oct"
tokens(232)="offsetof"
tokens(233)="once"
tokens(234)="on"
tokens(235)="open"
tokens(236)="operator"
tokens(237)="option"
tokens(238)="orelse"
tokens(239)="or"
tokens(240)="output"
tokens(241)="out"
tokens(242)="overload"
tokens(243)="paint"
tokens(244)="palette"
tokens(245)="pascal"
tokens(246)="pcopy"
tokens(247)="peek"
tokens(248)="pipe"
tokens(249)="pmap"
tokens(250)="pointer"
tokens(251)="point"
tokens(252)="poke"
tokens(253)="pos"
tokens(254)="preserve"
tokens(255)="preset"
tokens(256)="print"
tokens(257)="private"
tokens(258)="procptr"
tokens(259)="property"
tokens(260)="protected"
tokens(261)="pset"
tokens(262)="ptr"
tokens(263)="public"
tokens(264)="put"
tokens(265)="randomize"
tokens(266)="random"
tokens(267)="read"
tokens(268)="reallocate"
tokens(269)="redim"
tokens(270)="rem"
tokens(271)="reset"
tokens(272)="restore"
tokens(273)="resume"
tokens(274)="return"
tokens(275)="rgba"
tokens(276)="rgb"
tokens(277)="right"
tokens(278)="rmdir"
tokens(279)="rnd"
tokens(280)="rset"
tokens(281)="rtrim"
tokens(282)="run"
tokens(283)="sadd"
tokens(284)="scope"
tokens(285)="screenunlock"
tokens(286)="screencontrol"
tokens(287)="screencopy"
tokens(288)="screenevent"
tokens(289)="screenglproc"
tokens(290)="screeninfo"
tokens(291)="screenlist"
tokens(292)="screenlock"
tokens(293)="screenptr"
tokens(294)="screenres"
tokens(295)="screenset"
tokens(296)="screensync"
tokens(297)="screen"
tokens(298)="scrn"
tokens(299)="second"
tokens(300)="seek"
tokens(301)="select"
tokens(302)="setdate"
tokens(303)="setenviron"
tokens(304)="setmouse"
tokens(305)="settime"
tokens(306)="sgn"
tokens(307)="shared"
tokens(308)="shell"
tokens(309)="shl"
tokens(310)="short"
tokens(311)="shr"
tokens(312)="sin"
tokens(313)="single"
tokens(314)="sizeof"
tokens(315)="sleep"
tokens(316)="space"
tokens(317)="spc"
tokens(318)="sqr"
tokens(319)="static"
tokens(320)="stdcall"
tokens(321)="step"
tokens(322)="stop"
tokens(323)="string"
tokens(324)="strptr"
tokens(325)="str"
tokens(326)="sub"
tokens(327)="swap"
tokens(328)="system"
tokens(329)="tab"
tokens(330)="tan"
tokens(331)="then"
tokens(332)="this"
tokens(333)="threadcreate"
tokens(334)="threadwait"
tokens(335)="timer"
tokens(336)="time"
tokens(337)="timeserial"
tokens(338)="timevalue"
tokens(339)="to"
tokens(340)="trans"
tokens(341)="trim"
tokens(342)="true"
tokens(343)="type"
tokens(344)="ubound"
tokens(345)="ubyte"
tokens(346)="ucase"
tokens(347)="uinteger"
tokens(348)="ulongint"
tokens(349)="ulong"
tokens(350)="union"
tokens(351)="unlock"
tokens(352)="unsigned"
tokens(353)="until"
tokens(354)="ushort"
tokens(355)="using"
tokens(356)="va"
tokens(357)="arg"
tokens(358)="va"
tokens(359)="first"
tokens(360)="valulng"
tokens(361)="valuint"
tokens(362)="vallng"
tokens(363)="valint"
tokens(364)="val"
tokens(365)="va"
tokens(366)="next"
tokens(367)="varptr"
tokens(368)="var"
tokens(369)="view"
tokens(370)="virtual"
tokens(371)="wait"
tokens(372)="wbin"
tokens(373)="wchr"
tokens(374)="weekdayname"
tokens(375)="weekday"
tokens(376)="wend"
tokens(377)="whex"
tokens(378)="while"
tokens(379)="width"
tokens(380)="windowtitle"
tokens(381)="window"
tokens(382)="winput"
tokens(383)="with"
tokens(384)="woct"
tokens(385)="write"
tokens(386)="wspace"
tokens(387)="wstring"
tokens(388)="wstr"
tokens(389)="xor"
tokens(390)="year"
tokens(391)="zstring"
tokens(392)="#assert"
tokens(393)="#defined"
tokens(394)="#define"
tokens(395)="#elseif"
tokens(396)="#else"
tokens(397)="#endif"
tokens(398)="#endmacro"
tokens(399)="#error"
tokens(400)="#ifndef"
tokens(401)="#ifdef"
tokens(402)="#if"
tokens(403)="#inclib"
tokens(404)="#include"
tokens(405)="#lang"
tokens(406)="#libpath"
tokens(407)="#line"
tokens(408)="#macro"
tokens(409)="#once"
tokens(410)="#pragma"
tokens(411)="#print"
tokens(412)="#typeof"
tokens(413)="#undef"
t = timer() - t
sum += t
next
time1 = int( 1 / ( sum / count ) )
? time1
sum = 0.0
? "with 'data'"
for i as integer = 0 to count - 1
restore commandList
t = timer()
for j as integer = 0 to numTokens - 1
read tokens2( j )
next
t = timer() - t
sum += t
next
time2 = int( 1 / ( sum / count ) )
? time2
?
?
? "difference: "; time2 / time1
sleep()
commandList:
data "abs","access","acos","alias","allocate","alpha","andalso","and","any","append","asc","as","asin","asm","assertwarn","assert","atan2","atn","base","beep","binary","bin","bitset","bitreset","bit","bload","bsave","byref","byte","byval","callocate","call"
data "case","cast","cbyte","cdbl","cdecl","chain","chdir","chr","cint","circle","class","clear","clngint","clng","close","cls","color","","command","common","condbroadcast","condcreate","conddestroy","condsignal","condwait","constructor","const","cons","continue"
data "cos","cptr","cshort","csign","csng","csrlin","cubyte","cuint","culngint","culng","cunsg","curdir","cushort","custom","cvd","cvi","cvl","cvlongint","cvs","cvshort","data","datevalue","dateadd","datediff","datepart","dateserial","date","day","deallocate"
data "declare","defbyte","defdbl","defint","deflngint","deflng","defshort","defsng","defstr","defubyte","defuint","defulngint","defushort","delete","destructor","dim","dir","do","double","draw","dylibfree","dylibload","dylibsymbol","dynamic","elseif","else"
data "encoding","endif","end","enum","environ","eof","eqv","erase","erfn","erl","ermn","err","error","escape","exec","exepath","exit","exp","explicit","export","extends","extern","false","fboolean","field","fileattr","filecopy","filedatetime","fileexists"
data "filelen","fix","flip","format","for","frac","freefile","fre","function","getmouse","getjoystick","getkey","get","gosub","goto","hex","hibyte","hiword","hour","if","iif","imageconvertrow","imagecreate","imagedestroy","import","imp","inkey","input","input"
data "inp","instrrev","instr","integer","int","interface","is","isdate","kill","lbound","lcase","left","len","let","lib","line","lobyte","loc","local","locate","lock","lof","log","long","longint","loop","loword","lpos","lprint","lpt","lset","ltrim","mid","minute"
data "mkd","mkdir","mki","mkl","mklongint","mks","mkshort","mod","month","monthname","multikey","mutexcreate","mutexdestroy","mutexlock","mutexunlock","name","namespace","new","next","nokeyword","not","now","object","oct","offsetof","once","on","open","operator"
data "option","orelse","or","output","out","overload","paint","palette","pascal","pcopy","peek","pipe","pmap","pointer","point","poke","pos","preserve","preset","print","private","procptr","property","protected","pset","ptr","public","put","randomize","random"
data "read","reallocate","redim","rem","reset","restore","resume","return","rgba","rgb","right","rmdir","rnd","rset","rtrim","run","sadd","scope","screenunlock","screencontrol","screencopy","screenevent","screenglproc","screeninfo","screenlist","screenlock"
data "screenptr","screenres","screenset","screensync","screen","scrn","second","seek","select","setdate","setenviron","setmouse","settime","sgn","shared","shell","shl","short","shr","sin","single","sizeof","sleep","space","spc","sqr","static","stdcall","step"
data "stop","string","strptr","str","sub","swap","system","tab","tan","then","this","threadcreate","threadwait","timer","time","timeserial","timevalue","to","trans","trim","true","type","ubound","ubyte","ucase","uinteger","ulongint","ulong","union","unlock"
data "unsigned","until","ushort","using","va","arg","va","first","valulng","valuint","vallng","valint","val","va","next","varptr","var","view","virtual","wait","wbin","wchr","weekdayname","weekday","wend","whex","while","width","windowtitle","window","winput"
data "with","woct","write","wspace","wstring","wstr","xor","year","zstring","#assert","#defined","#define","#elseif","#else","#endif","#endmacro","#error","#ifndef","#ifdef","#if","#inclib","#include","#lang","#libpath","#line","#macro","#once","#pragma","#print"
Last edited by paul doe on Jan 15, 2018 20:34, edited 1 time in total.
Re: Routines to write syntax highlighted code
Geany's list is incomplete. If you want the latest you can always go over the keyword list in the manual.IchMagBier wrote:I took the keyword-list out of geany's file for FreeBasic. Notice how some of the OOP-stuff is missing aswell ("Virtual" or "Abstract" for example).
Re: Routines to write syntax highlighted code
According to my tests, reading an array from a textfile takes about ONE millisecond. Per 30,000 strings, that is. The whole debate is a bit strange ;-)paul doe wrote:It's about 1.6 times slower to populate the array, according to my tests
Re: Routines to write syntax highlighted code
I have uploaded the complete (I think) list of keywords here.IchMagBier wrote:I took the keyword-list out of geany's file for FreeBasic. Notice how some of the OOP-stuff is missing aswell ("Virtual" or "Abstract" for example).
They are in the format you are using:
Code: Select all
.Command(0) = "ABS":.Command(1) = "ABSTRACT":.Command(2) = "ACCESS"
.Command(4) = "ALIAS":.Command(5) = "AND":.Command(6) = "ANDALSO"
Edited to fix incomplete list.
Last edited by sancho3 on Jan 16, 2018 7:01, edited 1 time in total.
Re: Routines to write syntax highlighted code
And irrelevant for being boilerplate code, indeed =Djj2007 wrote:According to my tests, reading an array from a textfile takes about ONE millisecond. Per 30,000 strings, that is. The whole debate is a bit strange ;-)
-
- Posts: 52
- Joined: Jan 13, 2018 8:47
- Location: Germany
- Contact:
Re: Routines to write syntax highlighted code
@sancho
It's important that 'ABS' comes after 'ABSTRACT' in the list, because the routine would first find 'ABS' in the code and then completely ignores 'ABSTRACT'. Like this: ABSTRACT SUB
I updated the first post and added the following keywords to the list:
It's important that 'ABS' comes after 'ABSTRACT' in the list, because the routine would first find 'ABS' in the code and then completely ignores 'ABSTRACT'. Like this: ABSTRACT SUB
I updated the first post and added the following keywords to the list:
Code: Select all
abstract virtual boolean implemets naked override add threaddetach cbool deflongint defulongint imageinfo nogosub stick strig threadcall val64
Re: Routines to write syntax highlighted code
There might be a few more then.
AND ANDALSO
AS ASC ASM
CLNG CLNGINT
CONST CONSTRUCTOR
CULNG CULNGINT
CVS CVSSHORT
DO DOUBLE
ELSE ELSEIF
END ENDIF
ERR ERROR
IMP IMPLEMENTS IMPORT - implements isn't used but it is reserved
INSTR INSTRREV
LONG LONGINT
MKL MKLONGINT
NAME NAMESPACE
OR ORELSE
POINT POINTER
SIN SINGLE
STR STRING STRPTR
TYPE TYPEOF
VAR VARPTR
WSTR WSTRING
ERROR NOTICE:
Sorry my friend. I messed up in my code formatting and cut out lots of commands.
I updated the link in my previous post with the complete list of keywords. And above are the corrections needed to be made to work as you want.
AND ANDALSO
AS ASC ASM
CLNG CLNGINT
CONST CONSTRUCTOR
CULNG CULNGINT
CVS CVSSHORT
DO DOUBLE
ELSE ELSEIF
END ENDIF
ERR ERROR
IMP IMPLEMENTS IMPORT - implements isn't used but it is reserved
INSTR INSTRREV
LONG LONGINT
MKL MKLONGINT
NAME NAMESPACE
OR ORELSE
POINT POINTER
SIN SINGLE
STR STRING STRPTR
TYPE TYPEOF
VAR VARPTR
WSTR WSTRING
ERROR NOTICE:
Sorry my friend. I messed up in my code formatting and cut out lots of commands.
I updated the link in my previous post with the complete list of keywords. And above are the corrections needed to be made to work as you want.
Re: Routines to write syntax highlighted code
"AS" should change place with "ASIN"IchMagBier wrote:@sancho
It's important that 'ABS' comes after 'ABSTRACT' in the list, because the routine would first find 'ABS' in the code and then completely ignores 'ABSTRACT'. Like this: ABSTRACT SUB
"CULNG" should change place with "CULNGINT"
"DO" should change place with "DOUBLE"
"END" should change place with "ENDIF"
"ELSE" should change place with "ELSEIF"
"ERR" should change place with "ERROR"
"EXP" should change place with "EXPLICIT"
And so on. Or maybe better to search the table in reverse order Z - A.
Re: Routines to write syntax highlighted code
Here is the FBWiki : CatPgFullIndex today if it is any help.
Code: Select all
FBWiki : CatPgFullIndex
__DATE__
__DATE_ISO__
__FB_64BIT__
__FB_ARGC__
__FB_ARGV__
__FB_ARM__
__FB_ASM__
__FB_BACKEND__
__FB_BIGENDIAN__
__FB_BUILD_DATE__
__FB_CYGWIN__
__FB_DARWIN__
__FB_DEBUG__
__FB_DOS__
__FB_ERR__
__FB_FPMODE__
__FB_FPU__
__FB_FREEBSD__
__FB_GCC__
__FB_LANG__
__FB_LINUX__
__FB_MAIN__
__FB_MIN_VERSION__
__FB_MT__
__FB_NETBSD__
__FB_OPENBSD__
__FB_OPTION_BYVAL__
__FB_OPTION_DYNAMIC__
__FB_OPTION_ESCAPE__
__FB_OPTION_EXPLICIT__
__FB_OPTION_GOSUB__
__FB_OPTION_PRIVATE__
__FB_OUT_DLL__
__FB_OUT_EXE__
__FB_OUT_LIB__
__FB_OUT_OBJ__
__FB_PCOS__
__FB_SIGNATURE__
__FB_SSE__
__FB_UNIX__
__FB_VECTORIZE__
__FB_VER_MAJOR__
__FB_VER_MINOR__
__FB_VER_PATCH__
__FB_VERSION__
__FB_WIN32__
__FB_XBOX__
__FILE__
__FILE_NQ__
__FUNCTION__
__FUNCTION_NQ__
__LINE__
__PATH__
__TIME__
#ASSERT
#DEFINE
#ELSE
#ELSEIF
#ENDIF
#ENDMACRO
#ERROR
#IF
#IFDEF
#IFNDEF
#INCLIB
#INCLUDE
#LANG
#LIBPATH
#LINE
#MACRO
#PRAGMA
#PRINT
#UNDEF
$DYNAMIC
$INCLUDE
$LANG
$STATIC
ABS
ABSTRACT (member)
ACCESS
ACOS
ADD (Graphics PUT)
ALIAS
ALLOCATE
ALPHA (Graphics PUT)
AND
ANDALSO
AND (GRAPHICS PUT)
ANY
APPEND
AS
ASC
ASIN
ASM
ASSERT
ASSERTWARN
ATAN2
ATN
BASE (initialization)
BASE (member access)
BEEP
BIN
BINARY
BIT
BITRESET
BITSET
BLOAD
BOOLEAN
BSAVE
BYREF (parameters)
BYREF (function results)
BYREF (variables)
BYTE
BYVAL
CALL
CALLOCATE
CASE
CAST
CBOOL
CBYTE
CDBL
CDECL
CHAIN
CHDIR
CHR
CINT
CIRCLE
CLASS
CLEAR
CLNG
CLNGINT
CLOSE
CLS
COLOR
COMMAND
COMMON
CONDBROADCAST
CONDCREATE
CONDDESTROY
CONDSIGNAL
CONDWAIT
CONST
CONST (Member)
CONST (Qualifier)
CONSTRUCTOR
CONSTRUCTOR (Module)
CONTINUE
COS
CPTR
CSHORT
CSIGN
CSNG
CSRLIN
CUBYTE
CUINT
CULNG
CULNGINT
CUNSG
CURDIR
CUSHORT
CUSTOM (Graphics PUT)
CVD
CVI
CVL
CVLONGINT
CVS
CVSHORT
DATA
DATE
DATEADD
DATEDIFF
DATEPART
DATESERIAL
DATEVALUE
DAY
DEALLOCATE
DECLARE
DEFBYTE
DEFDBL
DEFINED
DEFINT
DEFLNG
DEFLONGINT
DEFSHORT
DEFSNG
DEFSTR
DEFUBYTE
DEFUINT
DEFULONGINT
DEFUSHORT
DELETE
DESTRUCTOR
DESTRUCTOR (Module)
DIM
DIR
DO
DO...LOOP
DOUBLE
DRAW
DRAW STRING
DYLIBFREE
DYLIBLOAD
DYLIBSYMBOL
ELSE
ELSEIF
ENCODING
END (Block)
END (Statement)
END IF
ENUM
ENVIRON statement
ENVIRON
EOF
EQV
ERASE
ERFN
ERL
ERMN
ERR
ERROR
EVENT (message data from ScreenEvent)
EXEC
EXEPATH
EXIT
EXP
EXPORT
EXTENDS
EXTERN
EXTERN...END EXTERN
FALSE
FIELD
FILEATTR
FILECOPY
FILEDATETIME
FILEEXISTS
FILELEN
FIX
FLIP
FOR
FOR...NEXT
FORMAT
FRAC
FRE
FREEFILE
FUNCTION
FUNCTION (Member)
GET (Graphics)
GET # (File I/O)
GETJOYSTICK
GETKEY
GETMOUSE
GOSUB
GOTO
HEX
HIBYTE
HIWORD
HOUR
IF...THEN
IIF
IMAGECONVERTROW
IMAGECREATE
IMAGEDESTROY
IMAGEINFO
IMP
IMPLEMENTS
IMPORT
INKEY
INP
INPUT (Statement)
INPUT (File I/O)
INPUT #
INPUT$
INSTR
INSTRREV
INT
INTEGER
IS (SELECT CASE)
IS (Run-time type information operator)
ISDATE
ISREDIRECTED
KILL
LBOUND
LCASE
LEFT
LEN
LET
LIB
LINE
LINE INPUT
LINE INPUT #
LOBYTE
LOC
LOCAL
LOCATE
LOCK
LOF
LOG
LONG
LONGINT
LOOP
LOWORD
LPOS
LPRINT
LSET
LTRIM
MID (Statement)
MID (Function)
MINUTE
MKD
MKDIR
MKI
MKL
MKLONGINT
MKS
MKSHORT
MOD
MONTH
MONTHNAME
MULTIKEY
MUTEXCREATE
MUTEXDESTROY
MUTEXLOCK
MUTEXUNLOCK
NAKED
NAME
NAMESPACE
NEW
NEW (Placement)
NEXT
NEXT (RESUME)
NOT
NOW
OBJECT
OCT
OFFSETOF
ON ERROR
ON...GOSUB
ON...GOTO
ONCE
OPEN
OPEN COM
OPEN CONS
OPEN ERR
OPEN LPT
OPEN PIPE
OPEN SCRN
OPERATOR
OPTION()
OPTION BASE
OPTION BYVAL
OPTION DYNAMIC
OPTION ESCAPE
OPTION EXPLICIT
OPTION GOSUB
OPTION NOGOSUB
OPTION NOKEYWORD
OPTION PRIVATE
OPTION STATIC
OR
OR (GRAPHICS PUT)
ORELSE
OUT
OUTPUT
OVERLOAD
OVERRIDE
PAINT
PALETTE
PASCAL
PCOPY
PEEK
PMAP
POINT
POINTCOORD
POINTER
POKE
POS
PRESERVE
PRESET
PRINT
?
PRINT #
? #
PRINT USING
? USING
PRIVATE
PRIVATE: (Access Control)
PROCPTR
PROPERTY
PROTECTED: (Access Control)
PSET (Statement)
PSET (Graphics PUT)
PTR
PUBLIC
PUBLIC: (Access Control)
PUT (Graphics)
PUT # (File I/O)
RANDOM
RANDOMIZE
READ
READ (File Access)
READ WRITE (File Access)
REALLOCATE
REDIM
REM
RESET
RESTORE
RESUME
RESUME NEXT
RETURN
RGB
RGBA
RIGHT
RMDIR
RND
RSET
RTRIM
RUN
SADD
SCOPE
SCREEN
SCREEN (Console)
SCREENCOPY
SCREENCONTROL
SCREENEVENT
SCREENGLPROC
SCREENINFO
SCREENLIST
SCREENLOCK
SCREENPTR
SCREENRES
SCREENSET
SCREENSYNC
SCREENUNLOCK
SECOND
SEEK (Statement)
SEEK (Function)
SELECT CASE
SETDATE
SETENVIRON
SETMOUSE
SETTIME
SGN
SHARED
SHELL
SHL
SHORT
SHR
SIN
SINGLE
SIZEOF
SLEEP
SPACE
SPC
SQR
STATIC
STATIC (Member)
STDCALL
STEP
STICK
STOP
STR
STRIG
STRING (Function)
STRING
STRPTR
SUB
SUB (Member)
SWAP
SYSTEM
TAB
TAN
THEN
THIS
THREADCALL
THREADCREATE
THREADDETACH
THREADWAIT
TIME
TIMER
TIMESERIAL
TIMEVALUE
TO
TRANS (Graphics PUT)
TRIM
TRUE
TYPE (Alias)
TYPE (Temporary)
TYPE (UDT)
TYPEOF
UBOUND
UBYTE
UCASE
UINTEGER
ULONG
ULONGINT
UNION
UNLOCK
UNSIGNED
UNTIL
USHORT
USING (PRINT)
USING (Namespaces)
VA_ARG
VA_FIRST
VA_NEXT
VAL
VALLNG
VALINT
VALUINT
VALULNG
VAR
VARPTR
VIEW PRINT
VIEW (Graphics)
VIRTUAL (member)
WAIT
WBIN
WCHR
WEEKDAY
WEEKDAYNAME
WEND
WHILE
WHILE...WEND
WHEX
WIDTH
WINDOW
WINDOWTITLE
WINPUT
WITH
WOCT
WRITE
WRITE #
WRITE (File Access)
WSPACE
WSTR
WSTRING (Data Type)
WSTRING (Function)
XOR
XOR (GRAPHICS PUT)
YEAR
ZSTRING
[Source]
[History]
2017-08-16 15:58:39
Owner: SysOp
Your hostname is ...
Valid XHTML :: Valid CSS: :: Powered by WikkaWiki