src/squirrel/squirrel/sqlexer.cpp

   1 /*\r
   2         see copyright notice in squirrel.h\r
   3 */\r
   4 #include "sqpcheader.h"\r
   5 #include <ctype.h>\r
   6 #include <stdlib.h>\r
   7 #include "sqtable.h"\r
   8 #include "sqstring.h"\r
   9 #include "sqcompiler.h"\r
  10 #include "sqlexer.h"\r
  11 \r
  12 #define CUR_CHAR (_currdata)\r
  13 #define RETURN_TOKEN(t) { _prevtoken = _curtoken; _curtoken = t; return t;}\r
  14 #define IS_EOB() (CUR_CHAR <= SQUIRREL_EOB)\r
  15 #define NEXT() {Next();_currentcolumn++;}\r
  16 #define INIT_TEMP_STRING() { _longstr.resize(0);}\r
  17 #define APPEND_CHAR(c) { _longstr.push_back(c);}\r
  18 #define TERMINATE_BUFFER() {_longstr.push_back(_SC('\0'));}\r
  19 #define ADD_KEYWORD(key,id) _keywords->NewSlot( SQString::Create(ss, _SC(#key)) ,SQInteger(id))\r
  20 \r
  21 SQLexer::SQLexer(){}\r
  22 SQLexer::~SQLexer()\r
  23 {\r
  24         _keywords->Release();\r
  25 }\r
  26 \r
  27 void SQLexer::Init(SQSharedState *ss, SQLEXREADFUNC rg, SQUserPointer up)\r
  28 {\r
  29         _sharedstate = ss;\r
  30         _keywords = SQTable::Create(ss, 26);\r
  31         ADD_KEYWORD(while, TK_WHILE);\r
  32         ADD_KEYWORD(do, TK_DO);\r
  33         ADD_KEYWORD(if, TK_IF);\r
  34         ADD_KEYWORD(else, TK_ELSE);\r
  35         ADD_KEYWORD(break, TK_BREAK);\r
  36         ADD_KEYWORD(continue, TK_CONTINUE);\r
  37         ADD_KEYWORD(return, TK_RETURN);\r
  38         ADD_KEYWORD(null, TK_NULL);\r
  39         ADD_KEYWORD(function, TK_FUNCTION);\r
  40         ADD_KEYWORD(local, TK_LOCAL);\r
  41         ADD_KEYWORD(for, TK_FOR);\r
  42         ADD_KEYWORD(foreach, TK_FOREACH);\r
  43         ADD_KEYWORD(in, TK_IN);\r
  44         ADD_KEYWORD(typeof, TK_TYPEOF);\r
  45         ADD_KEYWORD(delegate, TK_DELEGATE);\r
  46         ADD_KEYWORD(delete, TK_DELETE);\r
  47         ADD_KEYWORD(try, TK_TRY);\r
  48         ADD_KEYWORD(catch, TK_CATCH);\r
  49         ADD_KEYWORD(throw, TK_THROW);\r
  50         ADD_KEYWORD(clone, TK_CLONE);\r
  51         ADD_KEYWORD(yield, TK_YIELD);\r
  52         ADD_KEYWORD(resume, TK_RESUME);\r
  53         ADD_KEYWORD(switch, TK_SWITCH);\r
  54         ADD_KEYWORD(case, TK_CASE);\r
  55         ADD_KEYWORD(default, TK_DEFAULT);\r
  56         ADD_KEYWORD(this, TK_THIS);\r
  57         ADD_KEYWORD(parent,TK_PARENT);\r
  58         ADD_KEYWORD(class,TK_CLASS);\r
  59         ADD_KEYWORD(extends,TK_EXTENDS);\r
  60         ADD_KEYWORD(constructor,TK_CONSTRUCTOR);\r
  61         ADD_KEYWORD(instanceof,TK_INSTANCEOF);\r
  62         ADD_KEYWORD(vargc,TK_VARGC);\r
  63         ADD_KEYWORD(vargv,TK_VARGV);\r
  64         ADD_KEYWORD(true,TK_TRUE);\r
  65         ADD_KEYWORD(false,TK_FALSE);\r
  66 \r
  67         _readf = rg;\r
  68         _up = up;\r
  69         _lasttokenline = _currentline = 1;\r
  70         _currentcolumn = 0;\r
  71         _prevtoken = -1;\r
  72         Next();\r
  73 }\r
  74 \r
  75 void SQLexer::Next()\r
  76 {\r
  77         SQInteger t = _readf(_up);\r
  78         if(t > MAX_CHAR) throw ParserException(_SC("Invalid character"));\r
  79         if(t != 0) {\r
  80                 _currdata = t;\r
  81                 return;\r
  82         }\r
  83         _currdata = SQUIRREL_EOB;\r
  84 }\r
  85 \r
  86 SQObjectPtr SQLexer::Tok2Str(int tok)\r
  87 {\r
  88         SQObjectPtr itr, key, val;\r
  89         int nitr;\r
  90         while((nitr = _keywords->Next(itr, key, val)) != -1) {\r
  91                 itr = (SQInteger)nitr;\r
  92                 if(((int)_integer(val)) == tok)\r
  93                         return key;\r
  94         }\r
  95         return SQObjectPtr();\r
  96 }\r
  97 \r
  98 void SQLexer::LexBlockComment()\r
  99 {\r
 100         bool done = false;\r
 101         while(!done) {\r
 102                 switch(CUR_CHAR) {\r
 103                         case _SC('*'): { NEXT(); if(CUR_CHAR == _SC('/')) { done = true; NEXT(); }}; continue;\r
 104                         //case _SC('/'): { NEXT(); if(CUR_CHAR == _SC('*')) { nest++; NEXT(); }}; continue;\r
 105                         case _SC('\n'): _currentline++; NEXT(); continue;\r
 106                         case SQUIRREL_EOB: throw ParserException(_SC("missing \"*/\" in comment"));\r
 107                         default: NEXT();\r
 108                 }\r
 109         }\r
 110 }\r
 111 \r
 112 int SQLexer::Lex()\r
 113 {\r
 114         _lasttokenline = _currentline;\r
 115         while(CUR_CHAR != SQUIRREL_EOB) {\r
 116                 switch(CUR_CHAR){\r
 117                 case _SC('\t'): case _SC('\r'): case _SC(' '): NEXT(); continue;\r
 118                 case _SC('\n'):\r
 119                         _currentline++;\r
 120                         _prevtoken=_curtoken;\r
 121                         _curtoken=_SC('\n');\r
 122                         NEXT();\r
 123                         _currentcolumn=1;\r
 124                         continue;\r
 125                 case _SC('/'):\r
 126                         NEXT();\r
 127                         switch(CUR_CHAR){\r
 128                         case _SC('*'):\r
 129                                 NEXT();\r
 130                                 LexBlockComment();\r
 131                                 continue;       \r
 132                         case _SC('/'):\r
 133                                 do { NEXT(); } while (CUR_CHAR != _SC('\n') && (!IS_EOB()));\r
 134                                 continue;\r
 135                         case _SC('='):\r
 136                                 NEXT();\r
 137                                 RETURN_TOKEN(TK_DIVEQ);\r
 138                                 continue;\r
 139                         case _SC('>'):\r
 140                                 NEXT();\r
 141                                 RETURN_TOKEN(TK_ATTR_CLOSE);\r
 142                                 continue;\r
 143                         default:\r
 144                                 RETURN_TOKEN('/');\r
 145                         }\r
 146                 case _SC('='):\r
 147                         NEXT();\r
 148                         if (CUR_CHAR != _SC('=')){ RETURN_TOKEN('=') }\r
 149                         else { NEXT(); RETURN_TOKEN(TK_EQ); }\r
 150                 case _SC('<'):\r
 151                         NEXT();\r
 152                         if ( CUR_CHAR == _SC('=') ) { NEXT(); RETURN_TOKEN(TK_LE) }\r
 153                         else if ( CUR_CHAR == _SC('-') ) { NEXT(); RETURN_TOKEN(TK_NEWSLOT); }\r
 154                         else if ( CUR_CHAR == _SC('<') ) { NEXT(); RETURN_TOKEN(TK_SHIFTL); }\r
 155                         else if ( CUR_CHAR == _SC('/') ) { NEXT(); RETURN_TOKEN(TK_ATTR_OPEN); }\r
 156                         //else if ( CUR_CHAR == _SC('[') ) { NEXT(); ReadMultilineString(); RETURN_TOKEN(TK_STRING_LITERAL); }\r
 157                         else { RETURN_TOKEN('<') }\r
 158                 case _SC('>'):\r
 159                         NEXT();\r
 160                         if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_GE);}\r
 161                         else if(CUR_CHAR == _SC('>')){ \r
 162                                 NEXT(); \r
 163                                 if(CUR_CHAR == _SC('>')){\r
 164                                         NEXT();\r
 165                                         RETURN_TOKEN(TK_USHIFTR);\r
 166                                 }\r
 167                                 RETURN_TOKEN(TK_SHIFTR);\r
 168                         }\r
 169                         else { RETURN_TOKEN('>') }\r
 170                 case _SC('!'):\r
 171                         NEXT();\r
 172                         if (CUR_CHAR != _SC('=')){ RETURN_TOKEN('!')}\r
 173                         else { NEXT(); RETURN_TOKEN(TK_NE); }\r
 174                 case _SC('@'): {\r
 175                         int stype;\r
 176                         NEXT(); \r
 177                         if(CUR_CHAR != _SC('"'))\r
 178                                 throw ParserException(_SC("string expected"));\r
 179                         if((stype=ReadString('"',true))!=-1) {\r
 180                                 RETURN_TOKEN(stype);\r
 181                         }\r
 182                         throw ParserException(_SC("error parsing the string"));\r
 183                                            }\r
 184                 case _SC('"'):\r
 185                 case _SC('\''): {\r
 186                         int stype;\r
 187                         if((stype=ReadString(CUR_CHAR,false))!=-1){\r
 188                                 RETURN_TOKEN(stype);\r
 189                         }\r
 190                         throw ParserException(_SC("error parsing the string"));\r
 191                         }\r
 192                 case _SC('{'): case _SC('}'): case _SC('('): case _SC(')'): case _SC('['): case _SC(']'):\r
 193                 case _SC(';'): case _SC(','): case _SC('?'): case _SC('^'): case _SC('~'):\r
 194                         {int ret = CUR_CHAR;\r
 195                         NEXT(); RETURN_TOKEN(ret); }\r
 196                 case _SC('.'):\r
 197                         NEXT();\r
 198                         if (CUR_CHAR != _SC('.')){ RETURN_TOKEN('.') }\r
 199                         NEXT();\r
 200                         if (CUR_CHAR != _SC('.')){ throw ParserException(_SC("invalid token '..'")); }\r
 201                         NEXT();\r
 202                         RETURN_TOKEN(TK_VARPARAMS);\r
 203                 case _SC('&'):\r
 204                         NEXT();\r
 205                         if (CUR_CHAR != _SC('&')){ RETURN_TOKEN('&') }\r
 206                         else { NEXT(); RETURN_TOKEN(TK_AND); }\r
 207                 case _SC('|'):\r
 208                         NEXT();\r
 209                         if (CUR_CHAR != _SC('|')){ RETURN_TOKEN('|') }\r
 210                         else { NEXT(); RETURN_TOKEN(TK_OR); }\r
 211                 case _SC(':'):\r
 212                         NEXT();\r
 213                         if (CUR_CHAR != _SC(':')){ RETURN_TOKEN(':') }\r
 214                         else { NEXT(); RETURN_TOKEN(TK_DOUBLE_COLON); }\r
 215                 case _SC('*'):\r
 216                         NEXT();\r
 217                         if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_MULEQ);}\r
 218                         else RETURN_TOKEN('*');\r
 219                 case _SC('%'):\r
 220                         NEXT();\r
 221                         if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_MODEQ);}\r
 222                         else RETURN_TOKEN('%');\r
 223                 case _SC('-'):\r
 224                         NEXT();\r
 225                         if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_MINUSEQ);}\r
 226                         else if  (CUR_CHAR == _SC('-')){ NEXT(); RETURN_TOKEN(TK_MINUSMINUS);}\r
 227                         else RETURN_TOKEN('-');\r
 228                 case _SC('+'):\r
 229                         NEXT();\r
 230                         if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_PLUSEQ);}\r
 231                         else if (CUR_CHAR == _SC('+')){ NEXT(); RETURN_TOKEN(TK_PLUSPLUS);}\r
 232                         else RETURN_TOKEN('+');\r
 233                 case SQUIRREL_EOB:\r
 234                         return 0;\r
 235                 default:{\r
 236                                 if (scisdigit(CUR_CHAR)) {\r
 237                                         int ret = ReadNumber();\r
 238                                         RETURN_TOKEN(ret);\r
 239                                 }\r
 240                                 else if (scisalpha(CUR_CHAR) || CUR_CHAR == _SC('_')) {\r
 241                                         int t = ReadID();\r
 242                                         RETURN_TOKEN(t);\r
 243                                 }\r
 244                                 else {\r
 245                                         int c = CUR_CHAR;\r
 246                                         if (sciscntrl(c)) throw ParserException(_SC("unexpected character(control)"));\r
 247                                         NEXT();\r
 248                                         RETURN_TOKEN(c);  \r
 249                                 }\r
 250                                 RETURN_TOKEN(0);\r
 251                         }\r
 252                 }\r
 253         }\r
 254         return 0;    \r
 255 }\r
 256         \r
 257 int SQLexer::GetIDType(SQChar *s)\r
 258 {\r
 259         SQObjectPtr t;\r
 260         if(_keywords->Get(SQString::Create(_sharedstate, s), t)) {\r
 261                 return int(_integer(t));\r
 262         }\r
 263         return TK_IDENTIFIER;\r
 264 }\r
 265 \r
 266 \r
 267 int SQLexer::ReadString(int ndelim,bool verbatim)\r
 268 {\r
 269         INIT_TEMP_STRING();\r
 270         NEXT();\r
 271         if(IS_EOB()) return -1;\r
 272         for(;;) {\r
 273                 while(CUR_CHAR != ndelim) {\r
 274                         switch(CUR_CHAR) {\r
 275                         case SQUIRREL_EOB:\r
 276                                 throw ParserException(_SC("unfinished string"));\r
 277                                 return -1;\r
 278                         case _SC('\n'): \r
 279                                 if(!verbatim) throw ParserException(_SC("newline in a constant")); \r
 280                                 APPEND_CHAR(CUR_CHAR); NEXT(); \r
 281                                 break;\r
 282                         case _SC('\\'):\r
 283                                 if(verbatim) {\r
 284                                         APPEND_CHAR('\\'); NEXT(); \r
 285                                 }\r
 286                                 else {\r
 287 \r
 288                                         NEXT();\r
 289                                         switch(CUR_CHAR) {\r
 290                                         case _SC('t'): APPEND_CHAR(_SC('\t')); NEXT(); break;\r
 291                                         case _SC('a'): APPEND_CHAR(_SC('\a')); NEXT(); break;\r
 292                                         case _SC('b'): APPEND_CHAR(_SC('\b')); NEXT(); break;\r
 293                                         case _SC('n'): APPEND_CHAR(_SC('\n')); NEXT(); break;\r
 294                                         case _SC('r'): APPEND_CHAR(_SC('\r')); NEXT(); break;\r
 295                                         case _SC('v'): APPEND_CHAR(_SC('\v')); NEXT(); break;\r
 296                                         case _SC('f'): APPEND_CHAR(_SC('\f')); NEXT(); break;\r
 297                                         case _SC('0'): APPEND_CHAR(_SC('\0')); NEXT(); break;\r
 298                                         case _SC('\\'): APPEND_CHAR(_SC('\\')); NEXT(); break;\r
 299                                         case _SC('"'): APPEND_CHAR(_SC('"')); NEXT(); break;\r
 300                                         case _SC('\''): APPEND_CHAR(_SC('\'')); NEXT(); break;\r
 301                                         default:\r
 302                                                 throw ParserException(_SC("unrecognised escaper char"));\r
 303                                         break;\r
 304                                         }\r
 305                                 }\r
 306                                 break;\r
 307                         default:\r
 308                                 APPEND_CHAR(CUR_CHAR);\r
 309                                 NEXT();\r
 310                         }\r
 311                 }\r
 312                 NEXT();\r
 313                 if(verbatim && CUR_CHAR == '"') { //double quotation\r
 314                         APPEND_CHAR(CUR_CHAR);\r
 315                         NEXT();\r
 316                 }\r
 317                 else {\r
 318                         break;\r
 319                 }\r
 320         }\r
 321         TERMINATE_BUFFER();\r
 322         int len = _longstr.size()-1;\r
 323         if(ndelim == _SC('\'')) {\r
 324                 if(len == 0) throw ParserException(_SC("empty constant"));\r
 325                 if(len > 1) throw ParserException(_SC("constant too long"));\r
 326                 _nvalue = _longstr[0];\r
 327                 return TK_INTEGER;\r
 328         }\r
 329         _svalue = &_longstr[0];\r
 330         return TK_STRING_LITERAL;\r
 331 }\r
 332 \r
 333 int isexponent(int c) { return c == 'e' || c=='E'; }\r
 334 \r
 335 int SQLexer::ReadNumber()\r
 336 {\r
 337 #define TINT 1\r
 338 #define TFLOAT 2\r
 339 #define THEX 3\r
 340 #define TSCIENTIFIC 4\r
 341         int type = TINT, firstchar = CUR_CHAR;\r
 342         bool isfloat = false;\r
 343         SQChar *sTemp;\r
 344         INIT_TEMP_STRING();\r
 345         NEXT();\r
 346         if(firstchar == _SC('0') && toupper(CUR_CHAR) == _SC('X')) {\r
 347                 NEXT();\r
 348                 type = THEX;\r
 349                 while(isxdigit(CUR_CHAR)) {\r
 350                         APPEND_CHAR(CUR_CHAR);\r
 351                         NEXT();\r
 352                 }\r
 353                 if(_longstr.size() > 8) throw ParserException(_SC("Hex number over 8 digits"));\r
 354         }\r
 355         else {\r
 356                 APPEND_CHAR(firstchar);\r
 357                 while (CUR_CHAR == _SC('.') || scisdigit(CUR_CHAR) || isexponent(CUR_CHAR)) {\r
 358             if(CUR_CHAR == _SC('.')) type = TFLOAT;\r
 359                         if(isexponent(CUR_CHAR)) {\r
 360                                 if(type != TFLOAT) throw ParserException(_SC("invalid numeric format"));\r
 361                                 type = TSCIENTIFIC;\r
 362                                 APPEND_CHAR(CUR_CHAR);\r
 363                                 NEXT();\r
 364                                 if(CUR_CHAR == '+' || CUR_CHAR == '-'){\r
 365                                         APPEND_CHAR(CUR_CHAR);\r
 366                                         NEXT();\r
 367                                 }\r
 368                                 if(!scisdigit(CUR_CHAR)) throw ParserException(_SC("exponent expected"));\r
 369                         }\r
 370                         \r
 371                         APPEND_CHAR(CUR_CHAR);\r
 372                         NEXT();\r
 373                 }\r
 374         }\r
 375         TERMINATE_BUFFER();\r
 376         switch(type) {\r
 377         case TSCIENTIFIC:\r
 378         case TFLOAT:\r
 379                 _fvalue = (SQFloat)scstrtod(&_longstr[0],&sTemp);\r
 380                 return TK_FLOAT;\r
 381         case TINT:\r
 382                 _nvalue = (SQInteger)scatoi(&_longstr[0]);\r
 383                 return TK_INTEGER;\r
 384         case THEX:\r
 385                 *((unsigned long *)&_nvalue) = scstrtoul(&_longstr[0],&sTemp,16);\r
 386                 return TK_INTEGER;\r
 387         }\r
 388         return 0;\r
 389 }\r
 390 \r
 391 int SQLexer::ReadID()\r
 392 {\r
 393         int res, size = 0;\r
 394         INIT_TEMP_STRING();\r
 395         do {\r
 396                 APPEND_CHAR(CUR_CHAR);\r
 397                 NEXT();\r
 398         } while(scisalnum(CUR_CHAR) || CUR_CHAR == _SC('_'));\r
 399         TERMINATE_BUFFER();\r
 400         res = GetIDType(&_longstr[0]);\r
 401         if(res == TK_IDENTIFIER) {\r
 402                 _svalue = &_longstr[0];\r
 403         }\r
 404         return res;\r
 405 }\r