src/squirrel/squirrel/sqlexer.cpp

   1 /*
   2         see copyright notice in squirrel.h
   3 */
   4 #include "sqpcheader.h"
   5 #include <ctype.h>
   6 #include <stdlib.h>
   7 #include "sqtable.h"
   8 #include "sqstring.h"
   9 #include "sqcompiler.h"
  10 #include "sqlexer.h"
  11
  12 #define CUR_CHAR (_currdata)
  13 #define RETURN_TOKEN(t) { _prevtoken = _curtoken; _curtoken = t; return t;}
  14 #define IS_EOB() (CUR_CHAR <= SQUIRREL_EOB)
  15 #define NEXT() {Next();_currentcolumn++;}
  16 #define INIT_TEMP_STRING() { _longstr.resize(0);}
  17 #define APPEND_CHAR(c) { _longstr.push_back(c);}
  18 #define TERMINATE_BUFFER() {_longstr.push_back(_SC('\0'));}
  19 #define ADD_KEYWORD(key,id) _keywords->NewSlot( SQString::Create(ss, _SC(#key)) ,SQInteger(id))
  20
  21 SQLexer::SQLexer(){}
  22 SQLexer::~SQLexer()
  23 {
  24         _keywords->Release();
  25 }
  26
  27 void SQLexer::Init(SQSharedState *ss, SQLEXREADFUNC rg, SQUserPointer up)
  28 {
  29         _sharedstate = ss;
  30         _keywords = SQTable::Create(ss, 26);
  31         ADD_KEYWORD(while, TK_WHILE);
  32         ADD_KEYWORD(do, TK_DO);
  33         ADD_KEYWORD(if, TK_IF);
  34         ADD_KEYWORD(else, TK_ELSE);
  35         ADD_KEYWORD(break, TK_BREAK);
  36         ADD_KEYWORD(continue, TK_CONTINUE);
  37         ADD_KEYWORD(return, TK_RETURN);
  38         ADD_KEYWORD(null, TK_NULL);
  39         ADD_KEYWORD(function, TK_FUNCTION);
  40         ADD_KEYWORD(local, TK_LOCAL);
  41         ADD_KEYWORD(for, TK_FOR);
  42         ADD_KEYWORD(foreach, TK_FOREACH);
  43         ADD_KEYWORD(in, TK_IN);
  44         ADD_KEYWORD(typeof, TK_TYPEOF);
  45         ADD_KEYWORD(delegate, TK_DELEGATE);
  46         ADD_KEYWORD(delete, TK_DELETE);
  47         ADD_KEYWORD(try, TK_TRY);
  48         ADD_KEYWORD(catch, TK_CATCH);
  49         ADD_KEYWORD(throw, TK_THROW);
  50         ADD_KEYWORD(clone, TK_CLONE);
  51         ADD_KEYWORD(yield, TK_YIELD);
  52         ADD_KEYWORD(resume, TK_RESUME);
  53         ADD_KEYWORD(switch, TK_SWITCH);
  54         ADD_KEYWORD(case, TK_CASE);
  55         ADD_KEYWORD(default, TK_DEFAULT);
  56         ADD_KEYWORD(this, TK_THIS);
  57         ADD_KEYWORD(parent,TK_PARENT);
  58         ADD_KEYWORD(class,TK_CLASS);
  59         ADD_KEYWORD(extends,TK_EXTENDS);
  60         ADD_KEYWORD(constructor,TK_CONSTRUCTOR);
  61         ADD_KEYWORD(instanceof,TK_INSTANCEOF);
  62         ADD_KEYWORD(vargc,TK_VARGC);
  63         ADD_KEYWORD(vargv,TK_VARGV);
  64         ADD_KEYWORD(true,TK_TRUE);
  65         ADD_KEYWORD(false,TK_FALSE);
  66
  67         _readf = rg;
  68         _up = up;
  69         _lasttokenline = _currentline = 1;
  70         _currentcolumn = 0;
  71         _prevtoken = -1;
  72         Next();
  73 }
  74
  75 void SQLexer::Next()
  76 {
  77         SQInteger t = _readf(_up);
  78         if(t > MAX_CHAR) throw ParserException(_SC("Invalid character"));
  79         if(t != 0) {
  80                 _currdata = t;
  81                 return;
  82         }
  83         _currdata = SQUIRREL_EOB;
  84 }
  85
  86 SQObjectPtr SQLexer::Tok2Str(int tok)
  87 {
  88         SQObjectPtr itr, key, val;
  89         int nitr;
  90         while((nitr = _keywords->Next(itr, key, val)) != -1) {
  91                 itr = (SQInteger)nitr;
  92                 if(((int)_integer(val)) == tok)
  93                         return key;
  94         }
  95         return SQObjectPtr();
  96 }
  97
  98 void SQLexer::LexBlockComment()
  99 {
 100         bool done = false;
 101         while(!done) {
 102                 switch(CUR_CHAR) {
 103                         case _SC('*'): { NEXT(); if(CUR_CHAR == _SC('/')) { done = true; NEXT(); }}; continue;
 104                         //case _SC('/'): { NEXT(); if(CUR_CHAR == _SC('*')) { nest++; NEXT(); }}; continue;
 105                         case _SC('\n'): _currentline++; NEXT(); continue;
 106                         case SQUIRREL_EOB: throw ParserException(_SC("missing \"*/\" in comment"));
 107                         default: NEXT();
 108                 }
 109         }
 110 }
 111
 112 int SQLexer::Lex()
 113 {
 114         _lasttokenline = _currentline;
 115         while(CUR_CHAR != SQUIRREL_EOB) {
 116                 switch(CUR_CHAR){
 117                 case _SC('\t'): case _SC('\r'): case _SC(' '): NEXT(); continue;
 118                 case _SC('\n'):
 119                         _currentline++;
 120                         _prevtoken=_curtoken;
 121                         _curtoken=_SC('\n');
 122                         NEXT();
 123                         _currentcolumn=1;
 124                         continue;
 125                 case _SC('/'):
 126                         NEXT();
 127                         switch(CUR_CHAR){
 128                         case _SC('*'):
 129                                 NEXT();
 130                                 LexBlockComment();
 131                                 continue;
 132                         case _SC('/'):
 133                                 do { NEXT(); } while (CUR_CHAR != _SC('\n') && (!IS_EOB()));
 134                                 continue;
 135                         case _SC('='):
 136                                 NEXT();
 137                                 RETURN_TOKEN(TK_DIVEQ);
 138                                 continue;
 139                         case _SC('>'):
 140                                 NEXT();
 141                                 RETURN_TOKEN(TK_ATTR_CLOSE);
 142                                 continue;
 143                         default:
 144                                 RETURN_TOKEN('/');
 145                         }
 146                 case _SC('='):
 147                         NEXT();
 148                         if (CUR_CHAR != _SC('=')){ RETURN_TOKEN('=') }
 149                         else { NEXT(); RETURN_TOKEN(TK_EQ); }
 150                 case _SC('<'):
 151                         NEXT();
 152                         if ( CUR_CHAR == _SC('=') ) { NEXT(); RETURN_TOKEN(TK_LE) }
 153                         else if ( CUR_CHAR == _SC('-') ) { NEXT(); RETURN_TOKEN(TK_NEWSLOT); }
 154                         else if ( CUR_CHAR == _SC('<') ) { NEXT(); RETURN_TOKEN(TK_SHIFTL); }
 155                         else if ( CUR_CHAR == _SC('/') ) { NEXT(); RETURN_TOKEN(TK_ATTR_OPEN); }
 156                         //else if ( CUR_CHAR == _SC('[') ) { NEXT(); ReadMultilineString(); RETURN_TOKEN(TK_STRING_LITERAL); }
 157                         else { RETURN_TOKEN('<') }
 158                 case _SC('>'):
 159                         NEXT();
 160                         if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_GE);}
 161                         else if(CUR_CHAR == _SC('>')){
 162                                 NEXT();
 163                                 if(CUR_CHAR == _SC('>')){
 164                                         NEXT();
 165                                         RETURN_TOKEN(TK_USHIFTR);
 166                                 }
 167                                 RETURN_TOKEN(TK_SHIFTR);
 168                         }
 169                         else { RETURN_TOKEN('>') }
 170                 case _SC('!'):
 171                         NEXT();
 172                         if (CUR_CHAR != _SC('=')){ RETURN_TOKEN('!')}
 173                         else { NEXT(); RETURN_TOKEN(TK_NE); }
 174                 case _SC('@'): {
 175                         int stype;
 176                         NEXT();
 177                         if(CUR_CHAR != _SC('"'))
 178                                 throw ParserException(_SC("string expected"));
 179                         if((stype=ReadString('"',true))!=-1) {
 180                                 RETURN_TOKEN(stype);
 181                         }
 182                         throw ParserException(_SC("error parsing the string"));
 183                                            }
 184                 case _SC('"'):
 185                 case _SC('\''): {
 186                         int stype;
 187                         if((stype=ReadString(CUR_CHAR,false))!=-1){
 188                                 RETURN_TOKEN(stype);
 189                         }
 190                         throw ParserException(_SC("error parsing the string"));
 191                         }
 192                 case _SC('{'): case _SC('}'): case _SC('('): case _SC(')'): case _SC('['): case _SC(']'):
 193                 case _SC(';'): case _SC(','): case _SC('?'): case _SC('^'): case _SC('~'):
 194                         {int ret = CUR_CHAR;
 195                         NEXT(); RETURN_TOKEN(ret); }
 196                 case _SC('.'):
 197                         NEXT();
 198                         if (CUR_CHAR != _SC('.')){ RETURN_TOKEN('.') }
 199                         NEXT();
 200                         if (CUR_CHAR != _SC('.')){ throw ParserException(_SC("invalid token '..'")); }
 201                         NEXT();
 202                         RETURN_TOKEN(TK_VARPARAMS);
 203                 case _SC('&'):
 204                         NEXT();
 205                         if (CUR_CHAR != _SC('&')){ RETURN_TOKEN('&') }
 206                         else { NEXT(); RETURN_TOKEN(TK_AND); }
 207                 case _SC('|'):
 208                         NEXT();
 209                         if (CUR_CHAR != _SC('|')){ RETURN_TOKEN('|') }
 210                         else { NEXT(); RETURN_TOKEN(TK_OR); }
 211                 case _SC(':'):
 212                         NEXT();
 213                         if (CUR_CHAR != _SC(':')){ RETURN_TOKEN(':') }
 214                         else { NEXT(); RETURN_TOKEN(TK_DOUBLE_COLON); }
 215                 case _SC('*'):
 216                         NEXT();
 217                         if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_MULEQ);}
 218                         else RETURN_TOKEN('*');
 219                 case _SC('%'):
 220                         NEXT();
 221                         if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_MODEQ);}
 222                         else RETURN_TOKEN('%');
 223                 case _SC('-'):
 224                         NEXT();
 225                         if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_MINUSEQ);}
 226                         else if  (CUR_CHAR == _SC('-')){ NEXT(); RETURN_TOKEN(TK_MINUSMINUS);}
 227                         else RETURN_TOKEN('-');
 228                 case _SC('+'):
 229                         NEXT();
 230                         if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_PLUSEQ);}
 231                         else if (CUR_CHAR == _SC('+')){ NEXT(); RETURN_TOKEN(TK_PLUSPLUS);}
 232                         else RETURN_TOKEN('+');
 233                 case SQUIRREL_EOB:
 234                         return 0;
 235                 default:{
 236                                 if (scisdigit(CUR_CHAR)) {
 237                                         int ret = ReadNumber();
 238                                         RETURN_TOKEN(ret);
 239                                 }
 240                                 else if (scisalpha(CUR_CHAR) || CUR_CHAR == _SC('_')) {
 241                                         int t = ReadID();
 242                                         RETURN_TOKEN(t);
 243                                 }
 244                                 else {
 245                                         int c = CUR_CHAR;
 246                                         if (sciscntrl(c)) throw ParserException(_SC("unexpected character(control)"));
 247                                         NEXT();
 248                                         RETURN_TOKEN(c);
 249                                 }
 250                                 RETURN_TOKEN(0);
 251                         }
 252                 }
 253         }
 254         return 0;
 255 }
 256
 257 int SQLexer::GetIDType(SQChar *s)
 258 {
 259         SQObjectPtr t;
 260         if(_keywords->Get(SQString::Create(_sharedstate, s), t)) {
 261                 return int(_integer(t));
 262         }
 263         return TK_IDENTIFIER;
 264 }
 265
 266
 267 int SQLexer::ReadString(int ndelim,bool verbatim)
 268 {
 269         INIT_TEMP_STRING();
 270         NEXT();
 271         if(IS_EOB()) return -1;
 272         for(;;) {
 273                 while(CUR_CHAR != ndelim) {
 274                         switch(CUR_CHAR) {
 275                         case SQUIRREL_EOB:
 276                                 throw ParserException(_SC("unfinished string"));
 277                                 return -1;
 278                         case _SC('\n'):
 279                                 if(!verbatim) throw ParserException(_SC("newline in a constant"));
 280                                 APPEND_CHAR(CUR_CHAR); NEXT();
 281                                 break;
 282                         case _SC('\\'):
 283                                 if(verbatim) {
 284                                         APPEND_CHAR('\\'); NEXT();
 285                                 }
 286                                 else {
 287
 288                                         NEXT();
 289                                         switch(CUR_CHAR) {
 290                                         case _SC('t'): APPEND_CHAR(_SC('\t')); NEXT(); break;
 291                                         case _SC('a'): APPEND_CHAR(_SC('\a')); NEXT(); break;
 292                                         case _SC('b'): APPEND_CHAR(_SC('\b')); NEXT(); break;
 293                                         case _SC('n'): APPEND_CHAR(_SC('\n')); NEXT(); break;
 294                                         case _SC('r'): APPEND_CHAR(_SC('\r')); NEXT(); break;
 295                                         case _SC('v'): APPEND_CHAR(_SC('\v')); NEXT(); break;
 296                                         case _SC('f'): APPEND_CHAR(_SC('\f')); NEXT(); break;
 297                                         case _SC('0'): APPEND_CHAR(_SC('\0')); NEXT(); break;
 298                                         case _SC('\\'): APPEND_CHAR(_SC('\\')); NEXT(); break;
 299                                         case _SC('"'): APPEND_CHAR(_SC('"')); NEXT(); break;
 300                                         case _SC('\''): APPEND_CHAR(_SC('\'')); NEXT(); break;
 301                                         default:
 302                                                 throw ParserException(_SC("unrecognised escaper char"));
 303                                         break;
 304                                         }
 305                                 }
 306                                 break;
 307                         default:
 308                                 APPEND_CHAR(CUR_CHAR);
 309                                 NEXT();
 310                         }
 311                 }
 312                 NEXT();
 313                 if(verbatim && CUR_CHAR == '"') { //double quotation
 314                         APPEND_CHAR(CUR_CHAR);
 315                         NEXT();
 316                 }
 317                 else {
 318                         break;
 319                 }
 320         }
 321         TERMINATE_BUFFER();
 322         int len = _longstr.size()-1;
 323         if(ndelim == _SC('\'')) {
 324                 if(len == 0) throw ParserException(_SC("empty constant"));
 325                 if(len > 1) throw ParserException(_SC("constant too long"));
 326                 _nvalue = _longstr[0];
 327                 return TK_INTEGER;
 328         }
 329         _svalue = &_longstr[0];
 330         return TK_STRING_LITERAL;
 331 }
 332
 333 int isexponent(int c) { return c == 'e' || c=='E'; }
 334
 335 int SQLexer::ReadNumber()
 336 {
 337 #define TINT 1
 338 #define TFLOAT 2
 339 #define THEX 3
 340 #define TSCIENTIFIC 4
 341         int type = TINT, firstchar = CUR_CHAR;
 342         //bool isfloat = false;
 343         SQChar *sTemp;
 344         INIT_TEMP_STRING();
 345         NEXT();
 346         if(firstchar == _SC('0') && toupper(CUR_CHAR) == _SC('X')) {
 347                 NEXT();
 348                 type = THEX;
 349                 while(isxdigit(CUR_CHAR)) {
 350                         APPEND_CHAR(CUR_CHAR);
 351                         NEXT();
 352                 }
 353                 if(_longstr.size() > 8) throw ParserException(_SC("Hex number over 8 digits"));
 354         }
 355         else {
 356                 APPEND_CHAR(firstchar);
 357                 while (CUR_CHAR == _SC('.') || scisdigit(CUR_CHAR) || isexponent(CUR_CHAR)) {
 358             if(CUR_CHAR == _SC('.')) type = TFLOAT;
 359                         if(isexponent(CUR_CHAR)) {
 360                                 if(type != TFLOAT) throw ParserException(_SC("invalid numeric format"));
 361                                 type = TSCIENTIFIC;
 362                                 APPEND_CHAR(CUR_CHAR);
 363                                 NEXT();
 364                                 if(CUR_CHAR == '+' || CUR_CHAR == '-'){
 365                                         APPEND_CHAR(CUR_CHAR);
 366                                         NEXT();
 367                                 }
 368                                 if(!scisdigit(CUR_CHAR)) throw ParserException(_SC("exponent expected"));
 369                         }
 370
 371                         APPEND_CHAR(CUR_CHAR);
 372                         NEXT();
 373                 }
 374         }
 375         TERMINATE_BUFFER();
 376         switch(type) {
 377         case TSCIENTIFIC:
 378         case TFLOAT:
 379                 _fvalue = (SQFloat)scstrtod(&_longstr[0],&sTemp);
 380                 return TK_FLOAT;
 381         case TINT:
 382                 _nvalue = (SQInteger)scatoi(&_longstr[0]);
 383                 return TK_INTEGER;
 384         case THEX:
 385                 *((unsigned long *)&_nvalue) = scstrtoul(&_longstr[0],&sTemp,16);
 386                 return TK_INTEGER;
 387         }
 388         return 0;
 389 }
 390
 391 int SQLexer::ReadID()
 392 {
 393         int res;
 394         // int size = 0;
 395         INIT_TEMP_STRING();
 396         do {
 397                 APPEND_CHAR(CUR_CHAR);
 398                 NEXT();
 399         } while(scisalnum(CUR_CHAR) || CUR_CHAR == _SC('_'));
 400         TERMINATE_BUFFER();
 401         res = GetIDType(&_longstr[0]);
 402         if(res == TK_IDENTIFIER) {
 403                 _svalue = &_longstr[0];
 404         }
 405         return res;
 406 }