8 private StreamReader stream;
14 public class EOFException : Exception {
17 public enum TokenType {
29 private StringBuilder TokenStringBuilder;
30 public string TokenString {
31 get { return TokenStringBuilder.ToString(); }
33 public int LineNumber;
35 public Lexer(StreamReader stream) {
37 buffer = new char[1025];
41 public TokenType GetNextToken() {
43 while(Char.IsWhiteSpace(c)) {
49 TokenStringBuilder = new StringBuilder();
61 return GetNextToken();
64 return TokenType.OPEN_PAREN;
67 return TokenType.CLOSE_PAREN;
69 int startline = LineNumber;
86 TokenStringBuilder.Append(c);
89 return TokenType.STRING;
93 while(Char.IsLetterOrDigit(c) || c == '_') {
94 TokenStringBuilder.Append(c);
97 if(TokenString == "t")
98 return TokenType.TRUE;
99 if(TokenString == "f")
100 return TokenType.FALSE;
102 throw new Exception("Unknown constant '"
103 + TokenString + "'");
105 if(Char.IsDigit(c) || c == '-') {
106 bool have_nondigits = false;
107 bool have_digits = false;
108 int have_floating_point = 0;
114 have_floating_point++;
115 else if(Char.IsLetter(c) || c == '_')
116 have_nondigits = true;
118 TokenStringBuilder.Append(c);
120 } while(!Char.IsWhiteSpace(c) && c != '\"' && c != '('
121 && c != ')' && c != ';');
123 if(have_nondigits || !have_digits
124 || have_floating_point > 1)
125 return TokenType.SYMBOL;
126 else if(have_floating_point == 1)
127 return TokenType.REAL;
129 return TokenType.INTEGER;
132 TokenStringBuilder.Append(c);
134 } while(!Char.IsWhiteSpace(c) && c != '\"' && c != '('
135 && c != ')' && c != ';');
137 return TokenType.SYMBOL;
140 } catch(EOFException e) {
141 return TokenType.EOF;
145 private void NextChar() {
146 if(bufpos >= buflen) {
147 if(!stream.BaseStream.CanRead)
148 throw new EOFException();
149 buflen = stream.Read(buffer, 0, 1024);
151 // following hack appends an additional ' ' at the end of the file
152 // to avoid problems when parsing symbols/elements and a sudden EOF:
153 // This way we can avoid the need for an unget function.
154 if(!stream.BaseStream.CanRead) {
155 buffer[buflen] = ' ';
159 c = buffer[bufpos++];