9 private StreamReader stream;
10 private char[] buffer;
15 public class EOFException : Exception {
18 public enum TokenType {
30 private StringBuilder TokenStringBuilder;
31 public string TokenString {
32 get { return TokenStringBuilder.ToString(); }
34 public int LineNumber;
36 public Lexer(StreamReader stream) {
38 buffer = new char[1025];
42 public TokenType GetNextToken() {
44 while(Char.IsWhiteSpace(c)) {
50 TokenStringBuilder = new StringBuilder();
62 return GetNextToken();
65 return TokenType.OPEN_PAREN;
68 return TokenType.CLOSE_PAREN;
70 int startline = LineNumber;
87 TokenStringBuilder.Append(c);
90 return TokenType.STRING;
94 while(Char.IsLetterOrDigit(c) || c == '_') {
95 TokenStringBuilder.Append(c);
98 if(TokenString == "t")
99 return TokenType.TRUE;
100 if(TokenString == "f")
101 return TokenType.FALSE;
103 throw new Exception("Unknown constant '"
104 + TokenString + "'");
106 if(Char.IsDigit(c) || c == '-') {
107 bool have_nondigits = false;
108 bool have_digits = false;
109 int have_floating_point = 0;
115 have_floating_point++;
116 else if(Char.IsLetter(c) || c == '_')
117 have_nondigits = true;
119 TokenStringBuilder.Append(c);
121 } while(!Char.IsWhiteSpace(c) && c != '\"' && c != '('
122 && c != ')' && c != ';');
124 if(have_nondigits || !have_digits
125 || have_floating_point > 1)
126 return TokenType.SYMBOL;
127 else if(have_floating_point == 1)
128 return TokenType.REAL;
130 return TokenType.INTEGER;
133 TokenStringBuilder.Append(c);
135 } while(!Char.IsWhiteSpace(c) && c != '\"' && c != '('
136 && c != ')' && c != ';');
138 return TokenType.SYMBOL;
141 } catch(EOFException) {
142 return TokenType.EOF;
146 private void NextChar() {
147 if(bufpos >= buflen) {
148 if(!stream.BaseStream.CanRead)
149 throw new EOFException();
150 buflen = stream.Read(buffer, 0, 1024);
152 // following hack appends an additional ' ' at the end of the file
153 // to avoid problems when parsing symbols/elements and a sudden EOF:
154 // This way we can avoid the need for an unget function.
155 if(!stream.BaseStream.CanRead) {
156 buffer[buflen] = ' ';
160 c = buffer[bufpos++];