1 /*
2 ** $Id: llex.c,v 2.63.1.2 2013/08/30 15:49:41 roberto Exp $
3 ** Lexical Analyzer
4 ** See Copyright Notice in lua.h
5 */
6 
7 
8 #include <locale.h>
9 #include <string.h>
10 
11 #define llex_c
12 #define LUA_CORE
13 
14 #include "lua.h"
15 
16 #include "lctype.h"
17 #include "ldo.h"
18 #include "llex.h"
19 #include "lobject.h"
20 #include "lparser.h"
21 #include "lstate.h"
22 #include "lstring.h"
23 #include "ltable.h"
24 #include "lzio.h"
25 
26 
27 
28 #define next(ls) (ls->current = zgetc(ls->z))
29 
30 
31 
32 #define currIsNewline(ls)	(ls->current == '\n' || ls->current == '\r')
33 
34 
35 /* ORDER RESERVED */
36 static const char *const luaX_tokens [] = {
37     "and", "break", "do", "else", "elseif",
38     "end", "false", "for", "function", "goto", "if",
39     "in", "local", "nil", "not", "or", "repeat",
40     "return", "then", "true", "until", "while",
41     "..", "...", "==", ">=", "<=", "~=", "::", "<eof>",
42     "<number>", "<name>", "<string>"
43 };
44 
45 
46 #define save_and_next(ls) (save(ls, ls->current), next(ls))
47 
48 
49 static l_noret lexerror (LexState *ls, const char *msg, int token);
50 
51 
save(LexState * ls,int c)52 static void save (LexState *ls, int c) {
53   Mbuffer *b = ls->buff;
54   if (luaZ_bufflen(b) + 1 > luaZ_sizebuffer(b)) {
55     size_t newsize;
56     if (luaZ_sizebuffer(b) >= MAX_SIZET/2)
57       lexerror(ls, "lexical element too long", 0);
58     newsize = luaZ_sizebuffer(b) * 2;
59     luaZ_resizebuffer(ls->L, b, newsize);
60   }
61   b->buffer[luaZ_bufflen(b)++] = cast(char, c);
62 }
63 
64 
luaX_init(lua_State * L)65 void luaX_init (lua_State *L) {
66   int i;
67   for (i=0; i<NUM_RESERVED; i++) {
68     TString *ts = luaS_new(L, luaX_tokens[i]);
69     luaS_fix(ts);  /* reserved words are never collected */
70     ts->tsv.extra = cast_byte(i+1);  /* reserved word */
71   }
72 }
73 
74 
luaX_token2str(LexState * ls,int token)75 const char *luaX_token2str (LexState *ls, int token) {
76   if (token < FIRST_RESERVED) {  /* single-byte symbols? */
77     lua_assert(token == cast(unsigned char, token));
78     return (lisprint(token)) ? luaO_pushfstring(ls->L, LUA_QL("%c"), token) :
79                               luaO_pushfstring(ls->L, "char(%d)", token);
80   }
81   else {
82     const char *s = luaX_tokens[token - FIRST_RESERVED];
83     if (token < TK_EOS)  /* fixed format (symbols and reserved words)? */
84       return luaO_pushfstring(ls->L, LUA_QS, s);
85     else  /* names, strings, and numerals */
86       return s;
87   }
88 }
89 
90 
txtToken(LexState * ls,int token)91 static const char *txtToken (LexState *ls, int token) {
92   switch (token) {
93     case TK_NAME:
94     case TK_STRING:
95     case TK_NUMBER:
96       save(ls, '\0');
97       return luaO_pushfstring(ls->L, LUA_QS, luaZ_buffer(ls->buff));
98     default:
99       return luaX_token2str(ls, token);
100   }
101 }
102 
103 
lexerror(LexState * ls,const char * msg,int token)104 static l_noret lexerror (LexState *ls, const char *msg, int token) {
105   char buff[LUA_IDSIZE];
106   luaO_chunkid(buff, getstr(ls->source), LUA_IDSIZE);
107   msg = luaO_pushfstring(ls->L, "%s:%d: %s", buff, ls->linenumber, msg);
108   if (token)
109     luaO_pushfstring(ls->L, "%s near %s", msg, txtToken(ls, token));
110   luaD_throw(ls->L, LUA_ERRSYNTAX);
111 }
112 
113 
luaX_syntaxerror(LexState * ls,const char * msg)114 l_noret luaX_syntaxerror (LexState *ls, const char *msg) {
115   lexerror(ls, msg, ls->t.token);
116 }
117 
118 
119 /*
120 ** creates a new string and anchors it in function's table so that
121 ** it will not be collected until the end of the function's compilation
122 ** (by that time it should be anchored in function's prototype)
123 */
luaX_newstring(LexState * ls,const char * str,size_t l)124 TString *luaX_newstring (LexState *ls, const char *str, size_t l) {
125   lua_State *L = ls->L;
126   TValue *o;  /* entry for `str' */
127   TString *ts = luaS_newlstr(L, str, l);  /* create new string */
128   setsvalue2s(L, L->top++, ts);  /* temporarily anchor it in stack */
129   o = luaH_set(L, ls->fs->h, L->top - 1);
130   if (ttisnil(o)) {  /* not in use yet? (see 'addK') */
131     /* boolean value does not need GC barrier;
132        table has no metatable, so it does not need to invalidate cache */
133     setbvalue(o, 1);  /* t[string] = true */
134     luaC_checkGC(L);
135   }
136   else {  /* string already present */
137     ts = rawtsvalue(keyfromval(o));  /* re-use value previously stored */
138   }
139   L->top--;  /* remove string from stack */
140   return ts;
141 }
142 
143 
144 /*
145 ** increment line number and skips newline sequence (any of
146 ** \n, \r, \n\r, or \r\n)
147 */
inclinenumber(LexState * ls)148 static void inclinenumber (LexState *ls) {
149   int old = ls->current;
150   lua_assert(currIsNewline(ls));
151   next(ls);  /* skip `\n' or `\r' */
152   if (currIsNewline(ls) && ls->current != old)
153     next(ls);  /* skip `\n\r' or `\r\n' */
154   if (++ls->linenumber >= MAX_INT)
155     luaX_syntaxerror(ls, "chunk has too many lines");
156 }
157 
158 
luaX_setinput(lua_State * L,LexState * ls,ZIO * z,TString * source,int firstchar)159 void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source,
160                     int firstchar) {
161   ls->decpoint = '.';
162   ls->L = L;
163   ls->current = firstchar;
164   ls->lookahead.token = TK_EOS;  /* no look-ahead token */
165   ls->z = z;
166   ls->fs = NULL;
167   ls->linenumber = 1;
168   ls->lastline = 1;
169   ls->source = source;
170   ls->envn = luaS_new(L, LUA_ENV);  /* create env name */
171   luaS_fix(ls->envn);  /* never collect this name */
172   luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER);  /* initialize buffer */
173 }
174 
175 
176 
177 /*
178 ** =======================================================
179 ** LEXICAL ANALYZER
180 ** =======================================================
181 */
182 
183 
184 
check_next(LexState * ls,const char * set)185 static int check_next (LexState *ls, const char *set) {
186   if (ls->current == '\0' || !strchr(set, ls->current))
187     return 0;
188   save_and_next(ls);
189   return 1;
190 }
191 
192 
193 /*
194 ** change all characters 'from' in buffer to 'to'
195 */
buffreplace(LexState * ls,char from,char to)196 static void buffreplace (LexState *ls, char from, char to) {
197   size_t n = luaZ_bufflen(ls->buff);
198   char *p = luaZ_buffer(ls->buff);
199   while (n--)
200     if (p[n] == from) p[n] = to;
201 }
202 
203 
204 #if !defined(getlocaledecpoint)
205 #define getlocaledecpoint()	(localeconv()->decimal_point[0])
206 #endif
207 
208 
209 #define buff2d(b,e)	luaO_str2d(luaZ_buffer(b), luaZ_bufflen(b) - 1, e)
210 
211 /*
212 ** in case of format error, try to change decimal point separator to
213 ** the one defined in the current locale and check again
214 */
trydecpoint(LexState * ls,SemInfo * seminfo)215 static void trydecpoint (LexState *ls, SemInfo *seminfo) {
216   char old = ls->decpoint;
217   ls->decpoint = getlocaledecpoint();
218   buffreplace(ls, old, ls->decpoint);  /* try new decimal separator */
219   if (!buff2d(ls->buff, &seminfo->r)) {
220     /* format error with correct decimal point: no more options */
221     buffreplace(ls, ls->decpoint, '.');  /* undo change (for error message) */
222     lexerror(ls, "malformed number", TK_NUMBER);
223   }
224 }
225 
226 
227 /* LUA_NUMBER */
228 /*
229 ** this function is quite liberal in what it accepts, as 'luaO_str2d'
230 ** will reject ill-formed numerals.
231 */
read_numeral(LexState * ls,SemInfo * seminfo)232 static void read_numeral (LexState *ls, SemInfo *seminfo) {
233   const char *expo = "Ee";
234   int first = ls->current;
235   lua_assert(lisdigit(ls->current));
236   save_and_next(ls);
237   if (first == '0' && check_next(ls, "Xx"))  /* hexadecimal? */
238     expo = "Pp";
239   for (;;) {
240     if (check_next(ls, expo))  /* exponent part? */
241       check_next(ls, "+-");  /* optional exponent sign */
242     if (lisxdigit(ls->current) || ls->current == '.')
243       save_and_next(ls);
244     else  break;
245   }
246   save(ls, '\0');
247   buffreplace(ls, '.', ls->decpoint);  /* follow locale for decimal point */
248   if (!buff2d(ls->buff, &seminfo->r))  /* format error? */
249     trydecpoint(ls, seminfo); /* try to update decimal point separator */
250 }
251 
252 
253 /*
254 ** skip a sequence '[=*[' or ']=*]' and return its number of '='s or
255 ** -1 if sequence is malformed
256 */
skip_sep(LexState * ls)257 static int skip_sep (LexState *ls) {
258   int count = 0;
259   int s = ls->current;
260   lua_assert(s == '[' || s == ']');
261   save_and_next(ls);
262   while (ls->current == '=') {
263     save_and_next(ls);
264     count++;
265   }
266   return (ls->current == s) ? count : (-count) - 1;
267 }
268 
269 
read_long_string(LexState * ls,SemInfo * seminfo,int sep)270 static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) {
271   save_and_next(ls);  /* skip 2nd `[' */
272   if (currIsNewline(ls))  /* string starts with a newline? */
273     inclinenumber(ls);  /* skip it */
274   for (;;) {
275     switch (ls->current) {
276       case EOZ:
277         lexerror(ls, (seminfo) ? "unfinished long string" :
278                                  "unfinished long comment", TK_EOS);
279         break;  /* to avoid warnings */
280       case ']': {
281         if (skip_sep(ls) == sep) {
282           save_and_next(ls);  /* skip 2nd `]' */
283           goto endloop;
284         }
285         break;
286       }
287       case '\n': case '\r': {
288         save(ls, '\n');
289         inclinenumber(ls);
290         if (!seminfo) luaZ_resetbuffer(ls->buff);  /* avoid wasting space */
291         break;
292       }
293       default: {
294         if (seminfo) save_and_next(ls);
295         else next(ls);
296       }
297     }
298   } endloop:
299   if (seminfo)
300     seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep),
301                                      luaZ_bufflen(ls->buff) - 2*(2 + sep));
302 }
303 
304 
escerror(LexState * ls,int * c,int n,const char * msg)305 static void escerror (LexState *ls, int *c, int n, const char *msg) {
306   int i;
307   luaZ_resetbuffer(ls->buff);  /* prepare error message */
308   save(ls, '\\');
309   for (i = 0; i < n && c[i] != EOZ; i++)
310     save(ls, c[i]);
311   lexerror(ls, msg, TK_STRING);
312 }
313 
314 
readhexaesc(LexState * ls)315 static int readhexaesc (LexState *ls) {
316   int c[3], i;  /* keep input for error message */
317   int r = 0;  /* result accumulator */
318   c[0] = 'x';  /* for error message */
319   for (i = 1; i < 3; i++) {  /* read two hexadecimal digits */
320     c[i] = next(ls);
321     if (!lisxdigit(c[i]))
322       escerror(ls, c, i + 1, "hexadecimal digit expected");
323     r = (r << 4) + luaO_hexavalue(c[i]);
324   }
325   return r;
326 }
327 
328 
readdecesc(LexState * ls)329 static int readdecesc (LexState *ls) {
330   int c[3], i;
331   int r = 0;  /* result accumulator */
332   for (i = 0; i < 3 && lisdigit(ls->current); i++) {  /* read up to 3 digits */
333     c[i] = ls->current;
334     r = 10*r + c[i] - '0';
335     next(ls);
336   }
337   if (r > UCHAR_MAX)
338     escerror(ls, c, i, "decimal escape too large");
339   return r;
340 }
341 
342 
read_string(LexState * ls,int del,SemInfo * seminfo)343 static void read_string (LexState *ls, int del, SemInfo *seminfo) {
344   save_and_next(ls);  /* keep delimiter (for error messages) */
345   while (ls->current != del) {
346     switch (ls->current) {
347       case EOZ:
348         lexerror(ls, "unfinished string", TK_EOS);
349         break;  /* to avoid warnings */
350       case '\n':
351       case '\r':
352         lexerror(ls, "unfinished string", TK_STRING);
353         break;  /* to avoid warnings */
354       case '\\': {  /* escape sequences */
355         int c;  /* final character to be saved */
356         next(ls);  /* do not save the `\' */
357         switch (ls->current) {
358           case 'a': c = '\a'; goto read_save;
359           case 'b': c = '\b'; goto read_save;
360           case 'f': c = '\f'; goto read_save;
361           case 'n': c = '\n'; goto read_save;
362           case 'r': c = '\r'; goto read_save;
363           case 't': c = '\t'; goto read_save;
364           case 'v': c = '\v'; goto read_save;
365           case 'x': c = readhexaesc(ls); goto read_save;
366           case '\n': case '\r':
367             inclinenumber(ls); c = '\n'; goto only_save;
368           case '\\': case '\"': case '\'':
369             c = ls->current; goto read_save;
370           case EOZ: goto no_save;  /* will raise an error next loop */
371           case 'z': {  /* zap following span of spaces */
372             next(ls);  /* skip the 'z' */
373             while (lisspace(ls->current)) {
374               if (currIsNewline(ls)) inclinenumber(ls);
375               else next(ls);
376             }
377             goto no_save;
378           }
379           default: {
380             if (!lisdigit(ls->current))
381               escerror(ls, &ls->current, 1, "invalid escape sequence");
382             /* digital escape \ddd */
383             c = readdecesc(ls);
384             goto only_save;
385           }
386         }
387        read_save: next(ls);  /* read next character */
388        only_save: save(ls, c);  /* save 'c' */
389        no_save: break;
390       }
391       default:
392         save_and_next(ls);
393     }
394   }
395   save_and_next(ls);  /* skip delimiter */
396   seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1,
397                                    luaZ_bufflen(ls->buff) - 2);
398 }
399 
400 
llex(LexState * ls,SemInfo * seminfo)401 static int llex (LexState *ls, SemInfo *seminfo) {
402   luaZ_resetbuffer(ls->buff);
403   for (;;) {
404     switch (ls->current) {
405       case '\n': case '\r': {  /* line breaks */
406         inclinenumber(ls);
407         break;
408       }
409       case ' ': case '\f': case '\t': case '\v': {  /* spaces */
410         next(ls);
411         break;
412       }
413       case '-': {  /* '-' or '--' (comment) */
414         next(ls);
415         if (ls->current != '-') return '-';
416         /* else is a comment */
417         next(ls);
418         if (ls->current == '[') {  /* long comment? */
419           int sep = skip_sep(ls);
420           luaZ_resetbuffer(ls->buff);  /* `skip_sep' may dirty the buffer */
421           if (sep >= 0) {
422             read_long_string(ls, NULL, sep);  /* skip long comment */
423             luaZ_resetbuffer(ls->buff);  /* previous call may dirty the buff. */
424             break;
425           }
426         }
427         /* else short comment */
428         while (!currIsNewline(ls) && ls->current != EOZ)
429           next(ls);  /* skip until end of line (or end of file) */
430         break;
431       }
432       case '[': {  /* long string or simply '[' */
433         int sep = skip_sep(ls);
434         if (sep >= 0) {
435           read_long_string(ls, seminfo, sep);
436           return TK_STRING;
437         }
438         else if (sep == -1) return '[';
439         else lexerror(ls, "invalid long string delimiter", TK_STRING);
440       }
441       case '=': {
442         next(ls);
443         if (ls->current != '=') return '=';
444         else { next(ls); return TK_EQ; }
445       }
446       case '<': {
447         next(ls);
448         if (ls->current != '=') return '<';
449         else { next(ls); return TK_LE; }
450       }
451       case '>': {
452         next(ls);
453         if (ls->current != '=') return '>';
454         else { next(ls); return TK_GE; }
455       }
456       case '~': {
457         next(ls);
458         if (ls->current != '=') return '~';
459         else { next(ls); return TK_NE; }
460       }
461       case ':': {
462         next(ls);
463         if (ls->current != ':') return ':';
464         else { next(ls); return TK_DBCOLON; }
465       }
466       case '"': case '\'': {  /* short literal strings */
467         read_string(ls, ls->current, seminfo);
468         return TK_STRING;
469       }
470       case '.': {  /* '.', '..', '...', or number */
471         save_and_next(ls);
472         if (check_next(ls, ".")) {
473           if (check_next(ls, "."))
474             return TK_DOTS;   /* '...' */
475           else return TK_CONCAT;   /* '..' */
476         }
477         else if (!lisdigit(ls->current)) return '.';
478         /* else go through */
479       }
480       case '0': case '1': case '2': case '3': case '4':
481       case '5': case '6': case '7': case '8': case '9': {
482         read_numeral(ls, seminfo);
483         return TK_NUMBER;
484       }
485       case EOZ: {
486         return TK_EOS;
487       }
488       default: {
489         if (lislalpha(ls->current)) {  /* identifier or reserved word? */
490           TString *ts;
491           do {
492             save_and_next(ls);
493           } while (lislalnum(ls->current));
494           ts = luaX_newstring(ls, luaZ_buffer(ls->buff),
495                                   luaZ_bufflen(ls->buff));
496           seminfo->ts = ts;
497           if (isreserved(ts))  /* reserved word? */
498             return ts->tsv.extra - 1 + FIRST_RESERVED;
499           else {
500             return TK_NAME;
501           }
502         }
503         else {  /* single-char tokens (+ - / ...) */
504           int c = ls->current;
505           next(ls);
506           return c;
507         }
508       }
509     }
510   }
511 }
512 
513 
luaX_next(LexState * ls)514 void luaX_next (LexState *ls) {
515   ls->lastline = ls->linenumber;
516   if (ls->lookahead.token != TK_EOS) {  /* is there a look-ahead token? */
517     ls->t = ls->lookahead;  /* use this one */
518     ls->lookahead.token = TK_EOS;  /* and discharge it */
519   }
520   else
521     ls->t.token = llex(ls, &ls->t.seminfo);  /* read next token */
522 }
523 
524 
luaX_lookahead(LexState * ls)525 int luaX_lookahead (LexState *ls) {
526   lua_assert(ls->lookahead.token == TK_EOS);
527   ls->lookahead.token = llex(ls, &ls->lookahead.seminfo);
528   return ls->lookahead.token;
529 }
530 
531