octave-lyh: libinterp/parse-tree/lex.ll comparison

comparison libinterp/parse-tree/lex.ll @ 16255:12bf6a3f8c45

store more info in token value class * token.h, token.cc: Store token ID and trailing space info * lex.h, lex.ll (lexical_feedback::token_cache): Handle storing and retrieving extra info in the lexer.

author	John W. Eaton <jwe@octave.org>
date	Mon, 11 Mar 2013 14:08:50 -0400
parents	a89cf57ba3a5
children	b28062b977fd db7f07b22b9b

comparison

equal deleted inserted replaced

-:a89cf57ba3a5
+:12bf6a3f8c45
 bool full_line_comment = curr_lexer->current_input_column == 1;
 curr_lexer->input_line_number++;
 curr_lexer->current_input_column = 1;
+bool have_space = false;
 size_t len = yyleng;
 size_t i = 0;
 while (i < len)
 {
 char c = yytext[i];
-if (c == '#' || c == '%' || c == ' ' || c == '\t')
+if (c == ' ' || c == '\t')
+{
+have_space = true;
+i++;
+}
+else
+break;
+}
+while (i < len)
+{
+char c = yytext[i];
+if (c == '#' || c == '%')
 i++;
 else
 break;
 }
 curr_lexer->comment_text += &yytext[i];
 if (! full_line_comment)
 {
+if (have_space)
+curr_lexer->mark_previous_token_trailing_space ();
 curr_lexer->finish_comment (octave_comment_elt::end_of_line);
 curr_lexer->pop_start_state ();
 }
 }
 // the <MATRIX_START> start state code above.
 %}
 {S}* {
 curr_lexer->current_input_column += yyleng;
+curr_lexer->mark_previous_token_trailing_space ();
 }
 %{
 // Continuation lines.  Allow comments after continuations.
 %}
 nesting_level.reset ();
 tokens.clear ();
 }
+int
+lexical_feedback::previous_token_value (void) const
+{
+const token *tok = tokens.front ();
+return tok ? tok->token_value () : 0;
+}
+bool
+lexical_feedback::previous_token_value_is (int tok_val) const
+{
+const token *tok = tokens.front ();
+return tok ? tok->token_value_is (tok_val) : false;
+}
+void
+lexical_feedback::mark_previous_token_trailing_space (void)
+{
+token *tok = tokens.front ();
+if (tok && ! previous_token_value_is ('\n'))
+tok->mark_trailing_space ();
+}
+bool
+lexical_feedback::space_follows_previous_token (void) const
+{
+const token *tok = tokens.front ();
+return tok ? tok->space_follows_token () : false;
+}
 static bool
 looks_like_copyright (const std::string& s)
 {
 bool retval = false;
 && (defining_func
 && ! (looking_at_return_list
 || parsed_function_name.top ()))))
 return 0;
-tok_val = new token (token::simple_end, l, c);
+tok_val = new token (end_kw, token::simple_end, l, c);
 at_beginning_of_statement = true;
 break;
 case end_try_catch_kw:
-tok_val = new token (token::try_catch_end, l, c);
+tok_val = new token (end_try_catch_kw, token::try_catch_end, l, c);
 at_beginning_of_statement = true;
 break;
 case end_unwind_protect_kw:
-tok_val = new token (token::unwind_protect_end, l, c);
+tok_val = new token (end_unwind_protect_kw,
+token::unwind_protect_end, l, c);
 at_beginning_of_statement = true;
 break;
 case endfor_kw:
-tok_val = new token (token::for_end, l, c);
+tok_val = new token (endfor_kw, token::for_end, l, c);
 at_beginning_of_statement = true;
 break;
 case endfunction_kw:
-tok_val = new token (token::function_end, l, c);
+tok_val = new token (endfunction_kw, token::function_end, l, c);
 at_beginning_of_statement = true;
 break;
 case endif_kw:
-tok_val = new token (token::if_end, l, c);
+tok_val = new token (endif_kw, token::if_end, l, c);
 at_beginning_of_statement = true;
 break;
 case endparfor_kw:
-tok_val = new token (token::parfor_end, l, c);
+tok_val = new token (endparfor_kw, token::parfor_end, l, c);
 at_beginning_of_statement = true;
 break;
 case endswitch_kw:
-tok_val = new token (token::switch_end, l, c);
+tok_val = new token (endswitch_kw, token::switch_end, l, c);
 at_beginning_of_statement = true;
 break;
 case endwhile_kw:
-tok_val = new token (token::while_end, l, c);
+tok_val = new token (endwhile_kw, token::while_end, l, c);
 at_beginning_of_statement = true;
 break;
 case endclassdef_kw:
-tok_val = new token (token::classdef_end, l, c);
+tok_val = new token (endclassdef_kw, token::classdef_end, l, c);
 at_beginning_of_statement = true;
 break;
 case endenumeration_kw:
-tok_val = new token (token::enumeration_end, l, c);
+tok_val = new token (endenumeration_kw, token::enumeration_end,
+l, c);
 at_beginning_of_statement = true;
 break;
 case endevents_kw:
-tok_val = new token (token::events_end, l, c);
+tok_val = new token (endevents_kw, token::events_end, l, c);
 at_beginning_of_statement = true;
 break;
 case endmethods_kw:
-tok_val = new token (token::methods_end, l, c);
+tok_val = new token (endmethods_kw, token::methods_end, l, c);
 at_beginning_of_statement = true;
 break;
 case endproperties_kw:
-tok_val = new token (token::properties_end, l, c);
+tok_val = new token (endproperties_kw, token::properties_end, l, c);
 at_beginning_of_statement = true;
 break;
 case for_kw:
 case magic_file_kw:
 {
 if ((reading_fcn_file || reading_script_file
 || reading_classdef_file)
 && ! fcn_file_full_name.empty ())
-tok_val = new token (fcn_file_full_name, l, c);
+tok_val = new token (magic_file_kw, fcn_file_full_name, l, c);
 else
-tok_val = new token ("stdin", l, c);
+tok_val = new token (magic_file_kw, "stdin", l, c);
 }
 break;
 case magic_line_kw:
-tok_val = new token (static_cast<double> (l), "", l, c);
+tok_val = new token (magic_line_kw, static_cast<double> (l),
+"", l, c);
 break;
 default:
 panic_impossible ();
 }
 if (! tok_val)
-tok_val = new token (l, c);
+tok_val = new token (kw->tok, l, c);
 push_token (tok_val);
 return kw->tok;
 }
 quote_is_transpose = true;
 convert_spaces_to_comma = true;
 looking_for_object_index = false;
 at_beginning_of_statement = false;
-push_token (new token (value, yytxt, input_line_number,
+push_token (new token (NUM, value, yytxt, input_line_number,
 current_input_column));
 current_input_column += flex_yyleng ();
 do_comma_insert_check ();
 offset++;
 }
 else
 break;
 }
+if (have_space)
+mark_previous_token_trailing_space ();
 bool have_comment = false;
 while (offset < yylng)
 {
 char c = yytxt[offset];
 s = do_string_escapes (buf.str ());
 quote_is_transpose = true;
 convert_spaces_to_comma = true;
-push_token (new token (s, bos_line, bos_col));
 if (delim == '"')
 gripe_matlab_incompatible ("\" used as string delimiter");
 else if (delim == '\'')
 gripe_single_quote_string ();
 looking_for_object_index = true;
 at_beginning_of_statement = false;
-return delim == '"' ? DQ_STRING : SQ_STRING;
+int tok = delim == '"' ? DQ_STRING : SQ_STRING;
+push_token (new token (tok, s, bos_line, bos_col));
+return tok;
 }
 }
 }
 else
 {
 {
 error ("method, class and package names may not be keywords");
 return LEXICAL_ERROR;
 }
-push_token (new token (meth.empty () ? 0 : &(symbol_table::insert (meth)),
+push_token (new token (SUPERCLASSREF,
+meth.empty () ? 0 : &(symbol_table::insert (meth)),
 cls.empty () ? 0 : &(symbol_table::insert (cls)),
 pkg.empty () ? 0 : &(symbol_table::insert (pkg)),
 input_line_number, current_input_column));
 convert_spaces_to_comma = true;
 {
 error ("class and package names may not be keywords");
 return LEXICAL_ERROR;
 }
-push_token (new token (cls.empty () ? 0 : &(symbol_table::insert (cls)),
+push_token (new token (METAQUERY,
+cls.empty () ? 0 : &(symbol_table::insert (cls)),
 pkg.empty () ? 0 : &(symbol_table::insert (pkg)),
 input_line_number, current_input_column));
 convert_spaces_to_comma = true;
 current_input_column += flex_yyleng ();
 {
 do_comma_insert_check ();
 maybe_unput_comma (spc_gobbled);
-push_token (new token (tok, input_line_number,
+push_token (new token (STRUCT_ELT, tok, input_line_number,
 current_input_column));
 quote_is_transpose = true;
 convert_spaces_to_comma = true;
 looking_for_object_index = true;
 return LEXICAL_ERROR;
 }
 else
 {
-push_token (new token (tok, input_line_number,
+push_token (new token (FCN_HANDLE, tok, input_line_number,
 current_input_column));
 current_input_column += flex_yyleng ();
 quote_is_transpose = false;
 convert_spaces_to_comma = true;
 // transformation of the end keyword...
 if (tok == "end")
 tok = "__end__";
-push_token (new token (&(symbol_table::insert (tok)),
+push_token (new token (NAME, &(symbol_table::insert (tok)),
 input_line_number, current_input_column));
 // After seeing an identifer, it is ok to convert spaces to a comma
 // (if needed).
 lexer_debug (pattern);
 if (! compat)
 gripe_matlab_incompatible_operator (flex_yytext ());
-push_token (new token (input_line_number, current_input_column));
+push_token (new token (tok, input_line_number, current_input_column));
 current_input_column += flex_yyleng ();
 quote_is_transpose = qit;
 convert_spaces_to_comma = convert;
 looking_for_object_index = false;
 }
 int
 octave_lexer::handle_token (const std::string& name, int tok)
 {
-token *tok_val = new token (name, input_line_number, current_input_column);
+token *tok_val = new token (tok, name, input_line_number,
+current_input_column);
 return handle_token (tok, tok_val);
 }
 int
 octave_lexer::handle_token (int tok, token *tok_val)
 {
 if (! tok_val)
-tok_val = new token (input_line_number, current_input_column);
+tok_val = new token (tok, input_line_number, current_input_column);
 push_token (tok_val);
 current_input_column += flex_yyleng ();
 quote_is_transpose = false;

Mercurial > hg > octave-lyh

comparison libinterp/parse-tree/lex.ll @ 16255:12bf6a3f8c45