Mercurial > hg > octave-nkf
changeset 16258:5c32368509a2 classdef
maint: periodic merge of default to classdef
author | John W. Eaton <jwe@octave.org> |
---|---|
date | Mon, 11 Mar 2013 14:17:44 -0400 |
parents | b28062b977fd (current diff) db7f07b22b9b (diff) |
children | 6c211b8cfbd9 |
files | libinterp/parse-tree/lex.h libinterp/parse-tree/lex.ll libinterp/parse-tree/oct-parse.in.yy libinterp/parse-tree/token.cc libinterp/parse-tree/token.h |
diffstat | 5 files changed, 482 insertions(+), 288 deletions(-) [+] |
line wrap: on
line diff
--- a/libinterp/parse-tree/lex.h +++ b/libinterp/parse-tree/lex.h @@ -284,6 +284,10 @@ bool space_follows_previous_token (void) const; + bool previous_token_is_binop (void) const; + + bool previous_token_may_be_command (void) const; + // true means that we have encountered eof on the input stream. bool end_of_input; @@ -529,6 +533,8 @@ int eat_whitespace (void); + bool whitespace_is_significant (void); + void handle_number (void); void handle_continuation (void); @@ -637,6 +643,10 @@ bool convert = false, bool bos = false, bool qit = false); + int handle_assign_op (const char *pattern, int tok); + + int handle_incompatible_assign_op (const char *pattern, int tok); + int handle_op_internal (const char *pattern, int tok, bool convert, bool bos, bool qit, bool compat); @@ -646,6 +656,8 @@ int count_token (int tok); + int count_token_internal (int tok); + int show_token (int tok); // For unwind protect.
--- a/libinterp/parse-tree/lex.ll +++ b/libinterp/parse-tree/lex.ll @@ -51,6 +51,8 @@ %x BLOCK_COMMENT_START %x LINE_COMMENT_START +%x KLUGE + %{ #include <cctype> @@ -224,7 +226,7 @@ curr_lexer->current_input_column++; int tok = curr_lexer->handle_string (yytext[0]); - return curr_lexer->count_token (tok); + return curr_lexer->count_token_internal (tok); } <COMMAND_START>[^#% \t\r\n\;\,\"\'][^ \t\r\n\;\,]*{S}* { @@ -238,6 +240,27 @@ return curr_lexer->handle_token (tok, SQ_STRING); } +<MATRIX_START>{S}* { + curr_lexer->lexer_debug ("<MATRIX_START>{S}*"); + + curr_lexer->mark_previous_token_trailing_space (); + } + +<MATRIX_START>{NL} { + curr_lexer->lexer_debug ("<MATRIX_START>{NL}"); + + int tok = curr_lexer->previous_token_value (); + + if (! (tok == ';' || tok == '[' || tok == '{')) + curr_lexer->xunput (','); + } + +<KLUGE>@ { + curr_lexer->lexer_debug ("<KLUGE>@"); + curr_lexer->pop_start_state (); + return curr_lexer->count_token (CHOOSE_ASSIGNMENT); + } + %{ // For this and the next two rules, we're looking at ']', and we // need to know if the next token is '=' or '=='. @@ -252,8 +275,8 @@ // FIXME -- we need to handle block comments here. %} -<MATRIX_START>{SNLCMT}*\]{S}* { - curr_lexer->lexer_debug ("<MATRIX_START>{SNLCMT}*\\]{S}*"); +<MATRIX_START>\] { + curr_lexer->lexer_debug ("<MATRIX_START>\\]"); curr_lexer->scan_for_comments (yytext); curr_lexer->fixup_column_count (yytext); @@ -268,18 +291,15 @@ bool spc_gobbled = (cont_is_spc || c == ' ' || c == '\t'); int tok_to_return = curr_lexer->handle_close_bracket (spc_gobbled, ']'); - if (spc_gobbled) - curr_lexer->xunput (' '); - - return curr_lexer->count_token (tok_to_return); + return curr_lexer->count_token (']'); } %{ // FIXME -- we need to handle block comments here. %} -<MATRIX_START>{SNLCMT}*\}{S}* { - curr_lexer->lexer_debug ("<MATRIX_START>{SNLCMT}*\\}{S}*"); +<MATRIX_START>\} { + curr_lexer->lexer_debug ("<MATRIX_START>\\}*"); curr_lexer->scan_for_comments (yytext); curr_lexer->fixup_column_count (yytext); @@ -294,144 +314,11 @@ bool spc_gobbled = (cont_is_spc || c == ' ' || c == '\t'); int tok_to_return = curr_lexer->handle_close_bracket (spc_gobbled, '}'); - if (spc_gobbled) - curr_lexer->xunput (' '); - - return curr_lexer->count_token (tok_to_return); - } - -%{ -// Commas are element separators in matrix constants. If we don't -// check for continuations here we can end up inserting too many -// commas. -%} - -<MATRIX_START>{S}*\,{S}* { - curr_lexer->lexer_debug ("<MATRIX_START>{S}*\\,{S}*"); - - curr_lexer->current_input_column += yyleng; - - int tmp = curr_lexer->eat_continuation (); - - curr_lexer->quote_is_transpose = false; - curr_lexer->convert_spaces_to_comma = true; - curr_lexer->looking_for_object_index = false; - curr_lexer->at_beginning_of_statement = false; - - if (! curr_lexer->looking_at_object_index.front ()) - { - if ((tmp & octave_lexer::NEWLINE) == octave_lexer::NEWLINE) - { - curr_lexer->maybe_warn_separator_insert (';'); - - curr_lexer->xunput (';'); - } - } - - return curr_lexer->count_token (','); + return curr_lexer->count_token ('}'); } -%{ -// In some cases, spaces in matrix constants can turn into commas. -// If commas are required, spaces are not important in matrix -// constants so we just eat them. If we don't check for continuations -// here we can end up inserting too many commas. -%} - -<MATRIX_START>{S}+ { - curr_lexer->lexer_debug ("<MATRIX_START>{S}+"); - - curr_lexer->current_input_column += yyleng; - - curr_lexer->at_beginning_of_statement = false; - - int tmp = curr_lexer->eat_continuation (); - - if (! curr_lexer->looking_at_object_index.front ()) - { - bool bin_op = curr_lexer->next_token_is_bin_op (true); - bool postfix_un_op = curr_lexer->next_token_is_postfix_unary_op (true); - bool sep_op = curr_lexer->next_token_is_sep_op (); - - if (! (postfix_un_op || bin_op || sep_op) - && curr_lexer->nesting_level.is_bracket_or_brace () - && curr_lexer->convert_spaces_to_comma) - { - if ((tmp & octave_lexer::NEWLINE) == octave_lexer::NEWLINE) - { - curr_lexer->maybe_warn_separator_insert (';'); - - curr_lexer->xunput (';'); - } - - curr_lexer->quote_is_transpose = false; - curr_lexer->convert_spaces_to_comma = true; - curr_lexer->looking_for_object_index = false; - - curr_lexer->maybe_warn_separator_insert (','); - - return curr_lexer->count_token (','); - } - } - } - -%{ -// Semicolons are handled as row seprators in matrix constants. If we -// don't eat whitespace here we can end up inserting too many -// semicolons. - -// FIXME -- we need to handle block comments here. -%} - -<MATRIX_START>{SNLCMT}*;{SNLCMT}* { - curr_lexer->lexer_debug ("<MATRIX_START>{SNLCMT}*;{SNLCMT}*"); - - curr_lexer->scan_for_comments (yytext); - curr_lexer->fixup_column_count (yytext); - curr_lexer->eat_whitespace (); - - curr_lexer->quote_is_transpose = false; - curr_lexer->convert_spaces_to_comma = true; - curr_lexer->looking_for_object_index = false; - curr_lexer->at_beginning_of_statement = false; - - return curr_lexer->count_token (';'); - } - -%{ -// In some cases, new lines can also become row separators. If we -// don't eat whitespace here we can end up inserting too many -// semicolons. - -// FIXME -- we need to handle block comments here. -%} - -<MATRIX_START>{S}*{COMMENT}{SNLCMT}* | -<MATRIX_START>{S}*{NL}{SNLCMT}* { - curr_lexer->lexer_debug ("<MATRIX_START>{S}*{COMMENT}{SNLCMT}*|<MATRIX_START>{S}*{NL}{SNLCMT}*"); - - curr_lexer->scan_for_comments (yytext); - curr_lexer->fixup_column_count (yytext); - curr_lexer->eat_whitespace (); - - curr_lexer->quote_is_transpose = false; - curr_lexer->convert_spaces_to_comma = true; - curr_lexer->at_beginning_of_statement = false; - - if (curr_lexer->nesting_level.none ()) - return LEXICAL_ERROR; - - if (! curr_lexer->looking_at_object_index.front () - && curr_lexer->nesting_level.is_bracket_or_brace ()) - { - curr_lexer->maybe_warn_separator_insert (';'); - - return curr_lexer->count_token (';'); - } - } - -\[{S}* { - curr_lexer->lexer_debug ("\\[{S}*"); +\[ { + curr_lexer->lexer_debug ("\\["); curr_lexer->nesting_level.bracket (); @@ -450,7 +337,6 @@ curr_lexer->looking_at_matrix_or_assign_lhs = true; curr_lexer->decrement_promptflag (); - curr_lexer->eat_whitespace (); curr_lexer->bracketflag++; @@ -620,8 +506,18 @@ {NUMBER}{Im} { curr_lexer->lexer_debug ("{NUMBER}{Im}"); - curr_lexer->handle_number (); - return curr_lexer->count_token (IMAG_NUM); + if (curr_lexer->whitespace_is_significant () + && curr_lexer->space_follows_previous_token () + && ! curr_lexer->previous_token_is_binop ()) + { + yyless (0); + unput (','); + } + else + { + curr_lexer->handle_number (); + return curr_lexer->count_token_internal (IMAG_NUM); + } } %{ @@ -632,8 +528,19 @@ {D}+/\.[\*/\\^\'] | {NUMBER} { curr_lexer->lexer_debug ("{D}+/\\.[\\*/\\^\\']|{NUMBER}"); - curr_lexer->handle_number (); - return curr_lexer->count_token (NUM); + + if (curr_lexer->whitespace_is_significant () + && curr_lexer->space_follows_previous_token () + && ! curr_lexer->previous_token_is_binop ()) + { + yyless (0); + unput (','); + } + else + { + curr_lexer->handle_number (); + return curr_lexer->count_token_internal (NUM); + } } %{ @@ -671,13 +578,31 @@ // don't write directly on yytext. %} -{IDENT}{S}* { - curr_lexer->lexer_debug ("{IDENT}{S}*"); - - int id_tok = curr_lexer->handle_identifier (); - - if (id_tok >= 0) - return curr_lexer->count_token (id_tok); +{IDENT} { + curr_lexer->lexer_debug ("{IDENT}"); + + if (curr_lexer->whitespace_is_significant () + && curr_lexer->space_follows_previous_token () + && ! curr_lexer->previous_token_is_binop ()) + { + yyless (0); + unput (','); + } + else + { + if (curr_lexer->previous_token_may_be_command ()) + { + yyless (0); + curr_lexer->push_start_state (COMMAND_START); + } + else + { + int id_tok = curr_lexer->handle_identifier (); + + if (id_tok >= 0) + return curr_lexer->count_token_internal (id_tok); + } + } } %{ @@ -773,17 +698,49 @@ curr_lexer->lexer_debug ("'"); curr_lexer->current_input_column++; - curr_lexer->convert_spaces_to_comma = true; - - if (curr_lexer->quote_is_transpose) + + int tok = curr_lexer->previous_token_value (); + + bool transpose = false; + + if (curr_lexer->whitespace_is_significant ()) { - curr_lexer->do_comma_insert_check (); - return curr_lexer->count_token (QUOTE); + if (curr_lexer->space_follows_previous_token ()) + { + if (tok == '[' || tok == '{' + || curr_lexer->previous_token_is_binop ()) + { + int retval = curr_lexer->handle_string ('\''); + return curr_lexer->count_token_internal (retval); + } + else + { + yyless (0); + curr_lexer->xunput (','); + } + } + else + { + if (tok == ',' || tok == ';' + || curr_lexer->previous_token_is_binop ()) + { + int retval = curr_lexer->handle_string ('\''); + return curr_lexer->count_token_internal (retval); + } + else + return curr_lexer->count_token (QUOTE); + } } else { - int tok = curr_lexer->handle_string ('\''); - return curr_lexer->count_token (tok); + if (tok == NAME || tok == NUM || tok == IMAG_NUM + || tok == ')' || tok == ']' || tok == '}') + return curr_lexer->count_token (QUOTE); + else + { + int retval = curr_lexer->handle_string ('\''); + return curr_lexer->count_token_internal (retval); + } } } @@ -797,7 +754,7 @@ curr_lexer->current_input_column++; int tok = curr_lexer->handle_string ('"'); - return curr_lexer->count_token (tok); + return curr_lexer->count_token_internal (tok); } %{ @@ -828,7 +785,6 @@ "\\" { return curr_lexer->handle_op ("\\", LEFTDIV); } "^" { return curr_lexer->handle_op ("^", POW); } "**" { return curr_lexer->handle_incompatible_op ("**", POW); } -"=" { return curr_lexer->handle_op ("=", '=', true, false); } "&&" { return curr_lexer->handle_op ("&&", EXPR_AND_AND); } "||" { return curr_lexer->handle_op ("||", EXPR_OR_OR); } "<<" { return curr_lexer->handle_incompatible_op ("<<", LSHIFT); } @@ -914,30 +870,239 @@ } %{ -// op= operators. +// = and op= operators. %} -"+=" { return curr_lexer->handle_incompatible_op ("+=", ADD_EQ); } -"-=" { return curr_lexer->handle_incompatible_op ("-=", SUB_EQ); } -"*=" { return curr_lexer->handle_incompatible_op ("*=", MUL_EQ); } -"/=" { return curr_lexer->handle_incompatible_op ("/=", DIV_EQ); } -"\\=" { return curr_lexer->handle_incompatible_op ("\\=", LEFTDIV_EQ); } -".+=" { return curr_lexer->handle_incompatible_op (".+=", ADD_EQ); } -".-=" { return curr_lexer->handle_incompatible_op (".-=", SUB_EQ); } -".*=" { return curr_lexer->handle_incompatible_op (".*=", EMUL_EQ); } -"./=" { return curr_lexer->handle_incompatible_op ("./=", EDIV_EQ); } -".\\=" { return curr_lexer->handle_incompatible_op (".\\=", ELEFTDIV_EQ); } -"^=" { return curr_lexer->handle_incompatible_op ("^=", POW_EQ); } -"**=" { return curr_lexer->handle_incompatible_op ("^=", POW_EQ); } -".^=" { return curr_lexer->handle_incompatible_op (".^=", EPOW_EQ); } -".**=" { return curr_lexer->handle_incompatible_op (".^=", EPOW_EQ); } -"&=" { return curr_lexer->handle_incompatible_op ("&=", AND_EQ); } -"|=" { return curr_lexer->handle_incompatible_op ("|=", OR_EQ); } -"<<=" { return curr_lexer->handle_incompatible_op ("<<=", LSHIFT_EQ); } -">>=" { return curr_lexer->handle_incompatible_op (">>=", RSHIFT_EQ); } - -\{{S}* { - curr_lexer->lexer_debug ("\\{{S}*"); +"=" { + int tok = curr_lexer->handle_assign_op ("=", '='); + if (tok < 0) + { + yyless (0); + curr_lexer->xunput ('@'); + curr_lexer->push_start_state (KLUGE); + } + else + return tok; + } + +"+=" { + int tok = curr_lexer->handle_incompatible_assign_op ("+=", ADD_EQ); + if (tok < 0) + { + yyless (0); + curr_lexer->xunput ('@'); + curr_lexer->push_start_state (KLUGE); + } + else + return tok; + } + +"-=" { + int tok = curr_lexer->handle_incompatible_assign_op ("-=", SUB_EQ); + if (tok < 0) + { + yyless (0); + curr_lexer->xunput ('@'); + curr_lexer->push_start_state (KLUGE); + } + else + return tok; + } + +"*=" { + int tok = curr_lexer->handle_incompatible_assign_op ("*=", MUL_EQ); + if (tok < 0) + { + yyless (0); + curr_lexer->xunput ('@'); + curr_lexer->push_start_state (KLUGE); + } + else + return tok; + } + +"/=" { + int tok = curr_lexer->handle_incompatible_assign_op ("/=", DIV_EQ); + if (tok < 0) + { + yyless (0); + curr_lexer->xunput ('@'); + curr_lexer->push_start_state (KLUGE); + } + else + return tok; + } + +"\\=" { + int tok = curr_lexer->handle_incompatible_assign_op ("\\=", LEFTDIV_EQ); + if (tok < 0) + { + yyless (0); + curr_lexer->xunput ('@'); + curr_lexer->push_start_state (KLUGE); + } + else + return tok; + } + +".+=" { + int tok = curr_lexer->handle_incompatible_assign_op (".+=", ADD_EQ); + if (tok < 0) + { + yyless (0); + curr_lexer->xunput ('@'); + curr_lexer->push_start_state (KLUGE); + } + else + return tok; + } + +".-=" { + int tok = curr_lexer->handle_incompatible_assign_op (".-=", SUB_EQ); + if (tok < 0) + { + yyless (0); + curr_lexer->xunput ('@'); + curr_lexer->push_start_state (KLUGE); + } + else + return tok; + } + +".*=" { + int tok = curr_lexer->handle_incompatible_assign_op (".*=", EMUL_EQ); + if (tok < 0) + { + yyless (0); + curr_lexer->xunput ('@'); + curr_lexer->push_start_state (KLUGE); + } + else + return tok; + } + +"./=" { + int tok = curr_lexer->handle_incompatible_assign_op ("./=", EDIV_EQ); + if (tok < 0) + { + yyless (0); + curr_lexer->xunput ('@'); + curr_lexer->push_start_state (KLUGE); + } + else + return tok; + } + +".\\=" { + int tok = curr_lexer->handle_incompatible_assign_op (".\\=", ELEFTDIV_EQ); + if (tok < 0) + { + yyless (0); + curr_lexer->xunput ('@'); + curr_lexer->push_start_state (KLUGE); + } + else + return tok; + } + +"^=" { + int tok = curr_lexer->handle_incompatible_assign_op ("^=", POW_EQ); + if (tok < 0) + { + yyless (0); + curr_lexer->xunput ('@'); + curr_lexer->push_start_state (KLUGE); + } + else + return tok; + } + +"**=" { + int tok = curr_lexer->handle_incompatible_assign_op ("^=", POW_EQ); + if (tok < 0) + { + yyless (0); + curr_lexer->xunput ('@'); + curr_lexer->push_start_state (KLUGE); + } + else + return tok; + } + +".^=" { + int tok = curr_lexer->handle_incompatible_assign_op (".^=", EPOW_EQ); + if (tok < 0) + { + yyless (0); + curr_lexer->xunput ('@'); + curr_lexer->push_start_state (KLUGE); + } + else + return tok; + } + +".**=" { + int tok = curr_lexer->handle_incompatible_assign_op (".^=", EPOW_EQ); + if (tok < 0) + { + yyless (0); + curr_lexer->xunput ('@'); + curr_lexer->push_start_state (KLUGE); + } + else + return tok; + } + +"&=" { + int tok = curr_lexer->handle_incompatible_assign_op ("&=", AND_EQ); + if (tok < 0) + { + yyless (0); + curr_lexer->xunput ('@'); + curr_lexer->push_start_state (KLUGE); + } + else + return tok; + } + +"|=" { + int tok = curr_lexer->handle_incompatible_assign_op ("|=", OR_EQ); + if (tok < 0) + { + yyless (0); + curr_lexer->xunput ('@'); + curr_lexer->push_start_state (KLUGE); + } + else + return tok; + } + +"<<=" { + int tok = curr_lexer->handle_incompatible_assign_op ("<<=", LSHIFT_EQ); + if (tok < 0) + { + yyless (0); + curr_lexer->xunput ('@'); + curr_lexer->push_start_state (KLUGE); + } + else + return tok; + } + +">>=" { + int tok = curr_lexer->handle_incompatible_assign_op (">>=", RSHIFT_EQ); + if (tok < 0) + { + yyless (0); + curr_lexer->xunput ('@'); + curr_lexer->push_start_state (KLUGE); + } + else + return tok; + } + +"{" { + curr_lexer->lexer_debug ("{"); curr_lexer->nesting_level.brace (); @@ -1418,6 +1583,34 @@ return tok ? tok->space_follows_token () : false; } +bool +lexical_feedback::previous_token_is_binop (void) const +{ + int tok = previous_token_value (); + + return (tok == '+' || tok == '-' || tok == '@' + || tok == ',' || tok == ';' || tok == '*' || tok == '/' + || tok == ':' || tok == '=' || tok == ADD_EQ + || tok == AND_EQ || tok == DIV_EQ || tok == EDIV + || tok == EDIV_EQ || tok == ELEFTDIV || tok == ELEFTDIV_EQ + || tok == EMINUS || tok == EMUL || tok == EMUL_EQ + || tok == EPOW || tok == EPOW_EQ || tok == EXPR_AND + || tok == EXPR_AND_AND || tok == EXPR_EQ || tok == EXPR_GE + || tok == EXPR_GT || tok == EXPR_LE || tok == EXPR_LT + || tok == EXPR_NE || tok == EXPR_NOT || tok == EXPR_OR + || tok == EXPR_OR_OR || tok == LEFTDIV || tok == LEFTDIV_EQ + || tok == LSHIFT || tok == LSHIFT_EQ || tok == MUL_EQ + || tok == OR_EQ || tok == POW || tok == POW_EQ + || tok == RSHIFT || tok == RSHIFT_EQ || tok == SUB_EQ); +} + +bool +lexical_feedback::previous_token_may_be_command (void) const +{ + const token *tok = tokens.front (); + return tok ? tok->may_be_command () : false; +} + static bool looks_like_copyright (const std::string& s) { @@ -2292,6 +2485,14 @@ return retval; } +bool +octave_lexer::whitespace_is_significant (void) +{ + return (nesting_level.is_bracket () + || (nesting_level.is_brace () + && ! looking_at_object_index.front ())); +} + static inline bool looks_like_hex (const char *s, int len) { @@ -2351,7 +2552,7 @@ char *yytxt = flex_yytext (); int yylng = flex_yyleng (); - size_t offset = 1; + int offset = 1; if (yytxt[0] == '\\') gripe_matlab_incompatible_continuation (); else @@ -2768,46 +2969,6 @@ pop_start_state (); - if (bracket_type == ']' - && next_token_is_assign_op () - && ! looking_at_return_list) - { - retval = CLOSE_BRACE; - } - else if ((bracketflag || braceflag) - && convert_spaces_to_comma - && (nesting_level.is_bracket () - || (nesting_level.is_brace () - && ! looking_at_object_index.front ()))) - { - bool index_op = next_token_is_index_op (); - - // Don't insert comma if we are looking at something like - // - // [x{i}{j}] or [x{i}(j)] - // - // but do if we are looking at - // - // [x{i} {j}] or [x{i} (j)] - - if (spc_gobbled || ! (bracket_type == '}' && index_op)) - { - bool bin_op = next_token_is_bin_op (spc_gobbled); - - bool postfix_un_op = next_token_is_postfix_unary_op (spc_gobbled); - - bool sep_op = next_token_is_sep_op (); - - if (! (postfix_un_op || bin_op || sep_op)) - { - maybe_warn_separator_insert (','); - - xunput (','); - return retval; - } - } - } - quote_is_transpose = true; convert_spaces_to_comma = true; @@ -3201,17 +3362,13 @@ int octave_lexer::handle_identifier (void) { - bool at_bos = at_beginning_of_statement; - char *yytxt = flex_yytext (); - std::string tok = strip_trailing_whitespace (yytxt); + std::string tok = yytxt; int c = yytxt[flex_yyleng()-1]; - bool cont_is_spc = (eat_continuation () != octave_lexer::NO_WHITESPACE); - - int spc_gobbled = (cont_is_spc || c == ' ' || c == '\t'); + bool spc_gobbled = false; // If we are expecting a structure element, avoid recognizing // keywords and other special names and return STRUCT_ELT, which is @@ -3220,9 +3377,9 @@ if (looking_at_indirect_ref) { - do_comma_insert_check (); - - maybe_unput_comma (spc_gobbled); + // do_comma_insert_check (); + + // maybe_unput_comma (spc_gobbled); push_token (new token (STRUCT_ELT, tok, input_line_number, current_input_column)); @@ -3233,16 +3390,17 @@ current_input_column += flex_yyleng (); + at_beginning_of_statement = false; + return STRUCT_ELT; } - at_beginning_of_statement = false; - // The is_keyword_token may reset // at_beginning_of_statement. For example, if it sees // an else token, then the next token is at the beginning of a // statement. + // May set begenning_of_statement to true. int kw_token = is_keyword_token (tok); // If we found a keyword token, then the beginning_of_statement flag @@ -3267,6 +3425,8 @@ convert_spaces_to_comma = true; looking_for_object_index = true; + at_beginning_of_statement = false; + return FCN_HANDLE; } } @@ -3315,16 +3475,11 @@ if (! is_variable (tok)) { - if (at_bos && spc_gobbled && can_be_command (tok) - && looks_like_command_arg ()) - { - push_start_state (COMMAND_START); - } - else if (next_tok_is_eq - || looking_at_decl_list - || looking_at_return_list - || (looking_at_parameter_list - && ! looking_at_initializer_expression)) + if (next_tok_is_eq + || looking_at_decl_list + || looking_at_return_list + || (looking_at_parameter_list + && ! looking_at_initializer_expression)) { symbol_table::force_variable (tok); } @@ -3340,28 +3495,21 @@ if (tok == "end") tok = "__end__"; - push_token (new token (NAME, &(symbol_table::insert (tok)), - input_line_number, current_input_column)); - - // After seeing an identifer, it is ok to convert spaces to a comma - // (if needed). - - convert_spaces_to_comma = true; - - if (! (next_tok_is_eq || start_state () == COMMAND_START)) - { - quote_is_transpose = true; - - do_comma_insert_check (); - - maybe_unput_comma (spc_gobbled); - } + token *tok_val = new token (NAME, &(symbol_table::insert (tok)), + input_line_number, current_input_column); + + if (at_beginning_of_statement) + tok_val->mark_may_be_command (); + + push_token (tok_val); current_input_column += flex_yyleng (); if (tok != "__end__") looking_for_object_index = true; + at_beginning_of_statement = false; + return NAME; } @@ -3561,7 +3709,7 @@ case END_OF_INPUT: std::cerr << "END_OF_INPUT\n\n"; break; case LEXICAL_ERROR: std::cerr << "LEXICAL_ERROR\n\n"; break; case FCN: std::cerr << "FCN\n"; break; - case CLOSE_BRACE: std::cerr << "CLOSE_BRACE\n"; break; + case CHOOSE_ASSIGNMENT: std::cerr << "CHOOSE_ASSIGNMENT\n"; break; case INPUT_FILE: std::cerr << "INPUT_FILE\n"; break; case SUPERCLASSREF: std::cerr << "SUPERCLASSREF\n"; break; case METAQUERY: std::cerr << "METAQUERY\n"; break; @@ -3669,6 +3817,10 @@ std::cerr << "LINE_COMMENT_START" << std::endl; break; + case KLUGE: + std::cerr << "KLUGE" << std::endl; + break; + default: std::cerr << "UNKNOWN START STATE!" << std::endl; break; @@ -3690,6 +3842,24 @@ } int +octave_lexer::handle_assign_op (const char *pattern, int tok) +{ + lexer_debug (pattern); + + return (previous_token_value_is (']') && looking_at_matrix_or_assign_lhs) + ? -1 : handle_op_internal (pattern, tok, false, false, false, true); +} + +int +octave_lexer::handle_incompatible_assign_op (const char *pattern, int tok) +{ + lexer_debug (pattern); + + return (previous_token_value_is (']') && looking_at_matrix_or_assign_lhs) + ? -1 : handle_op_internal (pattern, tok, false, false, false, false); +} + +int octave_lexer::handle_op_internal (const char *pattern, int tok, bool convert, bool bos, bool qit, bool compat) { @@ -3730,12 +3900,22 @@ quote_is_transpose = false; convert_spaces_to_comma = true; - return count_token (tok); + return count_token_internal (tok); } int octave_lexer::count_token (int tok) { + token *tok_val = new token (tok, input_line_number, current_input_column); + + push_token (tok_val); + + return count_token_internal (tok); +} + +int +octave_lexer::count_token_internal (int tok) +{ if (tok != '\n') { Vtoken_count++;
--- a/libinterp/parse-tree/oct-parse.in.yy +++ b/libinterp/parse-tree/oct-parse.in.yy @@ -253,12 +253,12 @@ %token END_OF_INPUT LEXICAL_ERROR %token FCN INPUT_FILE // %token VARARGIN VARARGOUT -%token CLOSE_BRACE +%token CHOOSE_ASSIGNMENT // Nonterminals we construct. %type <comment_type> stash_comment function_beg %type <tok_type> classdef_beg -%type <sep_type> sep_no_nl opt_sep_no_nl nl opt_nl sep opt_sep opt_comma +%type <sep_type> sep_no_nl opt_sep_no_nl nl opt_nl sep opt_sep %type <tree_type> input %type <tree_constant_type> string constant magic_colon %type <tree_anon_fcn_handle_type> anon_fcn_handle @@ -470,25 +470,21 @@ matrix : '[' ']' { $$ = new tree_constant (octave_null_matrix::instance); - curr_lexer->looking_at_matrix_or_assign_lhs = false; curr_lexer->pending_local_variables.clear (); } | '[' ';' ']' { $$ = new tree_constant (octave_null_matrix::instance); - curr_lexer->looking_at_matrix_or_assign_lhs = false; curr_lexer->pending_local_variables.clear (); } | '[' ',' ']' { $$ = new tree_constant (octave_null_matrix::instance); - curr_lexer->looking_at_matrix_or_assign_lhs = false; curr_lexer->pending_local_variables.clear (); } | '[' matrix_rows ']' { $$ = curr_parser.finish_matrix ($2); - curr_lexer->looking_at_matrix_or_assign_lhs = false; curr_lexer->pending_local_variables.clear (); } ; @@ -533,9 +529,9 @@ cell_or_matrix_row : arg_list - { $$ = curr_parser.validate_matrix_row ($1); } + { $$ = $1; } | arg_list ',' // Ignore trailing comma. - { $$ = curr_parser.validate_matrix_row ($1); } + { $$ = $1; } ; fcn_handle : '@' FCN_HANDLE @@ -559,7 +555,10 @@ | fcn_handle { $$ = $1; } | matrix - { $$ = $1; } + { + curr_lexer->looking_at_matrix_or_assign_lhs = false; + $$ = $1; + } | cell { $$ = $1; } | meta_identifier @@ -709,17 +708,15 @@ { $$ = curr_parser.make_boolean_op (EXPR_OR_OR, $1, $2, $3); } ; -// Arrange for the lexer to return CLOSE_BRACE for ']' by looking ahead -// one token for an assignment op. - assign_lhs : simple_expr { $$ = new tree_argument_list ($1); $$->mark_as_simple_assign_lhs (); } - | '[' arg_list opt_comma CLOSE_BRACE + | matrix CHOOSE_ASSIGNMENT { - $$ = $2; + tree_matrix *tmp = dynamic_cast<tree_matrix *> ($1); + $$ = tmp->front (); curr_lexer->looking_at_matrix_or_assign_lhs = false; for (std::set<std::string>::const_iterator p = curr_lexer->pending_local_variables.begin (); p != curr_lexer->pending_local_variables.end (); @@ -1543,12 +1540,6 @@ { $$ = $1; } ; -opt_comma : // empty - { $$ = 0; } - | ',' - { $$ = ','; } - ; - %% // Generic error messages.
--- a/libinterp/parse-tree/token.cc +++ b/libinterp/parse-tree/token.cc @@ -34,6 +34,7 @@ token::token (int tv, int l, int c) { + maybe_cmd = false; tspc = false; line_num = l; column_num = c; @@ -43,6 +44,7 @@ token::token (int tv, const std::string& s, int l, int c) { + maybe_cmd = false; tspc = false; line_num = l; column_num = c; @@ -53,6 +55,7 @@ token::token (int tv, double d, const std::string& s, int l, int c) { + maybe_cmd = false; tspc = false; line_num = l; column_num = c; @@ -64,6 +67,7 @@ token::token (int tv, end_tok_type t, int l, int c) { + maybe_cmd = false; tspc = false; line_num = l; column_num = c; @@ -74,6 +78,7 @@ token::token (int tv, symbol_table::symbol_record *s, int l, int c) { + maybe_cmd = false; tspc = false; line_num = l; column_num = c; @@ -85,6 +90,7 @@ token::token (int tv, const std::string& pkg, const std::string& cls, int l, int c) { + maybe_cmd = false; tspc = false; line_num = l; column_num = c; @@ -97,6 +103,7 @@ token::token (int tv, const std::string& mth, const std::string& pkg, const std::string& cls, int l, int c) { + maybe_cmd = false; tspc = false; line_num = l; column_num = c;
--- a/libinterp/parse-tree/token.h +++ b/libinterp/parse-tree/token.h @@ -74,6 +74,9 @@ ~token (void); + void mark_may_be_command (void) { maybe_cmd = true; } + bool may_be_command (void) const { return maybe_cmd; } + void mark_trailing_space (void) { tspc = true; } bool space_follows_token (void) const { return tspc; } @@ -105,6 +108,7 @@ token& operator = (const token& tok); + bool maybe_cmd; bool tspc; int line_num; int column_num;