Mercurial > hg > octave-lyh
changeset 16253:a89cf57ba3a5
partial cleanup of continuation handling in lexer
* lex.h, lex.ll (octave_lexer::handle_continuation): New function.
(octave_lexer::finish_comment): Move to octave_lexer::finish_comment.
Don't return token. New argument, looking_at_continuation. If not
handling continuation, unput newline character.
(^{S}*{CCHAR}\{{S}*{NL}): Call yyless before finishing comment.
(^{S}*{CCHAR}\{{S}*{NL}, <BLOCK_COMMENT_START>^{S}*{CCHAR}\{{S}*{NL},
<LINE_COMMENT_START>{ANY_INCLUDING_NL},
<LINE_COMMENT_START>{S}*{CCHAR}.*{NL}): Don't return token.
(<LINE_COMMENT_START>{S}*{CCHAR}.*{NL}): Don't give COMMAND_START
start state special treatment.
({CONT}{S}*{NL}|{CONT}{S}*{COMMENT}):
Call octave_lexer::handle_continuation.
author | John W. Eaton <jwe@octave.org> |
---|---|
date | Sat, 09 Mar 2013 21:44:14 -0500 |
parents | 37af399267b2 |
children | b8a2df776118 12bf6a3f8c45 |
files | libinterp/parse-tree/lex.h libinterp/parse-tree/lex.ll |
diffstat | 2 files changed, 89 insertions(+), 72 deletions(-) [+] |
line wrap: on
line diff
--- a/libinterp/parse-tree/lex.h +++ b/libinterp/parse-tree/lex.h @@ -253,8 +253,6 @@ void reset (void); - int finish_comment (octave_comment_elt::comment_type typ); - // true means that we have encountered eof on the input stream. bool end_of_input; @@ -496,6 +494,11 @@ void handle_number (void); + void handle_continuation (void); + + void finish_comment (octave_comment_elt::comment_type typ, + bool looking_at_continuation = false); + bool have_continuation (bool trailing_comments_ok = true); bool have_ellipsis_continuation (bool trailing_comments_ok = true);
--- a/libinterp/parse-tree/lex.ll +++ b/libinterp/parse-tree/lex.ll @@ -485,22 +485,18 @@ ^{S}*{CCHAR}\{{S}*{NL} { curr_lexer->lexer_debug ("^{S}*{CCHAR}\{{S}*{NL}"); - int tok = 0; + yyless (0); if (curr_lexer->start_state () == LINE_COMMENT_START) { if (! curr_lexer->comment_text.empty ()) - tok = curr_lexer->finish_comment (octave_comment_elt::full_line); + curr_lexer->finish_comment (octave_comment_elt::full_line); curr_lexer->pop_start_state (); } curr_lexer->push_start_state (BLOCK_COMMENT_START); - yyless (0); - - if (tok > 0) - return curr_lexer->count_token (tok); } <BLOCK_COMMENT_START>^{S}*{CCHAR}\{{S}*{NL} { @@ -527,18 +523,13 @@ curr_lexer->input_line_number++; curr_lexer->current_input_column = 1; - int tok = 0; - if (curr_lexer->block_comment_nesting_level > 1) curr_lexer->comment_text = "\n"; else - tok = curr_lexer->finish_comment (octave_comment_elt::block); + curr_lexer->finish_comment (octave_comment_elt::block); curr_lexer->block_comment_nesting_level--; curr_lexer->pop_start_state (); - - if (tok > 0) - return curr_lexer->count_token (tok); } %{ @@ -584,26 +575,12 @@ curr_lexer->comment_text += &yytext[i]; - int tok = 0; - if (! full_line_comment) { - tok = curr_lexer->finish_comment (octave_comment_elt::end_of_line); + curr_lexer->finish_comment (octave_comment_elt::end_of_line); curr_lexer->pop_start_state (); - - if (curr_lexer->start_state () == COMMAND_START) - { - // Allow the actions for the end of a COMMAND line to be - // executed next. - - tok = 0; - curr_lexer->xunput ('\n'); - } } - - if (tok > 0) - return curr_lexer->count_token (tok); } %{ @@ -615,12 +592,9 @@ curr_lexer->xunput (yytext[0]); - int tok = curr_lexer->finish_comment (octave_comment_elt::full_line); + curr_lexer->finish_comment (octave_comment_elt::full_line); curr_lexer->pop_start_state (); - - if (tok > 0) - return curr_lexer->count_token (tok); } %{ @@ -660,15 +634,10 @@ %} {CONT}{S}*{NL} | -{CONT}{S}*{COMMENT} { - curr_lexer->lexer_debug ("{CONT}{S}*{NL}|{CONT}{S}*{COMMENT}"); - - if (yytext[0] == '\\') - curr_lexer->gripe_matlab_incompatible_continuation (); - curr_lexer->scan_for_comments (yytext); - curr_lexer->decrement_promptflag (); - curr_lexer->input_line_number++; - curr_lexer->current_input_column = 1; +{CONT}{S}*{CCHAR}.*{NL} { + curr_lexer->lexer_debug ("{CONT}{S}*{NL}|{CONT}{S}*{CCHAR}.*{NL}"); + + curr_lexer->handle_continuation (); } %{ @@ -1415,36 +1384,6 @@ return retval; } -int -lexical_feedback::finish_comment (octave_comment_elt::comment_type typ) -{ - bool copyright = looks_like_copyright (comment_text); - - if (nesting_level.none () && help_text.empty () - && ! comment_text.empty () && ! copyright) - help_text = comment_text; - - if (copyright) - typ = octave_comment_elt::copyright; - - octave_comment_buffer::append (comment_text, typ); - - comment_text = ""; - - quote_is_transpose = false; - convert_spaces_to_comma = true; - at_beginning_of_statement = true; - - if (nesting_level.none ()) - return '\n'; - else if (nesting_level.is_bracket_or_brace ()) - // FIXME -- this result will be different if the comment follows a - // continuation token. - return ';'; - else - return 0; -} - void octave_lexer::input_buffer::fill (const std::string& input, bool eof_arg) { @@ -2354,6 +2293,81 @@ do_comma_insert_check (); } +void +octave_lexer::handle_continuation (void) +{ + char *yytxt = flex_yytext (); + int yylng = flex_yyleng (); + + size_t offset = 1; + if (yytxt[0] == '\\') + gripe_matlab_incompatible_continuation (); + else + offset = 3; + + bool have_space = false; + while (offset < yylng) + { + char c = yytxt[offset]; + if (c == ' ' || c == '\t') + { + have_space = true; + offset++; + } + else + break; + } + + bool have_comment = false; + while (offset < yylng) + { + char c = yytxt[offset]; + if (c == '#' || c == '%') + { + have_comment = true; + offset++; + } + else + break; + } + + if (have_comment) + { + comment_text = &yytxt[offset]; + + finish_comment (octave_comment_elt::end_of_line, true); + } + + decrement_promptflag (); + input_line_number++; + current_input_column = 1; +} + +void +octave_lexer::finish_comment (octave_comment_elt::comment_type typ, + bool looking_at_continuation) +{ + bool copyright = looks_like_copyright (comment_text); + + if (nesting_level.none () && help_text.empty () + && ! comment_text.empty () && ! copyright) + help_text = comment_text; + + if (copyright) + typ = octave_comment_elt::copyright; + + octave_comment_buffer::append (comment_text, typ); + + comment_text = ""; + + quote_is_transpose = false; + convert_spaces_to_comma = true; + at_beginning_of_statement = true; + + if (! looking_at_continuation) + xunput ('\n'); +} + // We have seen a backslash and need to find out if it should be // treated as a continuation character. If so, this eats it, up to // and including the new line character.