octave-nkf: libinterp/parse-tree/lex.ll comparison

comparison libinterp/parse-tree/lex.ll @ 16293:57e87ddfee14

create base class for lexer * lex.h, lex.ll, parse.h, oct-parse.in.yy: (octave_base_lexer): New base class for lexer class. Move most of previous octave_lexer class here. Change all uses.

author	John W. Eaton <jwe@octave.org>
date	Wed, 13 Mar 2013 02:46:56 -0400
parents	6ce905b89cee
children	0925d1f6875e

comparison

equal deleted inserted replaced

-:6ce905b89cee
+:57e87ddfee14
 && defined (YY_FLEX_MAJOR_VERSION) && YY_FLEX_MAJOR_VERSION >= 2 \
 && defined (YY_FLEX_MINOR_VERSION) && YY_FLEX_MINOR_VERSION >= 5)
 #error lex.l requires flex version 2.5.4 or later
 #endif
-#define YY_EXTRA_TYPE octave_lexer *
+#define YY_EXTRA_TYPE octave_base_lexer *
 #define curr_lexer yyextra
 // Arrange to get input via readline.
 #ifdef YY_INPUT
 #undef YY_INPUT
 #endif
 #define YY_INPUT(buf, result, max_size) \
-result = curr_lexer->read (buf, max_size)
+result = curr_lexer->fill_flex_buffer (buf, max_size)
 // Try to avoid crashing out completely on fatal scanner errors.
 #ifdef YY_FATAL_ERROR
 #undef YY_FATAL_ERROR
 return retval;
 }
 void
-octave_lexer::input_buffer::fill (const std::string& input, bool eof_arg)
+octave_base_lexer::input_buffer::fill (const std::string& input, bool eof_arg)
 {
 buffer = input;
 chars_left = buffer.length ();
 pos = buffer.c_str ();
 eof = eof_arg;
 }
 int
-octave_lexer::input_buffer::copy_chunk (char *buf, size_t max_size)
+octave_base_lexer::input_buffer::copy_chunk (char *buf, size_t max_size)
 {
 static const char * const eol = "\n";
 size_t len = max_size > chars_left ? chars_left : max_size;
 assert (len > 0);
 }
 else
 {
 // There isn't enough room to plug the newline character
 // in the buffer so arrange to have it returned on the next
-// call to octave_lexer::read.
+// call to octave_base_lexer::read.
 pos = eol;
 chars_left = 1;
 }
 }
 return len;
 }
-octave_lexer::~octave_lexer (void)
+octave_base_lexer::~octave_base_lexer (void)
 {
 yylex_destroy (scanner);
 }
 void
-octave_lexer::init (void)
+octave_base_lexer::init (void)
 {
 yylex_init (&scanner);
-// Make octave_lexer object available through yyextra in
+// Make octave_base_lexer object available through yyextra in
 // flex-generated lexer.
 yyset_extra (this, scanner);
 clear_start_state ();
 }
 // Inside Flex-generated functions, yyg is the scanner cast to its real
-// type.  Some flex macros that we use in octave_lexer member functions
+// type.  Some flex macros that we use in octave_base_lexer member functions
 // (for example, BEGIN) use yyg.  If we could perform the actions of
 // these macros with functions instead, we could eliminate the
 // OCTAVE_YYG macro.
 #define OCTAVE_YYG \
 struct yyguts_t *yyg = static_cast<struct yyguts_t*> (scanner)
 void
-octave_lexer::reset (void)
+octave_base_lexer::reset (void)
 {
 // Start off on the right foot.
 clear_start_state ();
 parser_symtab_context.clear ();
 || reading_classdef_file
 || reading_script_file
 || input_from_eval_string ()))
 yyrestart (stdin, scanner);
-input_reader.reset ();
 lexical_feedback::reset ();
 }
 void
-octave_lexer::prep_for_file (void)
+octave_base_lexer::prep_for_file (void)
 {
 reading_script_file = true;
 push_start_state (INPUT_FILE_START);
 }
 int
-octave_lexer::read (char *buf, unsigned max_size)
+octave_base_lexer::handle_end_of_input (void)
-{
-int status = 0;
-if (input_buf.empty ())
-{
-bool eof = false;
-current_input_line = input_reader.get_input (eof);
-input_buf.fill (current_input_line, eof);
-}
-if (! input_buf.empty ())
-status = input_buf.copy_chunk (buf, max_size);
-else
-{
-status = YY_NULL;
-if (! input_buf.at_eof ())
-fatal_error ("octave_lexer::read () in flex scanner failed");
-}
-return status;
-}
-int
-octave_lexer::handle_end_of_input (void)
 {
 lexer_debug ("<<EOF>>");
 if (block_comment_nesting_level != 0)
 {
 return handle_token (END_OF_INPUT);
 }
 char *
-octave_lexer::flex_yytext (void)
+octave_base_lexer::flex_yytext (void)
 {
 return yyget_text (scanner);
 }
 int
-octave_lexer::flex_yyleng (void)
+octave_base_lexer::flex_yyleng (void)
 {
 return yyget_leng (scanner);
 }
 int
-octave_lexer::text_yyinput (void)
+octave_base_lexer::text_yyinput (void)
 {
 int c = yyinput (scanner);
 if (lexer_debug_flag)
 {
 return c;
 }
 void
-octave_lexer::xunput (char c, char *buf)
+octave_base_lexer::xunput (char c, char *buf)
 {
 if (c != EOF)
 {
 if (lexer_debug_flag)
 {
 yyunput (c, buf, scanner);
 }
 }
 void
-octave_lexer::xunput (char c)
+octave_base_lexer::xunput (char c)
 {
 char *yytxt = flex_yytext ();
 xunput (c, yytxt);
 }
 bool
-octave_lexer::looking_at_space (void)
+octave_base_lexer::looking_at_space (void)
 {
 int c = text_yyinput ();
 xunput (c);
 return (c == ' ' || c == '\t');
 }
 bool
-octave_lexer::inside_any_object_index (void)
+octave_base_lexer::inside_any_object_index (void)
 {
 bool retval = false;
 for (std::list<bool>::const_iterator i = looking_at_object_index.begin ();
 i != looking_at_object_index.end (); i++)
 }
 // Handle keywords.  Return -1 if the keyword should be ignored.
 int
-octave_lexer::is_keyword_token (const std::string& s)
+octave_base_lexer::is_keyword_token (const std::string& s)
 {
 int l = input_line_number;
 int c = current_input_column;
 int len = s.length ();
 return 0;
 }
 bool
-octave_lexer::whitespace_is_significant (void)
+octave_base_lexer::whitespace_is_significant (void)
 {
 return (nesting_level.is_bracket ()
 || (nesting_level.is_brace ()
 && ! looking_at_object_index.front ()));
 }
 {
 return (len > 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X'));
 }
 void
-octave_lexer::handle_number (void)
+octave_base_lexer::handle_number (void)
 {
 double value = 0.0;
 int nread = 0;
 char *yytxt = flex_yytext ();
 current_input_column += flex_yyleng ();
 }
 void
-octave_lexer::handle_continuation (void)
+octave_base_lexer::handle_continuation (void)
 {
 char *yytxt = flex_yytext ();
 int yylng = flex_yyleng ();
 int offset = 1;
 input_line_number++;
 current_input_column = 1;
 }
 void
-octave_lexer::finish_comment (octave_comment_elt::comment_type typ,
+octave_base_lexer::finish_comment (octave_comment_elt::comment_type typ,
 bool looking_at_continuation)
 {
 bool copyright = looks_like_copyright (comment_text);
 if (nesting_level.none () && help_text.empty ()
 && ! comment_text.empty () && ! copyright)
 // characters, return 0.  Otherwise, return 1.
 // FIXME -- we need to handle block comments here.
 bool
-octave_lexer::have_continuation (bool trailing_comments_ok)
+octave_base_lexer::have_continuation (bool trailing_comments_ok)
 {
 std::ostringstream buf;
 std::string comment_buf;
 // We have seen a '.' and need to see if it is the start of a
 // continuation.  If so, this eats it, up to and including the new
 // line character.
 bool
-octave_lexer::have_ellipsis_continuation (bool trailing_comments_ok)
+octave_base_lexer::have_ellipsis_continuation (bool trailing_comments_ok)
 {
 char c1 = text_yyinput ();
 if (c1 == '.')
 {
 char c2 = text_yyinput ();
 return false;
 }
 int
-octave_lexer::handle_string (char delim)
+octave_base_lexer::handle_string (char delim)
 {
 std::ostringstream buf;
 int bos_line = input_line_number;
 int bos_col = current_input_column;
 return LEXICAL_ERROR;
 }
 int
-octave_lexer::handle_close_bracket (int bracket_type)
+octave_base_lexer::handle_close_bracket (int bracket_type)
 {
 int retval = bracket_type;
 if (! nesting_level.none ())
 {
 return retval;
 }
 bool
-octave_lexer::looks_like_command_arg (void)
+octave_base_lexer::looks_like_command_arg (void)
 {
 bool space_before = space_follows_previous_token ();
 bool space_after = looking_at_space ();
 return (space_before && ! space_after
 && previous_token_may_be_command ());
 }
 int
-octave_lexer::handle_superclass_identifier (void)
+octave_base_lexer::handle_superclass_identifier (void)
 {
 std::string pkg;
 char *yytxt = flex_yytext ();
 std::string meth = strip_trailing_whitespace (yytxt);
 size_t pos = meth.find ("@");
 return SUPERCLASSREF;
 }
 int
-octave_lexer::handle_meta_identifier (void)
+octave_base_lexer::handle_meta_identifier (void)
 {
 std::string pkg;
 char *yytxt = flex_yytext ();
 std::string cls = strip_trailing_whitespace (yytxt).substr (1);
 size_t pos = cls.find (".");
 // Figure out exactly what kind of token to return when we have seen
 // an identifier.  Handles keywords.  Return -1 if the identifier
 // should be ignored.
 int
-octave_lexer::handle_identifier (void)
+octave_base_lexer::handle_identifier (void)
 {
 char *yytxt = flex_yytext ();
 std::string tok = yytxt;
 return NAME;
 }
 void
-octave_lexer::maybe_warn_separator_insert (char sep)
+octave_base_lexer::maybe_warn_separator_insert (char sep)
 {
 std::string nm = fcn_file_full_name;
 if (nm.empty ())
 warning_with_id ("Octave:separator-insert",
 "potential auto-insertion of '%c' near line %d of file %s",
 sep, input_line_number, nm.c_str ());
 }
 void
-octave_lexer::gripe_single_quote_string (void)
+octave_base_lexer::gripe_single_quote_string (void)
 {
 std::string nm = fcn_file_full_name;
 if (nm.empty ())
 warning_with_id ("Octave:single-quote-string",
 "single quote delimited string near line %d of file %s",
 input_line_number, nm.c_str ());
 }
 void
-octave_lexer::gripe_matlab_incompatible (const std::string& msg)
+octave_base_lexer::gripe_matlab_incompatible (const std::string& msg)
 {
 std::string nm = fcn_file_full_name;
 if (nm.empty ())
 warning_with_id ("Octave:matlab-incompatible",
 "potential Matlab compatibility problem: %s near line %d offile %s",
 msg.c_str (), input_line_number, nm.c_str ());
 }
 void
-octave_lexer::maybe_gripe_matlab_incompatible_comment (char c)
+octave_base_lexer::maybe_gripe_matlab_incompatible_comment (char c)
 {
 if (c == '#')
 gripe_matlab_incompatible ("# used as comment character");
 }
 void
-octave_lexer::gripe_matlab_incompatible_continuation (void)
+octave_base_lexer::gripe_matlab_incompatible_continuation (void)
 {
 gripe_matlab_incompatible ("\\ used as line continuation marker");
 }
 void
-octave_lexer::gripe_matlab_incompatible_operator (const std::string& op)
+octave_base_lexer::gripe_matlab_incompatible_operator (const std::string& op)
 {
 std::string t = op;
 int n = t.length ();
 if (t[n-1] == '\n')
 t.resize (n-1);
 gripe_matlab_incompatible (t + " used as operator");
 }
 void
-octave_lexer::push_token (token *tok)
+octave_base_lexer::push_token (token *tok)
 {
 YYSTYPE *lval = yyget_lval (scanner);
 lval->tok_val = tok;
 tokens.push (tok);
 }
 token *
-octave_lexer::current_token (void)
+octave_base_lexer::current_token (void)
 {
 YYSTYPE *lval = yyget_lval (scanner);
 return lval->tok_val;
 }
 void
-octave_lexer::display_token (int tok)
+octave_base_lexer::display_token (int tok)
 {
 switch (tok)
 {
 case '=': std::cerr << "'='\n"; break;
 case ':': std::cerr << "':'\n"; break;
 break;
 }
 }
 void
-octave_lexer::fatal_error (const char *msg)
+octave_base_lexer::fatal_error (const char *msg)
 {
 error (msg);
 OCTAVE_QUIT;
 yy_fatal_error (msg, scanner);
 }
 void
-octave_lexer::lexer_debug (const char *pattern)
+octave_base_lexer::lexer_debug (const char *pattern)
 {
 if (lexer_debug_flag)
 {
 std::cerr << std::endl;
 std::cerr << "T: " << flex_yytext () << std::endl;
 }
 }
 void
-octave_lexer::push_start_state (int state)
+octave_base_lexer::push_start_state (int state)
 {
 OCTAVE_YYG;
 start_state_stack.push (state);
 BEGIN (start_state ());
 }
 void
-octave_lexer::pop_start_state (void)
+octave_base_lexer::pop_start_state (void)
 {
 OCTAVE_YYG;
 start_state_stack.pop ();
 BEGIN (start_state ());
 }
 void
-octave_lexer::clear_start_state (void)
+octave_base_lexer::clear_start_state (void)
 {
 while (! start_state_stack.empty ())
 start_state_stack.pop ();
 push_start_state (INITIAL);
 }
 void
-octave_lexer::display_start_state (void) const
+octave_base_lexer::display_start_state (void) const
 {
 std::cerr << "S: ";
 switch (start_state ())
 {
 break;
 }
 }
 int
-octave_lexer::handle_op (const char *pattern, int tok, bool bos)
+octave_base_lexer::handle_op (const char *pattern, int tok, bool bos)
 {
 lexer_debug (pattern);
 return handle_op_internal (tok, bos, true);
 }
 int
-octave_lexer::handle_incompatible_op (const char *pattern, int tok, bool bos)
+octave_base_lexer::handle_incompatible_op (const char *pattern, int tok,
+bool bos)
 {
 lexer_debug (pattern);
 return handle_op_internal (tok, bos, false);
 }
 bool
-octave_lexer::maybe_unput_comma_before_unary_op (int tok)
+octave_base_lexer::maybe_unput_comma_before_unary_op (int tok)
 {
 int prev_tok = previous_token_value ();
 bool unput_comma = false;
 return unput_comma;
 }
 int
-octave_lexer::handle_unary_op (int tok, bool bos)
+octave_base_lexer::handle_unary_op (int tok, bool bos)
 {
 return maybe_unput_comma_before_unary_op (tok)
 ? -1 : handle_op_internal (tok, bos, true);
 }
 int
-octave_lexer::handle_incompatible_unary_op (int tok, bool bos)
+octave_base_lexer::handle_incompatible_unary_op (int tok, bool bos)
 {
 return maybe_unput_comma_before_unary_op (tok)
 ? -1 : handle_op_internal (tok, bos, false);
 }
 int
-octave_lexer::handle_op_internal (int tok, bool bos, bool compat)
+octave_base_lexer::handle_op_internal (int tok, bool bos, bool compat)
 {
 if (! compat)
 gripe_matlab_incompatible_operator (flex_yytext ());
 push_token (new token (tok, input_line_number, current_input_column));
 return count_token (tok);
 }
 int
-octave_lexer::handle_token (const std::string& name, int tok)
+octave_base_lexer::handle_token (const std::string& name, int tok)
 {
 token *tok_val = new token (tok, name, input_line_number,
 current_input_column);
 return handle_token (tok, tok_val);
 }
 int
-octave_lexer::handle_token (int tok, token *tok_val)
+octave_base_lexer::handle_token (int tok, token *tok_val)
 {
 if (! tok_val)
 tok_val = new token (tok, input_line_number, current_input_column);
 push_token (tok_val);
 return count_token_internal (tok);
 }
 int
-octave_lexer::count_token (int tok)
+octave_base_lexer::count_token (int tok)
 {
 token *tok_val = new token (tok, input_line_number, current_input_column);
 push_token (tok_val);
 return count_token_internal (tok);
 }
 int
-octave_lexer::count_token_internal (int tok)
+octave_base_lexer::count_token_internal (int tok)
 {
 if (tok != '\n')
 {
 Vtoken_count++;
 token_count++;
 return show_token (tok);
 }
 int
-octave_lexer::show_token (int tok)
+octave_base_lexer::show_token (int tok)
 {
 if (Vdisplay_tokens)
 display_token (tok);
 if (lexer_debug_flag)
 std::cerr << std::endl;
 }
 return tok;
 }
+int
+octave_lexer::fill_flex_buffer (char *buf, unsigned max_size)
+{
+int status = 0;
+if (input_buf.empty ())
+{
+bool eof = false;
+current_input_line = input_reader.get_input (eof);
+input_buf.fill (current_input_line, eof);
+}
+if (! input_buf.empty ())
+status = input_buf.copy_chunk (buf, max_size);
+else
+{
+status = YY_NULL;
+if (! input_buf.at_eof ())
+fatal_error ("octave_base_lexer::fill_flex_buffer failed");
+}
+return status;
+}

Mercurial > hg > octave-nkf

comparison libinterp/parse-tree/lex.ll @ 16293:57e87ddfee14