Mercurial > hg > octave-lyh
changeset 16230:4bf907906134
use a queue to hold tokens in the lexer
* lex.h, lex.ll (lexical_feedback::token_cache): New class.
(lexical_feedback::tokens): Rename from token_stack. Declare as
token_cache, not std::stack<token *>. Change all uses.
(lexical_feedback::reset_token_stack): Delete.
author | John W. Eaton <jwe@octave.org> |
---|---|
date | Sat, 09 Mar 2013 01:20:23 -0500 |
parents | 7b7b1e4968e8 |
children | 2b15ae55c721 |
files | libinterp/parse-tree/lex.h libinterp/parse-tree/lex.ll |
diffstat | 2 files changed, 70 insertions(+), 26 deletions(-) [+] |
line wrap: on
line diff
--- a/libinterp/parse-tree/lex.h +++ b/libinterp/parse-tree/lex.h @@ -23,12 +23,15 @@ #if !defined (octave_lex_h) #define octave_lex_h 1 +#include <deque> +#include <limits> #include <list> #include <set> #include <stack> #include "comment-list.h" #include "input.h" +#include "token.h" extern OCTINTERP_API void cleanup_parser (void); @@ -53,9 +56,6 @@ stream_reader& operator = (const stream_reader&); }; -// Forward decl for octave_lexer::token_stack. -class token; - // For communication between the lexer and parser. class @@ -160,6 +160,67 @@ std::stack<int> context; }; + + class token_cache + { + public: + + // Store an "unlimited" number of tokens. + token_cache (size_t sz_arg = std::numeric_limits<size_t>::max ()) + : buffer (), sz (sz_arg) + { } + + void push (token *tok) + { + if (buffer.size () == sz) + pop (); + + buffer.push_front (tok); + } + + void pop (void) + { + if (! empty ()) + { + delete buffer.back (); + buffer.pop_back (); + } + } + + // Direct access. + token *at (size_t n) { return buffer.at (n); } + const token *at (size_t n) const { return buffer.at (n); } + + // Most recently pushed. + token *front (void) { return buffer.front (); } + const token *front (void) const { return buffer.front (); } + + token *back (void) { return buffer.back (); } + const token *back (void) const { return buffer.back (); } + + // Number of elements currently in the buffer, max of sz. + size_t size (void) const { return buffer.size (); } + + bool empty (void) const { return buffer.empty (); } + + void clear (void) + { + while (! empty ()) + pop (); + } + + private: + + std::deque<token *> buffer; + + size_t sz; + + // No copying! + + token_cache (const token_cache&); + + token_cache& operator = (const token_cache&); + }; lexical_feedback (void) : end_of_input (false), convert_spaces_to_comma (true), @@ -181,7 +242,7 @@ current_input_line (), comment_text (), help_text (), fcn_file_name (), fcn_file_full_name (), looking_at_object_index (), parsed_function_name (), pending_local_variables (), - nesting_level (), token_stack () + nesting_level (), tokens () { init (); } @@ -322,15 +383,11 @@ // a paren? bbp_nesting_level nesting_level; - // Stack to hold tokens so that we can delete them when the parser is - // reset and avoid growing forever just because we are stashing some - // information. - std::stack <token*> token_stack; + // Tokens generated by the lexer. + token_cache tokens; private: - void reset_token_stack (void); - // No copying! lexical_feedback (const lexical_feedback&);
--- a/libinterp/parse-tree/lex.ll +++ b/libinterp/parse-tree/lex.ll @@ -1339,7 +1339,7 @@ lexical_feedback::~lexical_feedback (void) { - reset_token_stack (); + tokens.clear (); } void @@ -1397,7 +1397,7 @@ nesting_level.reset (); - reset_token_stack (); + tokens.clear (); } static bool @@ -1446,19 +1446,6 @@ } void -lexical_feedback::reset_token_stack (void) -{ - // Clear out the stack of token info used to track line and - // column numbers. - - while (! token_stack.empty ()) - { - delete token_stack.top (); - token_stack.pop (); - } -} - -void octave_lexer::input_buffer::fill (const std::string& input, bool eof_arg) { buffer = input; @@ -3374,7 +3361,7 @@ { YYSTYPE *lval = yyget_lval (scanner); lval->tok_val = tok; - token_stack.push (tok); + tokens.push (tok); } token *