# HG changeset patch
# User John W. Eaton <jwe@octave.org>
# Date 1362812542 18000
# Node ID 2b15ae55c72116bc4aaec09d6f186c9d1fe7ef6f
# Parent  4bf907906134c574f8603dd6d137c4d0a83d82df
put all tokens in the token cache

* lex.h, lex.ll (octave_lexer::handle_token (int, token *)): New arg,
tok_val, with default value 0.  If tok_val is 0, create token object.
Put token object in the token cache.
(octave_lexer::handle_token (const std::string&, int)): Rename from
push_token.  Change all uses.  Create token and pass to
octave_lexer::handle_token (int, tok_val *).

diff --git a/libinterp/parse-tree/lex.h b/libinterp/parse-tree/lex.h
--- a/libinterp/parse-tree/lex.h
+++ b/libinterp/parse-tree/lex.h
@@ -600,9 +600,9 @@
   int handle_op_internal (const char *pattern, int tok, bool convert,
                           bool bos, bool qit, bool compat);
 
-  int push_token (const std::string& name, int tok);
+  int handle_token (const std::string& name, int tok);
 
-  int handle_token (int tok);
+  int handle_token (int tok, token *tok_val = 0);
 
   int count_token (int tok);
 
diff --git a/libinterp/parse-tree/lex.ll b/libinterp/parse-tree/lex.ll
--- a/libinterp/parse-tree/lex.ll
+++ b/libinterp/parse-tree/lex.ll
@@ -235,7 +235,7 @@
     curr_lexer->looking_for_object_index = false;
     curr_lexer->at_beginning_of_statement = false;
 
-    return curr_lexer->push_token (tok, SQ_STRING);
+    return curr_lexer->handle_token (tok, SQ_STRING);
   }
 
 %{
@@ -3633,16 +3633,21 @@
 }
 
 int
-octave_lexer::push_token (const std::string& name, int tok)
+octave_lexer::handle_token (const std::string& name, int tok)
 {
-  push_token (new token (name, input_line_number, current_input_column));
-
-  return handle_token (tok);
+  token *tok_val = new token (name, input_line_number, current_input_column);
+
+  return handle_token (tok, tok_val);
 }
 
 int
-octave_lexer::handle_token (int tok)
+octave_lexer::handle_token (int tok, token *tok_val)
 {
+  if (! tok_val)
+    tok_val = new token (input_line_number, current_input_column);
+
+  push_token (tok_val);
+
   current_input_column += flex_yyleng ();
   quote_is_transpose = false;
   convert_spaces_to_comma = true;