changeset 20799:e34692daf663

Extend parser to accept '_' in numbers. * NEWS: Announce change. * lex.ll: Define NUMBER to be a NUMREAL (real number) or NUMHEX (hex number). Define NUMREAL to begin with a digit (D) followed by more digits or an '_'. Define NUMHEX to begin with 0[xX], a hex digit (a-fA-F0-9), followed by '_' or more hex digits. Define EXPON to have a digit (D) followed by more digits or an '_'. * lex.ll (octave_base_lexer::handle_number): Strip out any underscores before processing number with sscanf. * parser.tst: Add tests for new behavior.
author Rik <rik@octave.org>
date Mon, 05 Oct 2015 14:05:58 -0700
parents 128414587af2
children 7c0e10f035bd
files NEWS libinterp/parse-tree/lex.ll test/parser.tst
diffstat 3 files changed, 37 insertions(+), 12 deletions(-) [+]
line wrap: on
line diff
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,12 @@
 Summary of important user-visible changes for version 4.2:
 ---------------------------------------------------------
 
+ ** The parser has been extended to accept, but ignore, underscore characters
+    in numbers.  This facilitates writing more legible code by using '_' as
+    a thousands separator or to group nibbles into bytes in hex constants.
+
+    Examples: 1_000_000 == 1e6 or 0xDE_AD_BE_EF
+
  ** The default colormap is now set to 'viridis' which is also
     the default colormap in matplotlib. This new colormap fixes
     some of the main issues with the old default colormap 'jet'
--- a/libinterp/parse-tree/lex.ll
+++ b/libinterp/parse-tree/lex.ll
@@ -320,14 +320,17 @@
 %}
 
 D       [0-9]
+D_      [0-9_]
 S       [ \t]
 NL      ((\n)|(\r)|(\r\n))
 Im      [iIjJ]
 CCHAR   [#%]
 IDENT   ([_$a-zA-Z][_$a-zA-Z0-9]*)
 FQIDENT ({IDENT}(\.{IDENT})*)
-EXPON   ([DdEe][+-]?{D}+)
-NUMBER  (({D}+\.?{D}*{EXPON}?)|(\.{D}+{EXPON}?)|(0[xX][0-9a-fA-F]+))
+EXPON   ([DdEe][+-]?{D}{D_}*)
+NUMREAL (({D}{D_}*\.?{D_}*{EXPON}?)|(\.{D}{D_}*{EXPON}?))
+NUMHEX  (0[xX][0-9a-fA-F][0-9a-fA-F_]*)
+NUMBER  ({NUMREAL}|{NUMHEX})
 
 ANY_EXCEPT_NL [^\r\n]
 ANY_INCLUDING_NL (.|{NL})
@@ -1124,9 +1127,9 @@
 // the constant.
 %}
 
-{D}+/\.[\*/\\^\'] |
+{D}{D_}*/\.[\*/\\^\'] |
 {NUMBER} {
-    curr_lexer->lexer_debug ("{D}+/\\.[\\*/\\\\^\\']|{NUMBER}");
+    curr_lexer->lexer_debug ("{D}{D_}*/\\.[\\*/\\\\^\\']|{NUMBER}");
 
     if (curr_lexer->previous_token_may_be_command ()
         &&  curr_lexer->space_follows_previous_token ())
@@ -2668,28 +2671,37 @@
 
   char *yytxt = flex_yytext ();
 
-  if (looks_like_hex (yytxt, strlen (yytxt)))
+  // Strip any underscores
+  char *tmptxt = strsave (yytxt);
+  char *rptr = tmptxt;
+  char *wptr = tmptxt;
+  while (*rptr)
+  {
+    *wptr = *rptr++;
+     wptr += (*wptr != '_');
+  }
+  *wptr = '\0';
+
+  if (looks_like_hex (tmptxt, strlen (tmptxt)))
     {
       unsigned long ival;
 
-      nread = sscanf (yytxt, "%lx", &ival);
+      nread = sscanf (tmptxt, "%lx", &ival);
 
       value = static_cast<double> (ival);
     }
   else
     {
-      char *tmp = strsave (yytxt);
-
-      char *idx = strpbrk (tmp, "Dd");
+      char *idx = strpbrk (tmptxt, "Dd");
 
       if (idx)
         *idx = 'e';
 
-      nread = sscanf (tmp, "%lf", &value);
-
-      delete [] tmp;
+      nread = sscanf (tmptxt, "%lf", &value);
     }
 
+  delete [] tmptxt;
+
   // If yytext doesn't contain a valid number, we are in deep doo doo.
 
   assert (nread == 1);
--- a/test/parser.tst
+++ b/test/parser.tst
@@ -275,6 +275,13 @@
 %! assert (a += b *= c += 1, 42);
 %! assert (b == 40 && c == 8);
 
+## Test extended number format which allows '_' as NOP character
+%!assert (123_456, 123456)
+%!assert (.123_456, .123456)
+%!assert (123_456.123_456, 123456.123456)
+%!assert (0xAB_CD, 43981)
+%!assert (2e0_1, 20);
+
 ## Test creation of anonymous functions
 
 %!test