Mercurial > hg > octave-nkf

comparison libinterp/parse-tree/lex.ll @ 18588:932aca9a7c57

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Allow multi-level classdef package. * lex.h (octave_base_lexer::fq_identifier_contains_keyword, octave_base_lexer::handle_fq_identifier, octave_base_lexer::enable_fq_identifier): New methods. * lex.ll (octave_base_lexer::fq_identifier_contains_keyword, octave_base_lexer::handle_fq_identifier, octave_base_lexer::enable_fq_identifier): Likewise. (FQ_IDENT_START, FQIDENT): New exclusive start condition and regex to handle fully-qualified identifier. (superclass reference rule, metaclass query rule): Use FQIDENT. (octave_base_lexer::handle_superclass_identifier, octave_base_lexer::handle_meta_identifier): Don't split package name. * parse.h (octave_base_parser::make_superclass_ref, octave_base_parser::make_meta_class_query): Remove package_nm argument. * pt-classdef.h (tree_classdef_superclass::id, tree_classdef_superclass::pkg): Removed members, replaced by cls_name. (tree_classdef_superclass::~tree_classdef_superclass): Remove deletion of removed members. (tree_classdef_superclass::cls_name): New member. (tree_classdef_superclass::tree_classdef_superclass): Initialize it. (tree_classdef_superclass::ident, tree_classdef_superclass::package): Removed methods. (tree_classdef_superclass::class_name): New method. * token.h (token::meta_name_token): Remove enum value. (token(int, std::string, std::string, std::string, int, int)): Remove constructor. (token::superclass_package_name, token::meta_package_name, token::meta_class_name): Remove methods. (token::sc::package_nm, token::mc): Remove union members. * token.cc (token(int, std::string, std::string, std::string, int, int)): Remove constructor. (token::~token): Remove case of meta_name_token. (token::superclass_package_name, token::meta_package_name, token::meta_class_name): Remove methods. * oct-parse.in.yy (FQ_IDENT): New terminal. (superclass_identifier): Adapt to changes in class token and class octave_base_parser. (meta_identifier): Likewise. (superclass_list): Add mid-rule to enable fully-qualified identifier. (superclass): Use FQ_IDENT. (octave_base_parser::make_superclass_ref, octave_base_parser::make_meta_class_query): Remove package_nm argument. * ov-classdef.cc (octave_classdef_superclass_ref::do_multi_index_op): Adapt to removal of package_nm argument. (F__meta_class_query__): Likewise. (cdef_class::make_meta_class): Adapt to changes in class tree_classdef_superclass.
author Michael Goffioul <michael.goffioul@gmail.com>
date Sat, 22 Feb 2014 19:56:17 -0500
parents 4e0d72145c5a
children f958e8cd6348
comparison
equal deleted inserted replaced
18585:e1b9c8277ada 18588:932aca9a7c57
51 %x BLOCK_COMMENT_START 51 %x BLOCK_COMMENT_START
52 %x LINE_COMMENT_START 52 %x LINE_COMMENT_START
53 53
54 %x DQ_STRING_START 54 %x DQ_STRING_START
55 %x SQ_STRING_START 55 %x SQ_STRING_START
56
57 %x FQ_IDENT_START
56 58
57 %{ 59 %{
58 60
59 #include <cctype> 61 #include <cctype>
60 #include <cstring> 62 #include <cstring>
309 S [ \t] 311 S [ \t]
310 NL ((\n)|(\r)|(\r\n)) 312 NL ((\n)|(\r)|(\r\n))
311 Im [iIjJ] 313 Im [iIjJ]
312 CCHAR [#%] 314 CCHAR [#%]
313 IDENT ([_$a-zA-Z][_$a-zA-Z0-9]*) 315 IDENT ([_$a-zA-Z][_$a-zA-Z0-9]*)
316 FQIDENT ({IDENT}(\.{IDENT})*)
314 EXPON ([DdEe][+-]?{D}+) 317 EXPON ([DdEe][+-]?{D}+)
315 NUMBER (({D}+\.?{D}*{EXPON}?)|(\.{D}+{EXPON}?)|(0[xX][0-9a-fA-F]+)) 318 NUMBER (({D}+\.?{D}*{EXPON}?)|(\.{D}+{EXPON}?)|(0[xX][0-9a-fA-F]+))
316 319
317 ANY_EXCEPT_NL [^\r\n] 320 ANY_EXCEPT_NL [^\r\n]
318 ANY_INCLUDING_NL (.|{NL}) 321 ANY_INCLUDING_NL (.|{NL})
1026 curr_lexer->current_input_column = 1; 1029 curr_lexer->current_input_column = 1;
1027 1030
1028 error ("unterminated character string constant"); 1031 error ("unterminated character string constant");
1029 1032
1030 return LEXICAL_ERROR; 1033 return LEXICAL_ERROR;
1034 }
1035
1036 %{
1037 // Fully-qualified identifiers (used for classdef).
1038 %}
1039
1040 <FQ_IDENT_START>{FQIDENT} {
1041 curr_lexer->lexer_debug ("<FQ_IDENT_START>{FQIDENT}");
1042 curr_lexer->pop_start_state ();
1043
1044 int id_tok = curr_lexer->handle_fq_identifier ();
1045
1046 if (id_tok >= 0)
1047 {
1048 curr_lexer->looking_for_object_index = true;
1049
1050 return curr_lexer->count_token_internal (id_tok);
1051 }
1052 }
1053
1054 <FQ_IDENT_START>{S}+ {
1055 curr_lexer->current_input_column += yyleng;
1056
1057 curr_lexer->mark_previous_token_trailing_space ();
1058 }
1059
1060 <FQ_IDENT_START>. {
1061 yyless (0);
1062 curr_lexer->pop_start_state ();
1031 } 1063 }
1032 1064
1033 %{ 1065 %{
1034 // Imaginary numbers. 1066 // Imaginary numbers.
1035 %} 1067 %}
1166 1198
1167 %{ 1199 %{
1168 // Superclass method identifiers. 1200 // Superclass method identifiers.
1169 %} 1201 %}
1170 1202
1171 {IDENT}@{IDENT} | 1203 {IDENT}@{FQIDENT} {
1172 {IDENT}@{IDENT}\.{IDENT} { 1204 curr_lexer->lexer_debug ("{IDENT}@{FQIDENT}");
1173 curr_lexer->lexer_debug ("{IDENT}@{IDENT}|{IDENT}@{IDENT}\\.{IDENT}");
1174 1205
1175 if (curr_lexer->previous_token_may_be_command ()) 1206 if (curr_lexer->previous_token_may_be_command ())
1176 { 1207 {
1177 yyless (0); 1208 yyless (0);
1178 curr_lexer->push_start_state (COMMAND_START); 1209 curr_lexer->push_start_state (COMMAND_START);
1192 1223
1193 %{ 1224 %{
1194 // Metaclass query 1225 // Metaclass query
1195 %} 1226 %}
1196 1227
1197 \?{IDENT} | 1228 \?{FQIDENT} {
1198 \?{IDENT}\.{IDENT} { 1229 curr_lexer->lexer_debug ("\\?{FQIDENT}");
1199 curr_lexer->lexer_debug ("\\?{IDENT}|\\?{IDENT}\\.{IDENT}");
1200 1230
1201 if (curr_lexer->previous_token_may_be_command () 1231 if (curr_lexer->previous_token_may_be_command ()
1202 && curr_lexer->space_follows_previous_token ()) 1232 && curr_lexer->space_follows_previous_token ())
1203 { 1233 {
1204 yyless (0); 1234 yyless (0);
2558 2588
2559 return 0; 2589 return 0;
2560 } 2590 }
2561 2591
2562 bool 2592 bool
2593 octave_base_lexer::fq_identifier_contains_keyword (const std::string& s)
2594 {
2595 size_t p1 = 0;
2596 size_t p2;
2597
2598 std::string s_part;
2599
2600 do
2601 {
2602 p2 = s.find ('.', p1);
2603
2604 if (p2 != std::string::npos)
2605 {
2606 s_part = s.substr (p1, p2 - p1);
2607 p1 = p2 + 1;
2608 }
2609 else
2610 s_part = s.substr (p1);
2611
2612 if (is_keyword_token (s_part))
2613 return true;
2614 }
2615 while (p2 != std::string::npos);
2616
2617 return false;
2618 }
2619
2620 bool
2563 octave_base_lexer::whitespace_is_significant (void) 2621 octave_base_lexer::whitespace_is_significant (void)
2564 { 2622 {
2565 return (nesting_level.is_bracket () 2623 return (nesting_level.is_bracket ()
2566 || (nesting_level.is_brace () 2624 || (nesting_level.is_brace ()
2567 && ! looking_at_object_index.front ())); 2625 && ! looking_at_object_index.front ()));
2735 2793
2736 size_t pos = meth.find ("@"); 2794 size_t pos = meth.find ("@");
2737 std::string cls = meth.substr (pos + 1); 2795 std::string cls = meth.substr (pos + 1);
2738 meth = meth.substr (0, pos); 2796 meth = meth.substr (0, pos);
2739 2797
2740 std::string pkg; 2798 bool kw_token = (is_keyword_token (meth)
2741 pos = cls.find ("."); 2799 || fq_identifier_contains_keyword (cls));
2742 if (pos != std::string::npos) 2800
2743 {
2744 pkg = cls.substr (0, pos);
2745 cls = cls.substr (pos + 1);
2746 }
2747
2748 int kw_token = (is_keyword_token (meth) || is_keyword_token (cls)
2749 || is_keyword_token (pkg));
2750 if (kw_token) 2801 if (kw_token)
2751 { 2802 {
2752 error ("method, class, and package names may not be keywords"); 2803 error ("method, class, and package names may not be keywords");
2753 return LEXICAL_ERROR; 2804 return LEXICAL_ERROR;
2754 } 2805 }
2755 2806
2756 push_token (new token (SUPERCLASSREF, meth, pkg, cls, 2807 push_token (new token (SUPERCLASSREF, meth, cls,
2757 input_line_number, current_input_column)); 2808 input_line_number, current_input_column));
2758 2809
2759 current_input_column += flex_yyleng (); 2810 current_input_column += flex_yyleng ();
2760 2811
2761 return SUPERCLASSREF; 2812 return SUPERCLASSREF;
2764 int 2815 int
2765 octave_base_lexer::handle_meta_identifier (void) 2816 octave_base_lexer::handle_meta_identifier (void)
2766 { 2817 {
2767 std::string cls = std::string(flex_yytext ()).substr (1); 2818 std::string cls = std::string(flex_yytext ()).substr (1);
2768 2819
2769 std::string pkg; 2820 if (fq_identifier_contains_keyword (cls))
2770 size_t pos = cls.find (".");
2771 if (pos != std::string::npos)
2772 {
2773 pkg = cls.substr (0, pos);
2774 cls = cls.substr (pos + 1);
2775 }
2776
2777 int kw_token = is_keyword_token (cls) || is_keyword_token (pkg);
2778 if (kw_token)
2779 { 2821 {
2780 error ("class and package names may not be keywords"); 2822 error ("class and package names may not be keywords");
2781 return LEXICAL_ERROR; 2823 return LEXICAL_ERROR;
2782 } 2824 }
2783 2825
2784 push_token (new token (METAQUERY, pkg, cls, input_line_number, 2826 push_token (new token (METAQUERY, cls, input_line_number,
2785 current_input_column)); 2827 current_input_column));
2786 2828
2787 current_input_column += flex_yyleng (); 2829 current_input_column += flex_yyleng ();
2788 2830
2789 return METAQUERY; 2831 return METAQUERY;
2832 }
2833
2834 int
2835 octave_base_lexer::handle_fq_identifier (void)
2836 {
2837 std::string tok = flex_yytext ();
2838
2839 if (fq_identifier_contains_keyword (tok))
2840 {
2841 error ("function, method, class, and package names may not be keywords");
2842 return LEXICAL_ERROR;
2843 }
2844
2845 push_token (new token (FQ_IDENT, tok, input_line_number,
2846 current_input_column));
2847
2848 current_input_column += flex_yyleng ();
2849
2850 return FQ_IDENT;
2790 } 2851 }
2791 2852
2792 // Figure out exactly what kind of token to return when we have seen 2853 // Figure out exactly what kind of token to return when we have seen
2793 // an identifier. Handles keywords. Return -1 if the identifier 2854 // an identifier. Handles keywords. Return -1 if the identifier
2794 // should be ignored. 2855 // should be ignored.
3344 } 3405 }
3345 3406
3346 return tok; 3407 return tok;
3347 } 3408 }
3348 3409
3410 void
3411 octave_base_lexer::enable_fq_identifier (void)
3412 {
3413 push_start_state (FQ_IDENT_START);
3414 }
3415
3349 int 3416 int
3350 octave_lexer::fill_flex_buffer (char *buf, unsigned max_size) 3417 octave_lexer::fill_flex_buffer (char *buf, unsigned max_size)
3351 { 3418 {
3352 int status = 0; 3419 int status = 0;
3353 3420