Lines Matching refs:token
58 static Idx fetch_number (re_string_t *input, re_token_t *token,
60 static int peek_token (re_token_t *token, re_string_t *input,
65 re_token_t *token, reg_syntax_t syntax,
68 re_token_t *token, reg_syntax_t syntax,
71 re_token_t *token, reg_syntax_t syntax,
74 re_token_t *token, reg_syntax_t syntax,
77 re_dfa_t *dfa, re_token_t *token,
80 re_token_t *token, reg_syntax_t syntax,
84 re_token_t *token, int token_len,
90 re_token_t *token);
120 const re_token_t *token);
1272 if (node->token.type == OP_BACK_REF && dfa->subexp_map) in optimize_subexps()
1274 int idx = node->token.opr.idx; in optimize_subexps()
1275 node->token.opr.idx = dfa->subexp_map[idx]; in optimize_subexps()
1276 dfa->used_bkref_map |= 1 << node->token.opr.idx; in optimize_subexps()
1279 else if (node->token.type == SUBEXP in optimize_subexps()
1280 && node->left && node->left->token.type == SUBEXP) in optimize_subexps()
1282 Idx other_idx = node->left->token.opr.idx; in optimize_subexps()
1288 dfa->subexp_map[other_idx] = dfa->subexp_map[node->token.opr.idx]; in optimize_subexps()
1304 if (node->left && node->left->token.type == SUBEXP) in lower_subexps()
1310 if (node->right && node->right->token.type == SUBEXP) in lower_subexps()
1333 && (node->token.opr.idx >= BITSET_WORD_BITS in lower_subexp()
1335 & ((bitset_word_t) 1 << node->token.opr.idx)))) in lower_subexp()
1350 op->token.opr.idx = cls->token.opr.idx = node->token.opr.idx; in lower_subexp()
1351 op->token.opt_subexp = cls->token.opt_subexp = node->token.opt_subexp; in lower_subexp()
1361 if (node->token.type == CONCAT) in calc_first()
1369 node->node_idx = re_dfa_add_node (dfa, node->token); in calc_first()
1372 if (node->token.type == ANCHOR) in calc_first()
1373 dfa->nodes[node->node_idx].constraint = node->token.opr.ctx_type; in calc_first()
1382 switch (node->token.type) in calc_next()
1409 switch (node->token.type) in link_nfa_nodes()
1445 if (node->token.type == OP_BACK_REF) in link_nfa_nodes()
1450 assert (!IS_EPSILON_NODE (node->token.type)); in link_nfa_nodes()
1766 peek_token (re_token_t *token, re_string_t *input, reg_syntax_t syntax) in peek_token() argument
1772 token->type = END_OF_RE; in peek_token()
1777 token->opr.c = c; in peek_token()
1779 token->word_char = 0; in peek_token()
1781 token->mb_partial = 0; in peek_token()
1785 token->type = CHARACTER; in peek_token()
1786 token->mb_partial = 1; in peek_token()
1795 token->type = BACK_SLASH; in peek_token()
1800 token->opr.c = c2; in peek_token()
1801 token->type = CHARACTER; in peek_token()
1807 token->word_char = IS_WIDE_WORD_CHAR (wc) != 0; in peek_token()
1811 token->word_char = IS_WORD_CHAR (c2) != 0; in peek_token()
1817 token->type = OP_ALT; in peek_token()
1823 token->type = OP_BACK_REF; in peek_token()
1824 token->opr.idx = c2 - '1'; in peek_token()
1830 token->type = ANCHOR; in peek_token()
1831 token->opr.ctx_type = WORD_FIRST; in peek_token()
1837 token->type = ANCHOR; in peek_token()
1838 token->opr.ctx_type = WORD_LAST; in peek_token()
1844 token->type = ANCHOR; in peek_token()
1845 token->opr.ctx_type = WORD_DELIM; in peek_token()
1851 token->type = ANCHOR; in peek_token()
1852 token->opr.ctx_type = NOT_WORD_DELIM; in peek_token()
1857 token->type = OP_WORD; in peek_token()
1861 token->type = OP_NOTWORD; in peek_token()
1865 token->type = OP_SPACE; in peek_token()
1869 token->type = OP_NOTSPACE; in peek_token()
1874 token->type = ANCHOR; in peek_token()
1875 token->opr.ctx_type = BUF_FIRST; in peek_token()
1881 token->type = ANCHOR; in peek_token()
1882 token->opr.ctx_type = BUF_LAST; in peek_token()
1887 token->type = OP_OPEN_SUBEXP; in peek_token()
1891 token->type = OP_CLOSE_SUBEXP; in peek_token()
1895 token->type = OP_DUP_PLUS; in peek_token()
1899 token->type = OP_DUP_QUESTION; in peek_token()
1903 token->type = OP_OPEN_DUP_NUM; in peek_token()
1907 token->type = OP_CLOSE_DUP_NUM; in peek_token()
1915 token->type = CHARACTER; in peek_token()
1920 token->word_char = IS_WIDE_WORD_CHAR (wc) != 0; in peek_token()
1924 token->word_char = IS_WORD_CHAR (token->opr.c); in peek_token()
1930 token->type = OP_ALT; in peek_token()
1934 token->type = OP_ALT; in peek_token()
1937 token->type = OP_DUP_ASTERISK; in peek_token()
1941 token->type = OP_DUP_PLUS; in peek_token()
1945 token->type = OP_DUP_QUESTION; in peek_token()
1949 token->type = OP_OPEN_DUP_NUM; in peek_token()
1953 token->type = OP_CLOSE_DUP_NUM; in peek_token()
1957 token->type = OP_OPEN_SUBEXP; in peek_token()
1961 token->type = OP_CLOSE_SUBEXP; in peek_token()
1964 token->type = OP_OPEN_BRACKET; in peek_token()
1967 token->type = OP_PERIOD; in peek_token()
1977 token->type = ANCHOR; in peek_token()
1978 token->opr.ctx_type = LINE_FIRST; in peek_token()
1991 token->type = ANCHOR; in peek_token()
1992 token->opr.ctx_type = LINE_LAST; in peek_token()
2005 peek_token_bracket (re_token_t *token, re_string_t *input, reg_syntax_t syntax) in peek_token_bracket() argument
2010 token->type = END_OF_RE; in peek_token_bracket()
2014 token->opr.c = c; in peek_token_bracket()
2020 token->type = CHARACTER; in peek_token_bracket()
2032 token->opr.c = c2; in peek_token_bracket()
2033 token->type = CHARACTER; in peek_token_bracket()
2044 token->opr.c = c2; in peek_token_bracket()
2049 token->type = OP_OPEN_COLL_ELEM; in peek_token_bracket()
2052 token->type = OP_OPEN_EQUIV_CLASS; in peek_token_bracket()
2057 token->type = OP_OPEN_CHAR_CLASS; in peek_token_bracket()
2062 token->type = CHARACTER; in peek_token_bracket()
2063 token->opr.c = c; in peek_token_bracket()
2072 token->type = OP_CHARSET_RANGE; in peek_token_bracket()
2075 token->type = OP_CLOSE_BRACKET; in peek_token_bracket()
2078 token->type = OP_NON_MATCH_LIST; in peek_token_bracket()
2081 token->type = CHARACTER; in peek_token_bracket()
2135 parse_reg_exp (re_string_t *regexp, regex_t *preg, re_token_t *token, in parse_reg_exp() argument
2140 tree = parse_branch (regexp, preg, token, syntax, nest, err); in parse_reg_exp()
2144 while (token->type == OP_ALT) in parse_reg_exp()
2146 fetch_token (token, regexp, syntax | RE_CARET_ANCHORS_HERE); in parse_reg_exp()
2147 if (token->type != OP_ALT && token->type != END_OF_RE in parse_reg_exp()
2148 && (nest == 0 || token->type != OP_CLOSE_SUBEXP)) in parse_reg_exp()
2150 branch = parse_branch (regexp, preg, token, syntax, nest, err); in parse_reg_exp()
2176 parse_branch (re_string_t *regexp, regex_t *preg, re_token_t *token, in parse_branch() argument
2181 tree = parse_expression (regexp, preg, token, syntax, nest, err); in parse_branch()
2185 while (token->type != OP_ALT && token->type != END_OF_RE in parse_branch()
2186 && (nest == 0 || token->type != OP_CLOSE_SUBEXP)) in parse_branch()
2188 expr = parse_expression (regexp, preg, token, syntax, nest, err); in parse_branch()
2216 parse_expression (re_string_t *regexp, regex_t *preg, re_token_t *token, in parse_expression() argument
2221 switch (token->type) in parse_expression()
2224 tree = create_token_tree (dfa, NULL, NULL, token); in parse_expression()
2237 fetch_token (token, regexp, syntax); in parse_expression()
2238 mbc_remain = create_token_tree (dfa, NULL, NULL, token); in parse_expression()
2250 tree = parse_sub_exp (regexp, preg, token, syntax, nest + 1, err); in parse_expression()
2255 tree = parse_bracket_exp (regexp, dfa, token, syntax, err); in parse_expression()
2260 if (!BE (dfa->completed_bkref_map & (1 << token->opr.idx), 1)) in parse_expression()
2265 dfa->used_bkref_map |= 1 << token->opr.idx; in parse_expression()
2266 tree = create_token_tree (dfa, NULL, NULL, token); in parse_expression()
2292 fetch_token (token, regexp, syntax); in parse_expression()
2293 return parse_expression (regexp, preg, token, syntax, nest, err); in parse_expression()
2297 if ((token->type == OP_CLOSE_SUBEXP) && in parse_expression()
2308 token->type = CHARACTER; in parse_expression()
2311 tree = create_token_tree (dfa, NULL, NULL, token); in parse_expression()
2319 if ((token->opr.ctx_type in parse_expression()
2323 if (token->opr.ctx_type == WORD_DELIM in parse_expression()
2324 || token->opr.ctx_type == NOT_WORD_DELIM) in parse_expression()
2327 if (token->opr.ctx_type == WORD_DELIM) in parse_expression()
2329 token->opr.ctx_type = WORD_FIRST; in parse_expression()
2330 tree_first = create_token_tree (dfa, NULL, NULL, token); in parse_expression()
2331 token->opr.ctx_type = WORD_LAST; in parse_expression()
2335 token->opr.ctx_type = INSIDE_WORD; in parse_expression()
2336 tree_first = create_token_tree (dfa, NULL, NULL, token); in parse_expression()
2337 token->opr.ctx_type = INSIDE_NOTWORD; in parse_expression()
2339 tree_last = create_token_tree (dfa, NULL, NULL, token); in parse_expression()
2349 tree = create_token_tree (dfa, NULL, NULL, token); in parse_expression()
2360 fetch_token (token, regexp, syntax); in parse_expression()
2363 tree = create_token_tree (dfa, NULL, NULL, token); in parse_expression()
2377 token->type == OP_NOTWORD, err); in parse_expression()
2386 token->type == OP_NOTSPACE, err); in parse_expression()
2403 fetch_token (token, regexp, syntax); in parse_expression()
2405 while (token->type == OP_DUP_ASTERISK || token->type == OP_DUP_PLUS in parse_expression()
2406 || token->type == OP_DUP_QUESTION || token->type == OP_OPEN_DUP_NUM) in parse_expression()
2408 tree = parse_dup_op (tree, regexp, dfa, token, syntax, err); in parse_expression()
2413 && (token->type == OP_DUP_ASTERISK in parse_expression()
2414 || token->type == OP_OPEN_DUP_NUM)) in parse_expression()
2432 parse_sub_exp (re_string_t *regexp, regex_t *preg, re_token_t *token, in parse_sub_exp() argument
2440 fetch_token (token, regexp, syntax | RE_CARET_ANCHORS_HERE); in parse_sub_exp()
2443 if (token->type == OP_CLOSE_SUBEXP) in parse_sub_exp()
2447 tree = parse_reg_exp (regexp, preg, token, syntax, nest, err); in parse_sub_exp()
2448 if (BE (*err == REG_NOERROR && token->type != OP_CLOSE_SUBEXP, 0)) in parse_sub_exp()
2463 tree->token.opr.idx = cur_nsub; in parse_sub_exp()
2471 re_token_t *token, reg_syntax_t syntax, reg_errcode_t *err) in parse_dup_op() argument
2475 re_token_t start_token = *token; in parse_dup_op()
2477 if (token->type == OP_OPEN_DUP_NUM) in parse_dup_op()
2480 start = fetch_number (regexp, token, syntax); in parse_dup_op()
2483 if (token->type == CHARACTER && token->opr.c == ',') in parse_dup_op()
2494 end = ((token->type == OP_CLOSE_DUP_NUM) ? start in parse_dup_op()
2495 : ((token->type == CHARACTER && token->opr.c == ',') in parse_dup_op()
2496 ? fetch_number (regexp, token, syntax) : REG_ERROR)); in parse_dup_op()
2503 if (token->type == END_OF_RE) in parse_dup_op()
2513 *token = start_token; in parse_dup_op()
2514 token->type = CHARACTER; in parse_dup_op()
2529 start = (token->type == OP_DUP_PLUS) ? 1 : 0; in parse_dup_op()
2530 end = (token->type == OP_DUP_QUESTION) ? 1 : REG_MISSING; in parse_dup_op()
2533 fetch_token (token, regexp, syntax); in parse_dup_op()
2565 if (elem->token.type == SUBEXP) in parse_dup_op()
2566 postorder (elem, mark_opt_subexp, (void *) (intptr_t) elem->token.opr.idx); in parse_dup_op()
2755 parse_bracket_exp (re_string_t *regexp, re_dfa_t *dfa, re_token_t *token, in parse_bracket_exp() argument
3063 token_len = peek_token_bracket (token, regexp, syntax); in parse_bracket_exp()
3064 if (BE (token->type == END_OF_RE, 0)) in parse_bracket_exp()
3069 if (token->type == OP_NON_MATCH_LIST) in parse_bracket_exp()
3078 token_len = peek_token_bracket (token, regexp, syntax); in parse_bracket_exp()
3079 if (BE (token->type == END_OF_RE, 0)) in parse_bracket_exp()
3087 if (token->type == OP_CLOSE_BRACKET) in parse_bracket_exp()
3088 token->type = CHARACTER; in parse_bracket_exp()
3101 ret = parse_bracket_element (&start_elem, regexp, token, token_len, dfa, in parse_bracket_exp()
3111 token_len = peek_token_bracket (token, regexp, syntax); in parse_bracket_exp()
3116 if (BE (token->type == END_OF_RE, 0)) in parse_bracket_exp()
3121 if (token->type == OP_CHARSET_RANGE) in parse_bracket_exp()
3134 token->type = CHARACTER; in parse_bracket_exp()
3152 token_len = peek_token_bracket (token, regexp, syntax); in parse_bracket_exp()
3227 if (BE (token->type == END_OF_RE, 0)) in parse_bracket_exp()
3232 if (token->type == OP_CLOSE_BRACKET) in parse_bracket_exp()
3314 re_token_t *token, int token_len, re_dfa_t *dfa, in parse_bracket_element() argument
3329 if (token->type == OP_OPEN_COLL_ELEM || token->type == OP_OPEN_CHAR_CLASS in parse_bracket_element()
3330 || token->type == OP_OPEN_EQUIV_CLASS) in parse_bracket_element()
3331 return parse_bracket_symbol (elem, regexp, token); in parse_bracket_element()
3332 if (BE (token->type == OP_CHARSET_RANGE, 0) && !accept_hyphen) in parse_bracket_element()
3344 elem->opr.ch = token->opr.c; in parse_bracket_element()
3354 re_token_t *token) in parse_bracket_symbol() argument
3356 unsigned char ch, delim = token->opr.c; in parse_bracket_symbol()
3364 if (token->type == OP_OPEN_CHAR_CLASS) in parse_bracket_symbol()
3376 switch (token->type) in parse_bracket_symbol()
3688 fetch_number (re_string_t *input, re_token_t *token, reg_syntax_t syntax) in fetch_number() argument
3694 fetch_token (token, input, syntax); in fetch_number()
3695 c = token->opr.c; in fetch_number()
3696 if (BE (token->type == END_OF_RE, 0)) in fetch_number()
3698 if (token->type == OP_CLOSE_DUP_NUM || c == ',') in fetch_number()
3700 num = ((token->type != CHARACTER || c < '0' || '9' < c in fetch_number()
3740 const re_token_t *token) in create_token_tree() argument
3758 tree->token = *token; in create_token_tree()
3759 tree->token.duplicated = 0; in create_token_tree()
3760 tree->token.opt_subexp = 0; in create_token_tree()
3780 if (node->token.type == SUBEXP && node->token.opr.idx == idx) in mark_opt_subexp()
3781 node->token.opt_subexp = 1; in mark_opt_subexp()
3806 free_token (&node->token); in free_tree()
3826 *p_new = create_token_tree (dfa, NULL, NULL, &node->token); in duplicate_tree()
3830 (*p_new)->token.duplicated = 1; in duplicate_tree()