46 static struct tok_state *tok_new(void);
47 static int tok_nextc(struct tok_state *tok);
48 static void tok_backup(struct tok_state *tok, int c);
119 static struct tok_state *
122     struct tok_state *tok = (struct tok_state *)PyMem_MALLOC(  in tok_new()
123                                             sizeof(struct tok_state));  in tok_new()
156 new_string(const char *s, Py_ssize_t len, struct tok_state *tok)  in new_string()
171 decoding_fgets(char *s, int size, struct tok_state *tok)  in decoding_fgets()
177 decoding_feof(struct tok_state *tok)  in decoding_feof()
183 decode_str(const char *str, int exec_input, struct tok_state *tok)  in decode_str()
191 error_ret(struct tok_state *tok) /* XXX */  in error_ret()
234 get_coding_spec(const char *s, char **spec, Py_ssize_t size, struct tok_state *tok)  in get_coding_spec()
288 check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,  in check_coding_spec()
289                   int set_readline(struct tok_state *, const char *))  in check_coding_spec()  argument
347 check_bom(int get_char(struct tok_state *),  in check_bom()  argument
348           void unget_char(int, struct tok_state *),  in check_bom()
349           int set_readline(struct tok_state *, const char *),  in check_bom()  argument
350           struct tok_state *tok)  in check_bom()
424 fp_readl(char *s, int size, struct tok_state *tok)  in fp_readl()
495 fp_setreadl(struct tok_state *tok, const char* enc)  in fp_setreadl()
544 static int fp_getc(struct tok_state *tok) {  in fp_getc()
550 static void fp_ungetc(int c, struct tok_state *tok) {  in fp_ungetc()
586 decoding_fgets(char *s, int size, struct tok_state *tok)  in decoding_fgets()
643 decoding_feof(struct tok_state *tok)  in decoding_feof()
665 buf_getc(struct tok_state *tok) {  in buf_getc()
672 buf_ungetc(int c, struct tok_state *tok) {  in buf_ungetc()
681 buf_setreadl(struct tok_state *tok, const char* enc) {  in buf_setreadl()
702 translate_newlines(const char *s, int exec_input, struct tok_state *tok) {  in translate_newlines()
747 decode_str(const char *input, int single, struct tok_state *tok)  in decode_str()
806 struct tok_state *
809     struct tok_state *tok = tok_new();  in PyTokenizer_FromString()
823 struct tok_state *
826     struct tok_state *tok = tok_new();  in PyTokenizer_FromUTF8()
854 struct tok_state *
858     struct tok_state *tok = tok_new();  in PyTokenizer_FromFile()
888 PyTokenizer_Free(struct tok_state *tok)  in PyTokenizer_Free()
907 tok_nextc(struct tok_state *tok)  in tok_nextc()
1106 tok_backup(struct tok_state *tok, int c)  in tok_backup()
1289 indenterror(struct tok_state *tok)  in indenterror()
1303 verify_identifier(struct tok_state *tok)  in verify_identifier()
1328 tok_decimal_tail(struct tok_state *tok)  in tok_decimal_tail()
1352 tok_get(struct tok_state *tok, char **p_start, char **p_end)  in tok_get()
1822 PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end)  in PyTokenizer_Get()
1845     struct tok_state *tok;  in PyTokenizer_FindEncodingFilename()