aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
-rw-r--r--psi/Grammar/Token26
1 files changed, 2 insertions, 24 deletions
diff --git a/psi/Grammar/Token b/psi/Grammar/Token
index f73152f..61dd757 100644
--- a/psi/Grammar/Token
+++ b/psi/Grammar/Token
@@ -1,11 +1,3 @@
-ENDMARKER
-NAME
-NUMBER
-STRING
-NEWLINE
-INDENT
-DEDENT
-
LPAR '('
RPAR ')'
LSQB '['
@@ -54,19 +46,5 @@ RARROW '->'
ELLIPSIS '...'
COLONEQUAL ':='
EXCLAMATION '!'
-
-OP
-AWAIT
-ASYNC
-TYPE_IGNORE
-TYPE_COMMENT
-SOFT_KEYWORD
-FSTRING_START
-FSTRING_MIDDLE
-FSTRING_END
-COMMENT
-NL
-ERRORTOKEN
-
-# These aren't used by the C tokenizer but are needed for tokenize.py
-ENCODING \ No newline at end of file
+INTEGER 'INTEGER'
+EOF 'EOF' \ No newline at end of file