llcppg/ast:marco & token

2024-08-16 15:03:00 +08:00
parent 876aea39e5
commit 961d4c4a3a
2 changed files with 38 additions and 0 deletions
--- a/chore/llcppg/token/token.go
+++ b/chore/llcppg/token/token.go
@@ -0,0 +1,27 @@
+package token
+
+type Token uint
+
+const (
+	/**
+	 * A token that contains some kind of punctuation.
+	 */
+	PunctuationToken Token = iota
+	/**
+	 * A language keyword.
+	 */
+	KeywordToken
+
+	/**
+	 * An identifier (that is not a keyword).
+	 */
+	IdentifierToken
+	/**
+	 * A numeric, string, or character literal.
+	 */
+	LiteralToken
+	/**
+	 * A comment.
+	 */
+	CommentToken
+)