token.go 4.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225
  1. package token
  2. import "strconv"
  3. var keywords map[string]Token
  4. // Token represents a token.
  5. type Token int
  6. // List of tokens
  7. const (
  8. Illegal Token = iota
  9. EOF
  10. Comment
  11. _literalBeg
  12. Ident
  13. Int
  14. Float
  15. Char
  16. String
  17. _literalEnd
  18. _operatorBeg
  19. Add // +
  20. Sub // -
  21. Mul // *
  22. Quo // /
  23. Rem // %
  24. And // &
  25. Or // |
  26. Xor // ^
  27. Shl // <<
  28. Shr // >>
  29. AndNot // &^
  30. AddAssign // +=
  31. SubAssign // -=
  32. MulAssign // *=
  33. QuoAssign // /=
  34. RemAssign // %=
  35. AndAssign // &=
  36. OrAssign // |=
  37. XorAssign // ^=
  38. ShlAssign // <<=
  39. ShrAssign // >>=
  40. AndNotAssign // &^=
  41. LAnd // &&
  42. LOr // ||
  43. Inc // ++
  44. Dec // --
  45. Equal // ==
  46. Less // <
  47. Greater // >
  48. Assign // =
  49. Not // !
  50. NotEqual // !=
  51. LessEq // <=
  52. GreaterEq // >=
  53. Define // :=
  54. Ellipsis // ...
  55. LParen // (
  56. LBrack // [
  57. LBrace // {
  58. Comma // ,
  59. Period // .
  60. RParen // )
  61. RBrack // ]
  62. RBrace // }
  63. Semicolon // ;
  64. Colon // :
  65. Question // ?
  66. _operatorEnd
  67. _keywordBeg
  68. Break
  69. Continue
  70. Else
  71. For
  72. Func
  73. Error
  74. Immutable
  75. If
  76. Return
  77. Export
  78. True
  79. False
  80. In
  81. Undefined
  82. Import
  83. _keywordEnd
  84. )
  85. var tokens = [...]string{
  86. Illegal: "ILLEGAL",
  87. EOF: "EOF",
  88. Comment: "COMMENT",
  89. Ident: "IDENT",
  90. Int: "INT",
  91. Float: "FLOAT",
  92. Char: "CHAR",
  93. String: "STRING",
  94. Add: "+",
  95. Sub: "-",
  96. Mul: "*",
  97. Quo: "/",
  98. Rem: "%",
  99. And: "&",
  100. Or: "|",
  101. Xor: "^",
  102. Shl: "<<",
  103. Shr: ">>",
  104. AndNot: "&^",
  105. AddAssign: "+=",
  106. SubAssign: "-=",
  107. MulAssign: "*=",
  108. QuoAssign: "/=",
  109. RemAssign: "%=",
  110. AndAssign: "&=",
  111. OrAssign: "|=",
  112. XorAssign: "^=",
  113. ShlAssign: "<<=",
  114. ShrAssign: ">>=",
  115. AndNotAssign: "&^=",
  116. LAnd: "&&",
  117. LOr: "||",
  118. Inc: "++",
  119. Dec: "--",
  120. Equal: "==",
  121. Less: "<",
  122. Greater: ">",
  123. Assign: "=",
  124. Not: "!",
  125. NotEqual: "!=",
  126. LessEq: "<=",
  127. GreaterEq: ">=",
  128. Define: ":=",
  129. Ellipsis: "...",
  130. LParen: "(",
  131. LBrack: "[",
  132. LBrace: "{",
  133. Comma: ",",
  134. Period: ".",
  135. RParen: ")",
  136. RBrack: "]",
  137. RBrace: "}",
  138. Semicolon: ";",
  139. Colon: ":",
  140. Question: "?",
  141. Break: "break",
  142. Continue: "continue",
  143. Else: "else",
  144. For: "for",
  145. Func: "func",
  146. Error: "error",
  147. Immutable: "immutable",
  148. If: "if",
  149. Return: "return",
  150. Export: "export",
  151. True: "true",
  152. False: "false",
  153. In: "in",
  154. Undefined: "undefined",
  155. Import: "import",
  156. }
  157. func (tok Token) String() string {
  158. s := ""
  159. if 0 <= tok && tok < Token(len(tokens)) {
  160. s = tokens[tok]
  161. }
  162. if s == "" {
  163. s = "token(" + strconv.Itoa(int(tok)) + ")"
  164. }
  165. return s
  166. }
  167. // LowestPrec represents lowest operator precedence.
  168. const LowestPrec = 0
  169. // Precedence returns the precedence for the operator token.
  170. func (tok Token) Precedence() int {
  171. switch tok {
  172. case LOr:
  173. return 1
  174. case LAnd:
  175. return 2
  176. case Equal, NotEqual, Less, LessEq, Greater, GreaterEq:
  177. return 3
  178. case Add, Sub, Or, Xor:
  179. return 4
  180. case Mul, Quo, Rem, Shl, Shr, And, AndNot:
  181. return 5
  182. }
  183. return LowestPrec
  184. }
  185. // IsLiteral returns true if the token is a literal.
  186. func (tok Token) IsLiteral() bool {
  187. return _literalBeg < tok && tok < _literalEnd
  188. }
  189. // IsOperator returns true if the token is an operator.
  190. func (tok Token) IsOperator() bool {
  191. return _operatorBeg < tok && tok < _operatorEnd
  192. }
  193. // IsKeyword returns true if the token is a keyword.
  194. func (tok Token) IsKeyword() bool {
  195. return _keywordBeg < tok && tok < _keywordEnd
  196. }
  197. // Lookup returns corresponding keyword if ident is a keyword.
  198. func Lookup(ident string) Token {
  199. if tok, isKeyword := keywords[ident]; isKeyword {
  200. return tok
  201. }
  202. return Ident
  203. }
  204. func init() {
  205. keywords = make(map[string]Token)
  206. for i := _keywordBeg + 1; i < _keywordEnd; i++ {
  207. keywords[tokens[i]] = i
  208. }
  209. }