token.py 2.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140
  1. """Token constants (from "token.h")."""
  2. # This file is automatically generated; please don't muck it up!
  3. #
  4. # To update the symbols in this file, 'cd' to the top directory of
  5. # the python source tree after building the interpreter and run:
  6. #
  7. # ./python Lib/token.py
  8. #--start constants--
  9. ENDMARKER = 0
  10. NAME = 1
  11. NUMBER = 2
  12. STRING = 3
  13. NEWLINE = 4
  14. INDENT = 5
  15. DEDENT = 6
  16. LPAR = 7
  17. RPAR = 8
  18. LSQB = 9
  19. RSQB = 10
  20. COLON = 11
  21. COMMA = 12
  22. SEMI = 13
  23. PLUS = 14
  24. MINUS = 15
  25. STAR = 16
  26. SLASH = 17
  27. VBAR = 18
  28. AMPER = 19
  29. LESS = 20
  30. GREATER = 21
  31. EQUAL = 22
  32. DOT = 23
  33. PERCENT = 24
  34. BACKQUOTE = 25
  35. LBRACE = 26
  36. RBRACE = 27
  37. EQEQUAL = 28
  38. NOTEQUAL = 29
  39. LESSEQUAL = 30
  40. GREATEREQUAL = 31
  41. TILDE = 32
  42. CIRCUMFLEX = 33
  43. LEFTSHIFT = 34
  44. RIGHTSHIFT = 35
  45. DOUBLESTAR = 36
  46. PLUSEQUAL = 37
  47. MINEQUAL = 38
  48. STAREQUAL = 39
  49. SLASHEQUAL = 40
  50. PERCENTEQUAL = 41
  51. AMPEREQUAL = 42
  52. VBAREQUAL = 43
  53. CIRCUMFLEXEQUAL = 44
  54. LEFTSHIFTEQUAL = 45
  55. RIGHTSHIFTEQUAL = 46
  56. DOUBLESTAREQUAL = 47
  57. DOUBLESLASH = 48
  58. DOUBLESLASHEQUAL = 49
  59. AT = 50
  60. OP = 51
  61. ERRORTOKEN = 52
  62. N_TOKENS = 53
  63. NT_OFFSET = 256
  64. #--end constants--
  65. tok_name = {}
  66. for _name, _value in globals().items():
  67. if type(_value) is type(0):
  68. tok_name[_value] = _name
  69. del _name, _value
  70. def ISTERMINAL(x):
  71. return x < NT_OFFSET
  72. def ISNONTERMINAL(x):
  73. return x >= NT_OFFSET
  74. def ISEOF(x):
  75. return x == ENDMARKER
  76. def main():
  77. import re
  78. import sys
  79. args = sys.argv[1:]
  80. inFileName = args and args[0] or "Include/token.h"
  81. outFileName = "Lib/token.py"
  82. if len(args) > 1:
  83. outFileName = args[1]
  84. try:
  85. fp = open(inFileName)
  86. except IOError, err:
  87. sys.stdout.write("I/O error: %s\n" % str(err))
  88. sys.exit(1)
  89. lines = fp.read().split("\n")
  90. fp.close()
  91. prog = re.compile(
  92. "#define[ \t][ \t]*([A-Z0-9][A-Z0-9_]*)[ \t][ \t]*([0-9][0-9]*)",
  93. re.IGNORECASE)
  94. tokens = {}
  95. for line in lines:
  96. match = prog.match(line)
  97. if match:
  98. name, val = match.group(1, 2)
  99. val = int(val)
  100. tokens[val] = name # reverse so we can sort them...
  101. keys = tokens.keys()
  102. keys.sort()
  103. # load the output skeleton from the target:
  104. try:
  105. fp = open(outFileName)
  106. except IOError, err:
  107. sys.stderr.write("I/O error: %s\n" % str(err))
  108. sys.exit(2)
  109. format = fp.read().split("\n")
  110. fp.close()
  111. try:
  112. start = format.index("#--start constants--") + 1
  113. end = format.index("#--end constants--")
  114. except ValueError:
  115. sys.stderr.write("target does not contain format markers")
  116. sys.exit(3)
  117. lines = []
  118. for val in keys:
  119. lines.append("%s = %d" % (tokens[val], val))
  120. format[start:end] = lines
  121. try:
  122. fp = open(outFileName, 'w')
  123. except IOError, err:
  124. sys.stderr.write("I/O error: %s\n" % str(err))
  125. sys.exit(4)
  126. fp.write("\n".join(format))
  127. fp.close()
  128. if __name__ == "__main__":
  129. main()