; Íâ"Ic @s@dZddlmZddlmZmZmZmZmZmZm Z m Z m Z m Z ddl mZddlmZddlZddlZddlZd„Zd„Zd „ZGd „d eƒZGd „d eƒZGd„deƒZGd„deƒZied6ed6Zd„Zedko eƒndS(u6O Tests for the tokenize module. The tests can be really simple. Given a small fragment of source code, print out a table with tokens. The ENDMARK is omitted for brevity. >>> dump_tokens("1 + 1") ENCODING 'utf-8' (0, 0) (0, 0) NUMBER '1' (1, 0) (1, 1) OP '+' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) >>> dump_tokens("if False:\n" ... " # NL\n" ... " True = False # NEWLINE\n") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'if' (1, 0) (1, 2) NAME 'False' (1, 3) (1, 8) OP ':' (1, 8) (1, 9) NEWLINE '\n' (1, 9) (1, 10) COMMENT '# NL' (2, 4) (2, 8) NL '\n' (2, 8) (2, 9) INDENT ' ' (3, 0) (3, 4) NAME 'True' (3, 4) (3, 8) OP '=' (3, 9) (3, 10) NAME 'False' (3, 11) (3, 16) COMMENT '# NEWLINE' (3, 17) (3, 26) NEWLINE '\n' (3, 26) (3, 27) DEDENT '' (4, 0) (4, 0) >>> indent_error_file = """ ... def k(x): ... x += 2 ... x += 5 ... """ >>> readline = BytesIO(indent_error_file.encode('utf-8')).readline >>> for tok in tokenize(readline): pass Traceback (most recent call last): ... IndentationError: unindent does not match any outer indentation level There are some standard formattig practises that are easy to get right. >>> roundtrip("if x == 1:\n" ... " print(x)\n") True >>> roundtrip("# This is a comment\n# This also") True Some people use different formatting conventions, which makes untokenize a little trickier. Note that this test involves trailing whitespace after the colon. Note that we use hex escapes to make the two trailing blanks apparent in the expected output. >>> roundtrip("if x == 1 : \n" ... " print(x)\n") True >>> f = support.findfile("tokenize_tests.txt") >>> roundtrip(open(f, 'rb')) True >>> roundtrip("if x == 1:\n" ... " # A comment by itself.\n" ... " print(x) # Comment here, too.\n" ... " # Another comment.\n" ... "after_if = True\n") True >>> roundtrip("if (x # The comments need to go in the right place\n" ... " == 1):\n" ... " print('x==1')\n") True >>> roundtrip("class Test: # A comment here\n" ... " # A comment with weird indent\n" ... " after_com = 5\n" ... " def x(m): return m*5 # a one liner\n" ... " def y(m): # A whitespace after the colon\n" ... " return y*4 # 3-space indent\n") True Some error-handling code >>> roundtrip("try: import somemodule\n" ... "except ImportError: # comment\n" ... " print('Can not import' # comment2\n)" ... "else: print('Loaded')\n") True Balancing continuation >>> roundtrip("a = (3,4, \n" ... "5,6)\n" ... "y = [3, 4,\n" ... "5]\n" ... "z = {'a': 5,\n" ... "'b':15, 'c':True}\n" ... "x = len(y) + 5 - a[\n" ... "3] - a[2]\n" ... "+ len(z) - z[\n" ... "'b']\n") True Ordinary integers and binary operators >>> dump_tokens("0xff <= 255") ENCODING 'utf-8' (0, 0) (0, 0) NUMBER '0xff' (1, 0) (1, 4) OP '<=' (1, 5) (1, 7) NUMBER '255' (1, 8) (1, 11) >>> dump_tokens("0b10 <= 255") ENCODING 'utf-8' (0, 0) (0, 0) NUMBER '0b10' (1, 0) (1, 4) OP '<=' (1, 5) (1, 7) NUMBER '255' (1, 8) (1, 11) >>> dump_tokens("0o123 <= 0O123") ENCODING 'utf-8' (0, 0) (0, 0) NUMBER '0o123' (1, 0) (1, 5) OP '<=' (1, 6) (1, 8) NUMBER '0O123' (1, 9) (1, 14) >>> dump_tokens("1234567 > ~0x15") ENCODING 'utf-8' (0, 0) (0, 0) NUMBER '1234567' (1, 0) (1, 7) OP '>' (1, 8) (1, 9) OP '~' (1, 10) (1, 11) NUMBER '0x15' (1, 11) (1, 15) >>> dump_tokens("2134568 != 1231515") ENCODING 'utf-8' (0, 0) (0, 0) NUMBER '2134568' (1, 0) (1, 7) OP '!=' (1, 8) (1, 10) NUMBER '1231515' (1, 11) (1, 18) >>> dump_tokens("(-124561-1) & 200000000") ENCODING 'utf-8' (0, 0) (0, 0) OP '(' (1, 0) (1, 1) OP '-' (1, 1) (1, 2) NUMBER '124561' (1, 2) (1, 8) OP '-' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP ')' (1, 10) (1, 11) OP '&' (1, 12) (1, 13) NUMBER '200000000' (1, 14) (1, 23) >>> dump_tokens("0xdeadbeef != -1") ENCODING 'utf-8' (0, 0) (0, 0) NUMBER '0xdeadbeef' (1, 0) (1, 10) OP '!=' (1, 11) (1, 13) OP '-' (1, 14) (1, 15) NUMBER '1' (1, 15) (1, 16) >>> dump_tokens("0xdeadc0de & 12345") ENCODING 'utf-8' (0, 0) (0, 0) NUMBER '0xdeadc0de' (1, 0) (1, 10) OP '&' (1, 11) (1, 12) NUMBER '12345' (1, 13) (1, 18) >>> dump_tokens("0xFF & 0x15 | 1234") ENCODING 'utf-8' (0, 0) (0, 0) NUMBER '0xFF' (1, 0) (1, 4) OP '&' (1, 5) (1, 6) NUMBER '0x15' (1, 7) (1, 11) OP '|' (1, 12) (1, 13) NUMBER '1234' (1, 14) (1, 18) Long integers >>> dump_tokens("x = 0") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '0' (1, 4) (1, 5) >>> dump_tokens("x = 0xfffffffffff") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '0xffffffffff (1, 4) (1, 17) >>> dump_tokens("x = 123141242151251616110") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '123141242151 (1, 4) (1, 25) >>> dump_tokens("x = -15921590215012591") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) OP '-' (1, 4) (1, 5) NUMBER '159215902150 (1, 5) (1, 22) Floating point numbers >>> dump_tokens("x = 3.14159") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3.14159' (1, 4) (1, 11) >>> dump_tokens("x = 314159.") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '314159.' (1, 4) (1, 11) >>> dump_tokens("x = .314159") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '.314159' (1, 4) (1, 11) >>> dump_tokens("x = 3e14159") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3e14159' (1, 4) (1, 11) >>> dump_tokens("x = 3E123") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3E123' (1, 4) (1, 9) >>> dump_tokens("x+y = 3e-1230") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'x' (1, 0) (1, 1) OP '+' (1, 1) (1, 2) NAME 'y' (1, 2) (1, 3) OP '=' (1, 4) (1, 5) NUMBER '3e-1230' (1, 6) (1, 13) >>> dump_tokens("x = 3.14e159") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3.14e159' (1, 4) (1, 12) String literals >>> dump_tokens("x = ''; y = \"\"") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING "''" (1, 4) (1, 6) OP ';' (1, 6) (1, 7) NAME 'y' (1, 8) (1, 9) OP '=' (1, 10) (1, 11) STRING '""' (1, 12) (1, 14) >>> dump_tokens("x = '\"'; y = \"'\"") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING '\'"\'' (1, 4) (1, 7) OP ';' (1, 7) (1, 8) NAME 'y' (1, 9) (1, 10) OP '=' (1, 11) (1, 12) STRING '"\'"' (1, 13) (1, 16) >>> dump_tokens("x = \"doesn't \"shrink\", does it\"") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING '"doesn\'t "' (1, 4) (1, 14) NAME 'shrink' (1, 14) (1, 20) STRING '", does it"' (1, 20) (1, 31) >>> dump_tokens("x = 'abc' + 'ABC'") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING "'abc'" (1, 4) (1, 9) OP '+' (1, 10) (1, 11) STRING "'ABC'" (1, 12) (1, 17) >>> dump_tokens('y = "ABC" + "ABC"') ENCODING 'utf-8' (0, 0) (0, 0) NAME 'y' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING '"ABC"' (1, 4) (1, 9) OP '+' (1, 10) (1, 11) STRING '"ABC"' (1, 12) (1, 17) >>> dump_tokens("x = r'abc' + r'ABC' + R'ABC' + R'ABC'") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING "r'abc'" (1, 4) (1, 10) OP '+' (1, 11) (1, 12) STRING "r'ABC'" (1, 13) (1, 19) OP '+' (1, 20) (1, 21) STRING "R'ABC'" (1, 22) (1, 28) OP '+' (1, 29) (1, 30) STRING "R'ABC'" (1, 31) (1, 37) >>> dump_tokens('y = r"abc" + r"ABC" + R"ABC" + R"ABC"') ENCODING 'utf-8' (0, 0) (0, 0) NAME 'y' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING 'r"abc"' (1, 4) (1, 10) OP '+' (1, 11) (1, 12) STRING 'r"ABC"' (1, 13) (1, 19) OP '+' (1, 20) (1, 21) STRING 'R"ABC"' (1, 22) (1, 28) OP '+' (1, 29) (1, 30) STRING 'R"ABC"' (1, 31) (1, 37) Operators >>> dump_tokens("def d22(a, b, c=2, d=2, *k): pass") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'def' (1, 0) (1, 3) NAME 'd22' (1, 4) (1, 7) OP '(' (1, 7) (1, 8) NAME 'a' (1, 8) (1, 9) OP ',' (1, 9) (1, 10) NAME 'b' (1, 11) (1, 12) OP ',' (1, 12) (1, 13) NAME 'c' (1, 14) (1, 15) OP '=' (1, 15) (1, 16) NUMBER '2' (1, 16) (1, 17) OP ',' (1, 17) (1, 18) NAME 'd' (1, 19) (1, 20) OP '=' (1, 20) (1, 21) NUMBER '2' (1, 21) (1, 22) OP ',' (1, 22) (1, 23) OP '*' (1, 24) (1, 25) NAME 'k' (1, 25) (1, 26) OP ')' (1, 26) (1, 27) OP ':' (1, 27) (1, 28) NAME 'pass' (1, 29) (1, 33) >>> dump_tokens("def d01v_(a=1, *k, **w): pass") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'def' (1, 0) (1, 3) NAME 'd01v_' (1, 4) (1, 9) OP '(' (1, 9) (1, 10) NAME 'a' (1, 10) (1, 11) OP '=' (1, 11) (1, 12) NUMBER '1' (1, 12) (1, 13) OP ',' (1, 13) (1, 14) OP '*' (1, 15) (1, 16) NAME 'k' (1, 16) (1, 17) OP ',' (1, 17) (1, 18) OP '**' (1, 19) (1, 21) NAME 'w' (1, 21) (1, 22) OP ')' (1, 22) (1, 23) OP ':' (1, 23) (1, 24) NAME 'pass' (1, 25) (1, 29) Comparison >>> dump_tokens("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != " + ... "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'if' (1, 0) (1, 2) NUMBER '1' (1, 3) (1, 4) OP '<' (1, 5) (1, 6) NUMBER '1' (1, 7) (1, 8) OP '>' (1, 9) (1, 10) NUMBER '1' (1, 11) (1, 12) OP '==' (1, 13) (1, 15) NUMBER '1' (1, 16) (1, 17) OP '>=' (1, 18) (1, 20) NUMBER '5' (1, 21) (1, 22) OP '<=' (1, 23) (1, 25) NUMBER '0x15' (1, 26) (1, 30) OP '<=' (1, 31) (1, 33) NUMBER '0x12' (1, 34) (1, 38) OP '!=' (1, 39) (1, 41) NUMBER '1' (1, 42) (1, 43) NAME 'and' (1, 44) (1, 47) NUMBER '5' (1, 48) (1, 49) NAME 'in' (1, 50) (1, 52) NUMBER '1' (1, 53) (1, 54) NAME 'not' (1, 55) (1, 58) NAME 'in' (1, 59) (1, 61) NUMBER '1' (1, 62) (1, 63) NAME 'is' (1, 64) (1, 66) NUMBER '1' (1, 67) (1, 68) NAME 'or' (1, 69) (1, 71) NUMBER '5' (1, 72) (1, 73) NAME 'is' (1, 74) (1, 76) NAME 'not' (1, 77) (1, 80) NUMBER '1' (1, 81) (1, 82) OP ':' (1, 82) (1, 83) NAME 'pass' (1, 84) (1, 88) Shift >>> dump_tokens("x = 1 << 1 >> 5") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '<<' (1, 6) (1, 8) NUMBER '1' (1, 9) (1, 10) OP '>>' (1, 11) (1, 13) NUMBER '5' (1, 14) (1, 15) Additive >>> dump_tokens("x = 1 - y + 15 - 1 + 0x124 + z + a[5]") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '-' (1, 6) (1, 7) NAME 'y' (1, 8) (1, 9) OP '+' (1, 10) (1, 11) NUMBER '15' (1, 12) (1, 14) OP '-' (1, 15) (1, 16) NUMBER '1' (1, 17) (1, 18) OP '+' (1, 19) (1, 20) NUMBER '0x124' (1, 21) (1, 26) OP '+' (1, 27) (1, 28) NAME 'z' (1, 29) (1, 30) OP '+' (1, 31) (1, 32) NAME 'a' (1, 33) (1, 34) OP '[' (1, 34) (1, 35) NUMBER '5' (1, 35) (1, 36) OP ']' (1, 36) (1, 37) Multiplicative >>> dump_tokens("x = 1//1*1/5*12%0x12") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '//' (1, 5) (1, 7) NUMBER '1' (1, 7) (1, 8) OP '*' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP '/' (1, 10) (1, 11) NUMBER '5' (1, 11) (1, 12) OP '*' (1, 12) (1, 13) NUMBER '12' (1, 13) (1, 15) OP '%' (1, 15) (1, 16) NUMBER '0x12' (1, 16) (1, 20) Unary >>> dump_tokens("~1 ^ 1 & 1 |1 ^ -1") ENCODING 'utf-8' (0, 0) (0, 0) OP '~' (1, 0) (1, 1) NUMBER '1' (1, 1) (1, 2) OP '^' (1, 3) (1, 4) NUMBER '1' (1, 5) (1, 6) OP '&' (1, 7) (1, 8) NUMBER '1' (1, 9) (1, 10) OP '|' (1, 11) (1, 12) NUMBER '1' (1, 12) (1, 13) OP '^' (1, 14) (1, 15) OP '-' (1, 16) (1, 17) NUMBER '1' (1, 17) (1, 18) >>> dump_tokens("-1*1/1+1*1//1 - ---1**1") ENCODING 'utf-8' (0, 0) (0, 0) OP '-' (1, 0) (1, 1) NUMBER '1' (1, 1) (1, 2) OP '*' (1, 2) (1, 3) NUMBER '1' (1, 3) (1, 4) OP '/' (1, 4) (1, 5) NUMBER '1' (1, 5) (1, 6) OP '+' (1, 6) (1, 7) NUMBER '1' (1, 7) (1, 8) OP '*' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP '//' (1, 10) (1, 12) NUMBER '1' (1, 12) (1, 13) OP '-' (1, 14) (1, 15) OP '-' (1, 16) (1, 17) OP '-' (1, 17) (1, 18) OP '-' (1, 18) (1, 19) NUMBER '1' (1, 19) (1, 20) OP '**' (1, 20) (1, 22) NUMBER '1' (1, 22) (1, 23) Selector >>> dump_tokens("import sys, time\nx = sys.modules['time'].time()") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'import' (1, 0) (1, 6) NAME 'sys' (1, 7) (1, 10) OP ',' (1, 10) (1, 11) NAME 'time' (1, 12) (1, 16) NEWLINE '\n' (1, 16) (1, 17) NAME 'x' (2, 0) (2, 1) OP '=' (2, 2) (2, 3) NAME 'sys' (2, 4) (2, 7) OP '.' (2, 7) (2, 8) NAME 'modules' (2, 8) (2, 15) OP '[' (2, 15) (2, 16) STRING "'time'" (2, 16) (2, 22) OP ']' (2, 22) (2, 23) OP '.' (2, 23) (2, 24) NAME 'time' (2, 24) (2, 28) OP '(' (2, 28) (2, 29) OP ')' (2, 29) (2, 30) Methods >>> dump_tokens("@staticmethod\ndef foo(x,y): pass") ENCODING 'utf-8' (0, 0) (0, 0) OP '@' (1, 0) (1, 1) NAME 'staticmethod (1, 1) (1, 13) NEWLINE '\n' (1, 13) (1, 14) NAME 'def' (2, 0) (2, 3) NAME 'foo' (2, 4) (2, 7) OP '(' (2, 7) (2, 8) NAME 'x' (2, 8) (2, 9) OP ',' (2, 9) (2, 10) NAME 'y' (2, 10) (2, 11) OP ')' (2, 11) (2, 12) OP ':' (2, 12) (2, 13) NAME 'pass' (2, 14) (2, 18) Backslash means line continuation, except for comments >>> roundtrip("x=1+\\n" ... "1\n" ... "# This is a comment\\n" ... "# This also\n") True >>> roundtrip("# Comment \\nx = 0") True Two string literals on the same line >>> roundtrip("'' ''") True Test roundtrip on random python modules. pass the '-ucompiler' option to process the full directory. >>> import random >>> tempdir = os.path.dirname(f) or os.curdir >>> testfiles = glob.glob(os.path.join(tempdir, "test*.py")) >>> if not support.is_resource_enabled("compiler"): ... testfiles = random.sample(testfiles, 10) ... >>> for testfile in testfiles: ... if not roundtrip(open(testfile, 'rb')): ... print("Roundtrip failed for file %s" % testfile) ... break ... else: True True i(usupport( utokenizeu _tokenizeu untokenizeuNUMBERuNAMEuOPuSTRINGu ENDMARKERutok_nameudetect_encoding(uBytesIO(uTestCaseNcCsot|jdƒƒ}xSt|jƒD]B\}}}}}|tkoPnt|}tdtƒƒq%WdS(uPPrint out the tokens in s in a table format. The ENDMARKER is omitted. uutf-8u0%(type)-10.10s %(token)-13.13r %(start)s %(end)sN(uBytesIOuencodeutokenizeureadlineu ENDMARKERutok_nameuprintulocals(usufutypeutokenustartuenduline((u//mit/python/lib/python3.0/test/test_tokenize.pyu dump_tokenss  cCs t|tƒot|jdƒƒ}ntt|jƒƒ}|jƒd„|Dƒ}t|ƒ}d„|j dƒDƒj }d„t|ƒDƒ}||kS(u Test roundtrip for `untokenize`. `f` is an open file or a string. The source code in f is tokenized, converted back to source code via tokenize.untokenize(), and tokenized again from the latter. The test fails if the second tokenization doesn't match the first. uutf-8cSs%g}|]}||dd…q S(Ni((u.0u_[1]utok((u//mit/python/lib/python3.0/test/test_tokenize.pyu 6s css|] }|VqdS(N((u.0uline((u//mit/python/lib/python3.0/test/test_tokenize.pyu 8sicSs%g}|]}||dd…q S(Ni((u.0u_[1]utok((u//mit/python/lib/python3.0/test/test_tokenize.pyu 9s ( u isinstanceustruBytesIOuencodeulistutokenizeureadlineucloseu untokenizeu splitlinesu__next__(ufu token_listutokens1u new_bytesureadlineutokens2((u//mit/python/lib/python3.0/test/test_tokenize.pyu roundtrip+s  cCs¿g}tt|jdƒƒjƒ}x…|D]}\}}}}}|tkoHd|ko;|jtdftdftt |ƒftdfgƒq+|j ||fƒq+Wt |ƒj dƒS(uÎSubstitute Decimals for floats in a string of statements. >>> from decimal import Decimal >>> s = 'print(+21.3e-5*-.1234/81.7)' >>> decistmt(s) "print (+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7'))" The format of the exponent is inherited from the platform C library. Known cases are "e-007" (Windows) and "e-07" (not Windows). Since we're only showing 12 digits, and the 13th isn't close to 5, the rest of the output should be platform-independent. >>> exec(s) #doctest: +ELLIPSIS -3.21716034272e-0...7 Output from calculations with Decimal should be identical across all platforms. >>> exec(decistmt(s)) -3.217160342717258261933904529E-7 uutf-8u.uDecimalu(u)( utokenizeuBytesIOuencodeureadlineuNUMBERuextenduNAMEuOPuSTRINGurepruappendu untokenizeudecode(usuresultugutoknumutokvalu_((u//mit/python/lib/python3.0/test/test_tokenize.pyudecistmt=s  cBsA|EeZdZd„Zd„Zd„Zd„Zd„ZdS(uU Test that tokenizer adheres to the coding behaviour stipulated in PEP 0263. cCs4tjjtjjtƒ|ƒ}tt|dƒƒS(Nurb(uosupathujoinudirnameu__file__u roundtripuopen(uselfufilenameupath((u//mit/python/lib/python3.0/test/test_tokenize.pyu _testFilegs!cCs d}|j|j|ƒƒdS(Nu6tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt(u assertTrueu _testFile(uselfuf((u//mit/python/lib/python3.0/test/test_tokenize.pyu'test_utf8_coding_cookie_and_no_utf8_bomkscCs d}|jt|j|ƒdS(u† As per PEP 0263, if a file starts with a utf-8 BOM signature, the only allowed encoding for the comment is 'utf-8'. The text file used in this test starts with a BOM signature, but specifies latin1 as the coding, so verify that a SyntaxError is raised, which matches the behaviour of the interpreter when it encounters a similar condition. u8tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txtN(ufailUnlessRaisesu SyntaxErroru _testFile(uselfuf((u//mit/python/lib/python3.0/test/test_tokenize.pyu&test_latin1_coding_cookie_and_utf8_bomoscCs d}|j|j|ƒƒdS(Nu9tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txt(u assertTrueu _testFile(uselfuf((u//mit/python/lib/python3.0/test/test_tokenize.pyu"test_no_coding_cookie_and_utf8_bomzscCs d}|j|j|ƒƒdS(Nu6tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt(u assertTrueu _testFile(uselfuf((u//mit/python/lib/python3.0/test/test_tokenize.pyu$test_utf8_coding_cookie_and_utf8_bom~sN(u__name__u __module__u__doc__u _testFileu'test_utf8_coding_cookie_and_no_utf8_bomu&test_latin1_coding_cookie_and_utf8_bomu"test_no_coding_cookie_and_utf8_bomu$test_utf8_coding_cookie_and_utf8_bom(u __locals__((u//mit/python/lib/python3.0/test/test_tokenize.pyuTestTokenizerAdheresToPep0263bs    uTestTokenizerAdheresToPep0263cBs |EeZd„Zd„ZdS(cs~d}|jdƒ‰d ‰‡‡fd†}tt|ddƒƒdd…}ddd d dfg}|j||d ƒdS(Nu "ЉЊЈÐЂ"uutf-8csˆp d‰ˆSdSdS(NsT(uTrue((ulineufirst(u//mit/python/lib/python3.0/test/test_tokenize.pyureadline‰suencodingiiÿÿÿÿiiiubytes not decoded with encodingF(ii(ii(uencodeuFalseulistu _tokenizeu assertEquals(uselfuliteralureadlineutokensuexpected_tokens((ulineufirstu//mit/python/lib/python3.0/test/test_tokenize.pyu.test__tokenize_decodes_with_specified_encoding…s " csod‰d ‰‡‡fd†}tt|ddƒƒdd…}ddd d dfg}|j||d ƒdS( Nu "ЉЊЈÐЂ"csˆp d‰ˆSdSdS(NsT(uTrue((uliteralufirst(u//mit/python/lib/python3.0/test/test_tokenize.pyureadlinešsuencodingiÿÿÿÿiiiiu*string not tokenized when encoding is NoneF(ii(ii(uFalseulistu _tokenizeuNoneu assertEquals(uselfureadlineutokensuexpected_tokens((uliteralufirstu//mit/python/lib/python3.0/test/test_tokenize.pyu1test__tokenize_does_not_decode_with_encoding_none—s " N(u__name__u __module__u.test__tokenize_decodes_with_specified_encodingu1test__tokenize_does_not_decode_with_encoding_none(u __locals__((u//mit/python/lib/python3.0/test/test_tokenize.pyu Test_Tokenizeƒs  u Test_TokenizecBsh|EeZd„Zd„Zd„Zd„Zd„Zd„Zd„Zd„Z d„Z d „Z d S( csd‰‡‡fd†}|S(Nics5ˆtˆƒko t‚nˆˆ}ˆd7‰|S(Ni(ulenu StopIteration(uline(uindexulines(u//mit/python/lib/python3.0/test/test_tokenize.pyureadline­s    ((uselfulinesureadline((uindexulinesu//mit/python/lib/python3.0/test/test_tokenize.pyu get_readline«scCsUd}t|j|ƒƒ\}}|j|dƒ|j|t|dd…ƒƒdS(Ns # something sprint(something) sdo_something(else) uutf-8i(s # something sprint(something) sdo_something(else) (udetect_encodingu get_readlineu assertEqualsulist(uselfulinesuencodinguconsumed_lines((u//mit/python/lib/python3.0/test/test_tokenize.pyutest_no_bom_no_encoding_cookie¶s cCsKd}t|j|ƒƒ\}}|j|dƒ|j|ddgƒdS(Ns# something sprint(something) sdo_something(else) uutf-8s # something (s# something sprint(something) sdo_something(else) (udetect_encodingu get_readlineu assertEquals(uselfulinesuencodinguconsumed_lines((u//mit/python/lib/python3.0/test/test_tokenize.pyutest_bom_no_cookieÀs cCsHd}t|j|ƒƒ\}}|j|dƒ|j|dgƒdS(Ns# -*- coding: latin-1 -*- sprint(something) sdo_something(else) ulatin-1(s# -*- coding: latin-1 -*- sprint(something) sdo_something(else) (udetect_encodingu get_readlineu assertEquals(uselfulinesuencodinguconsumed_lines((u//mit/python/lib/python3.0/test/test_tokenize.pyutest_cookie_first_line_no_bomËs cCsHd}t|j|ƒƒ\}}|j|dƒ|j|dgƒdS(Ns# coding=utf-8 sprint(something) sdo_something(else) uutf-8s# coding=utf-8 (s# coding=utf-8 sprint(something) sdo_something(else) (udetect_encodingu get_readlineu assertEquals(uselfulinesuencodinguconsumed_lines((u//mit/python/lib/python3.0/test/test_tokenize.pyu&test_matched_bom_and_cookie_first_lineÕs cCs,d}|j|ƒ}|jtt|ƒdS(Ns## vim: set fileencoding=ascii : sprint(something) sdo_something(else) (s## vim: set fileencoding=ascii : sprint(something) sdo_something(else) (u get_readlineu assertRaisesu SyntaxErrorudetect_encoding(uselfulinesureadline((u//mit/python/lib/python3.0/test/test_tokenize.pyu<test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorßs cCsQd}t|j|ƒƒ\}}|j|dƒddg}|j||ƒdS(Ns #! something s # vim: set fileencoding=ascii : sprint(something) sdo_something(else) uascii(s #! something s # vim: set fileencoding=ascii : sprint(something) sdo_something(else) (udetect_encodingu get_readlineu assertEquals(uselfulinesuencodinguconsumed_linesuexpected((u//mit/python/lib/python3.0/test/test_tokenize.pyutest_cookie_second_line_no_bomès cCsKd}t|j|ƒƒ\}}|j|dƒ|j|ddgƒdS(Ns#! something sf# coding=utf-8 sprint(something) sdo_something(else) uutf-8s #! something (s#! something sf# coding=utf-8 sprint(something) sdo_something(else) (udetect_encodingu get_readlineu assertEquals(uselfulinesuencodinguconsumed_lines((u//mit/python/lib/python3.0/test/test_tokenize.pyu'test_matched_bom_and_cookie_second_lineôs cCs,d}|j|ƒ}|jtt|ƒdS(Ns#! something s # vim: set fileencoding=ascii : sprint(something) sdo_something(else) (s#! something s # vim: set fileencoding=ascii : sprint(something) sdo_something(else) (u get_readlineu assertRaisesu SyntaxErrorudetect_encoding(uselfulinesureadline((u//mit/python/lib/python3.0/test/test_tokenize.pyu=test_mismatched_bom_and_cookie_second_line_raises_syntaxerrors cCs|jdƒ}t|ƒ\}}|j|dƒ|j|dgƒt|jdƒƒ\}}|j|dƒ|j|gƒ|jdƒ}t|ƒ\}}|j|dƒ|j|dgƒ|jdƒ}t|ƒ\}}|j|dƒ|j|gƒdS( Nsprint(something) uutf-8sprint(something) s(sprint(something) ((sprint(something) (s(u get_readlineudetect_encodingu assertEquals(uselfureadlineuencodinguconsumed_lines((u//mit/python/lib/python3.0/test/test_tokenize.pyutest_short_files sN( u__name__u __module__u get_readlineutest_no_bom_no_encoding_cookieutest_bom_no_cookieutest_cookie_first_line_no_bomu&test_matched_bom_and_cookie_first_lineu<test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorutest_cookie_second_line_no_bomu'test_matched_bom_and_cookie_second_lineu=test_mismatched_bom_and_cookie_second_line_raises_syntaxerrorutest_short_files(u __locals__((u//mit/python/lib/python3.0/test/test_tokenize.pyuTestDetectEncoding©s  uTestDetectEncodingcBs|EeZd„ZdS(csÔddl}tƒ‰d‰‡fd†}‡fd†}d‰‡fd†}|j}|j}||_||_z8t|ƒ}|jt|ƒddddd d gƒWd||_||_X|jˆˆƒdS( NicsˆddgfS(Nufirstusecond((ureadline(uencoding(u//mit/python/lib/python3.0/test/test_tokenize.pyumock_detect_encoding%scs;|‰g}x(|ƒ}|o|j|ƒqn|SdS(N(uappend(ureadlineuencodinguoutu next_line(u encoding_used(u//mit/python/lib/python3.0/test/test_tokenize.pyumock__tokenize(s  cs ˆd7‰ˆdkodSˆS(Niis(((ucounter(u//mit/python/lib/python3.0/test/test_tokenize.pyu mock_readline4s  ufirstusecondiiii(utokenizeuobjectuNoneudetect_encodingu _tokenizeu assertEqualsulistu assertTrue(uselfutokenize_moduleumock_detect_encodingumock__tokenizeu mock_readlineuorig_detect_encodinguorig__tokenizeuresults((ucounteruencodingu encoding_usedu//mit/python/lib/python3.0/test/test_tokenize.pyu test_tokenize!s"        ,  N(u__name__u __module__u test_tokenize(u __locals__((u//mit/python/lib/python3.0/test/test_tokenize.pyu TestTokenizes u TestTokenizeudoctestsudecistmtcCsXddlm}tj|dƒtjtƒtjtƒtjtƒtjt ƒdS(Ni(u test_tokenizeT( utestu test_tokenizeusupportu run_doctestuTrueu run_unittestuTestTokenizerAdheresToPep0263u Test_TokenizeuTestDetectEncodingu TestTokenize(u test_tokenize((u//mit/python/lib/python3.0/test/test_tokenize.pyu test_mainKs    u__main__(udoctestsutestusupportutokenizeu _tokenizeu untokenizeuNUMBERuNAMEuOPuSTRINGu ENDMARKERutok_nameudetect_encodinguiouBytesIOuunittestuTestCaseuosusysuglobu dump_tokensu roundtripudecistmtuTestTokenizerAdheresToPep0263u Test_TokenizeuTestDetectEncodingu TestTokenizeu__test__u test_mainu__name__(((u//mit/python/lib/python3.0/test/test_tokenize.pyus F$  %!&v*