; Ìâ"Ic @süddlZddlZddlZddlmZGd„dejƒZejƒZ dd„Z dZ dd gZ d d d gZ e d e e e ƒdZ dd d:gZ d gZ e de e e ƒdZ gZ d ddgZ e de e e ƒdZ gZ dddddddgZ e de e e dƒe de e e dƒd Z dgZ ddddd!d"gZ e d#e e e ƒd$Z d gZ gZ e d%e e e ƒd&Z d'gZ d(gZ e d)e e e d*d+ƒd,Z gZ d-gZ e d.e e e d*d+ƒgZ d-gZ e d/e e e d*d0ƒd1Z gZ d-gZ e d2e e e d*d+ƒd-gZ gZ e d3e e e d*d0ƒd&Z d'gZ d(gZ e d4e e e d*d5ƒGd6„d7ejƒZd8„Zed9kod e_eƒndS(;iN(usupportcBs)|EeZd„Zd„Zd„ZdS(cCsitjj|ƒ|od||f|_nd||f|_||_||_||_||_dS(NuRobotTest(%d, good, %s)uRobotTest(%d, bad, %s)(uunittestuTestCaseu__init__ustruparseruurlugooduagent(uselfuindexuparseruurlugooduagent((u2/mit/python/lib/python3.0/test/test_robotparser.pyu__init__s   cCs‚t|jtƒo|j\}}n|j}|j}|jo |j|jj||ƒƒn|j|jj||ƒƒdS(N( u isinstanceuurlutupleuagentugoodu failUnlessuparseru can_fetchufailIf(uselfuagentuurl((u2/mit/python/lib/python3.0/test/test_robotparser.pyurunTests    cCs|jS(N(ustr(uself((u2/mit/python/lib/python3.0/test/test_robotparser.pyu__str__sN(u__name__u __module__u__init__urunTestu__str__(u __locals__((u2/mit/python/lib/python3.0/test/test_robotparser.pyu RobotTestCases  u RobotTestCaseutest_robotparserc Cs•tj|ƒjƒ}tjjƒ}|j|ƒx-|D]%}tjt |||d|ƒƒq8Wx-|D]%}tjt |||d|ƒƒqhWdS(Nii( uiouStringIOu readlinesuurllibu robotparseruRobotFileParseruparseutestsuaddTestu RobotTestCase(uindexu robots_txtu good_urlsubad_urlsuagentulinesuparseruurl((u2/mit/python/lib/python3.0/test/test_robotparser.pyu RobotTest"s #u’ User-agent: * Disallow: /cyberworld/map/ # This is an infinite virtual URL space Disallow: /tmp/ # these will soon disappear Disallow: /foo.html u/u /test.htmlu/cyberworld/map/index.htmlu/tmp/xxxu /foo.htmliuÁ # robots.txt for http://www.example.com/ User-agent: * Disallow: /cyberworld/map/ # This is an infinite virtual URL space # Cybermapper knows where to go. User-agent: cybermapper Disallow: u cybermapperiu% # go away User-agent: * Disallow: / u/tmp/ium User-agent: figtree Disallow: /tmp Disallow: /a%3cd.html Disallow: /a%2fb.html Disallow: /%7ejoe/index.html u/tmpu /tmp.htmlu /tmp/a.htmlu /a%3cd.htmlu /a%3Cd.htmlu /a%2fb.htmlu/~joe/index.htmliufigtreeiuFigTree Robot libwww-perl/5.04uf User-agent: * Disallow: /tmp/ Disallow: /a%3Cd.html Disallow: /a/b.html Disallow: /%7ejoe/index.html u /a/b.htmlu/%7Ejoe/index.htmliu User-Agent: * Disallow: /. iuG User-agent: Googlebot Allow: /folder1/myfile.html Disallow: /folder1/ u/folder1/myfile.htmlu/folder1/anotherfile.htmliuagentu GooglebotuJ User-agent: Googlebot Disallow: / User-agent: Googlebot-Mobile Allow: / u/something.jpgi i uGooglebot-MobileuJ User-agent: Googlebot-Mobile Allow: / User-agent: Googlebot Disallow: / i i i u googlebotcBs |EeZd„Zd„ZdS(cCsetjdƒpdSd}tjjƒ}|j|ƒ|jƒ|j|jd|dƒdƒdS(Nunetworkuhttp://mueblesmoraleda.comu*u /robots.txtF( usupportuis_resource_enableduurllibu robotparseruRobotFileParseruset_urlureadu assertEqualu can_fetchuFalse(uselfuurluparser((u2/mit/python/lib/python3.0/test/test_robotparser.pyutestPasswordProtectedSiteÒs  cCsNtjdƒpdStjjdƒ}|jƒ|j|jddƒƒdS(Nunetworku http://www.python.org/robots.txtu*(usupportuis_resource_enableduurllibu robotparseruRobotFileParserureadu assertTrueu can_fetch(uselfuparser((u2/mit/python/lib/python3.0/test/test_robotparser.pyu testPythonOrgÜs   N(u__name__u __module__utestPasswordProtectedSiteu testPythonOrg(u __locals__((u2/mit/python/lib/python3.0/test/test_robotparser.pyuNetworkTestCaseÐs  uNetworkTestCasecCstjtƒtjtƒdS(N(usupportu run_unittestuNetworkTestCaseutests(((u2/mit/python/lib/python3.0/test/test_robotparser.pyu test_mainås u__main__(u cybermapperu/cyberworld/map/index.html(uiouunittestuurllib.robotparseruurllibutestusupportuTestCaseu RobotTestCaseu TestSuiteutestsu RobotTestudocugoodubaduNetworkTestCaseu test_mainu__name__uverbose(((u2/mit/python/lib/python3.0/test/test_robotparser.pyus|