‡Æ
ÿMö9c       sP     d  Z   	 d f  d „  ƒ  YZ V d „  Z ` e d j o a e ƒ  n d S(   sµ   

Robots.txt file parser class.  Accepts a list of lines or robots.txt URL as
input, builds a set of rules from that list, then answers questions about
fetchability of other URLs.

s   RobotFileParserc      sY   	  d „  Z   d „  Z  d „  Z  d „  Z  d „  Z  d „  Z F d „  Z RS(   Nc    s7     h  |  _  d |  _  d |  _  d |  _ d  S(   Ni    s    (   s   selfs   ruless   debugs   urls   last_checked(   s   selfs(   /mit/python/lib/python2.0/robotparser.pys   __init__ s   c    s     |  i Sd  S(   N(   s   selfs   last_checked(   s   selfs(   /mit/python/lib/python2.0/robotparser.pys   mtime s   c    s%     d  k  }  | i  ƒ  |  _ d  S(   N(   s   times   selfs   last_checked(   s   selfs   times(   /mit/python/lib/python2.0/robotparser.pys   modified s   c    s     | |  _  d  S(   N(   s   urls   self(   s   selfs   urls(   /mit/python/lib/python2.0/robotparser.pys   set_url s   c    s5     d  k  }  |  i | i |  i ƒ i ƒ  ƒ d  S(   N(   s   urllibs   selfs   parses   urlopens   urls	   readlines(   s   selfs   urllibs(   /mit/python/lib/python2.0/robotparser.pys   read s   c    s{     ! d k  } d k } " g  } # x@| d # r3} $ |  i o $ d G| Gn & | d  o ' g  } ( q0 n * | i | | i | d ƒ  ƒ } + | o
 , q0 n - | i	 d | ƒ } . t
 | ƒ d j oŒ/ | i | d ƒ | d <0 | d d j ol 2 |  i o 2 d	 G| d GHn 3 | i | d ƒ 4 |  i i | d ƒ o 5 g  |  i | d <n nï 6 | d d
 j o½ 7 | d ob 8 |  i o 8 d G| d GHn 9 x7 | d 9 r* } : |  i | i | i | d ƒ ƒ qÃWnJ < = x@ | d = r3 } > |  i o > d G| GHn ? g  |  i | <qWn A |  i o A d G| GHn n q0 WC |  i ƒ  d S(   s+   parse the input lines from a robot.txt fileNi    s   >i   s   #s    *: *i   s
   user-agents   >> user-agent:s   disallows   >> disallow:s   >> allows   >> unknown:(   s   strings   res   actives   liness   lines   selfs   debugs   strips   finds   splits   lens   lowers   appends   ruless   has_keys   agents   compiles   modified(   s   selfs   liness   strings   res   actives   lines   agents(   /mit/python/lib/python2.0/robotparser.pys   parse sL   	 	 	
"
   	, 	  c    s5  F G H d k  } I | } J |  i i | ƒ o J d } n K |  i i | ƒ o0 L |  i o L d G| Gd G| GHn M d Sn N | i  | ƒ d } O xc |  i | d O rO } P | i
 | ƒ t j	 o0 Q |  i o Q d G| Gd G| GHn R d Sn q¯ WS |  i o S d G| Gd G| GHn T d Sd S(	   s=   using the parsed robots.txt decide if useragent can fetch urlNs   *s   >> allowings   fetch byi   i   i    s   >> disallowing(   s   urlparses	   useragents   ags   selfs   ruless   has_keys   debugs   urls   paths   rules   matchs   None(   s   selfs	   useragents   urls   urlparses   ags   paths   rules(   /mit/python/lib/python2.0/robotparser.pys	   can_fetchF s&   	   	  (   s   __init__s   mtimes   modifieds   set_urls   reads   parses	   can_fetch(    s(   /mit/python/lib/python2.0/robotparser.pys   RobotFileParser	 s   'c     sr   V W t  ƒ  }  X d |  _ Y |  i d ƒ Z |  i ƒ  [ |  i GH\ |  i d d ƒ GH] |  i d ^ d ƒ GHd  S(   Ni   s"   http://www.musi-cal.com/robots.txts   *s   http://www.musi-cal.com.com/s   Musi-Cal-Robots?   http://www.musi-cal.com/cgi-bin/event-search?city=San+Francisco(   s   RobotFileParsers   rps   debugs   set_urls   reads   ruless	   can_fetch(   s   rps(   /mit/python/lib/python2.0/robotparser.pys   _testV s   s   __main__N(   s   __doc__s   RobotFileParsers   _tests   __name__(    s(   /mit/python/lib/python2.0/robotparser.pys   ? s   M
