‡Æ
ÿMö9c       s>   d  Z  d f  d „  ƒ  YZ d „  Z e d j o e ƒ  n d S(   sµ   

Robots.txt file parser class.  Accepts a list of lines or robots.txt URL as
input, builds a set of rules from that list, then answers questions about
fetchability of other URLs.

s   RobotFileParserc      sA   d „  Z  d „  Z d „  Z d „  Z d „  Z d „  Z d „  Z RS(   Nc    s(   h  |  _ d |  _ d |  _ d |  _ d  S(   Ni    s    (   s   selfs   ruless   debugs   urls   last_checked(   s   selfs(   /mit/python/lib/python2.0/robotparser.pys   __init__ s    			c    s   |  i Sd  S(   N(   s   selfs   last_checked(   s   selfs(   /mit/python/lib/python2.0/robotparser.pys   mtime s    c    s   d  k  } | i  ƒ  |  _ d  S(   N(   s   times   selfs   last_checked(   s   selfs   times(   /mit/python/lib/python2.0/robotparser.pys   modified s    	c    s   | |  _  d  S(   N(   s   urls   self(   s   selfs   urls(   /mit/python/lib/python2.0/robotparser.pys   set_url s    c    s,   d  k  } |  i | i |  i ƒ i ƒ  ƒ d  S(   N(   s   urllibs   selfs   parses   urlopens   urls	   readlines(   s   selfs   urllibs(   /mit/python/lib/python2.0/robotparser.pys   read s    	c    s  d k  } d k } g  } xÝ| d rÓ} |  i o d G| Gn | d  o g  } q! n | i | | i | d ƒ  ƒ } | o q! n | i	 d | ƒ } t
 | ƒ d j oJ| i | d ƒ | d <| d d j o] |  i o d	 G| d GHn | i | d ƒ |  i i | d ƒ o g  |  i | d <n nÂ | d d
 j o™ | d oS |  i o d G| d GHn x1 | d r' } |  i | i | i | d ƒ ƒ qoWn8 x4 | d r* } |  i o d G| GHn g  |  i | <q§Wn |  i o d G| GHn n q! W|  i ƒ  d S(   s+   parse the input lines from a robot.txt fileNi    s   >i   s   #s    *: *i   s
   user-agents   >> user-agent:s   disallows   >> disallow:s   >> allows   >> unknown:(   s   strings   res   actives   liness   lines   selfs   debugs   strips   finds   splits   lens   lowers   appends   ruless   has_keys   agents   compiles   modified(   s   selfs   liness   strings   res   actives   lines   agents(   /mit/python/lib/python2.0/robotparser.pys   parse sL     	 
 
 
 	 ) 	 
 
 c    sù   d k  } | } |  i i | ƒ o
 d } n |  i i | ƒ o' |  i o d G| Gd G| GHn d Sn | i  | ƒ d } xT |  i | d rC } | i
 | ƒ t j	 o' |  i o d G| Gd G| GHn d Sn q‹ W|  i o d G| Gd G| GHn d Sd S(	   s=   using the parsed robots.txt decide if useragent can fetch urlNs   *s   >> allowings   fetch byi   i   i    s   >> disallowing(   s   urlparses	   useragents   ags   selfs   ruless   has_keys   debugs   urls   paths   rules   matchs   None(   s   selfs	   useragents   urls   urlparses   ags   paths   rules(   /mit/python/lib/python2.0/robotparser.pys	   can_fetchF s&     	 

  
 
 (   s   __init__s   mtimes   modifieds   set_urls   reads   parses	   can_fetch(    s(   /mit/python/lib/python2.0/robotparser.pys   RobotFileParser	 s    						'c     sW   t  ƒ  }  d |  _ |  i d ƒ |  i ƒ  |  i GH|  i d d ƒ GH|  i d d ƒ GHd  S(   Ni   s"   http://www.musi-cal.com/robots.txts   *s   http://www.musi-cal.com.com/s   Musi-Cal-Robots?   http://www.musi-cal.com/cgi-bin/event-search?city=San+Francisco(   s   RobotFileParsers   rps   debugs   set_urls   reads   ruless	   can_fetch(   s   rps(   /mit/python/lib/python2.0/robotparser.pys   _testV s    		
	s   __main__N(   s   __doc__s   RobotFileParsers   _tests   __name__(    s(   /mit/python/lib/python2.0/robotparser.pys   ? s   M	
