ó ¡_yec@sådZddlmZmZddlZddlZddlZddlmZddl m Z ddl Z ej j dkr‘ddlmZn ddlZyeWnek rÀZeZnXe jdd ƒZd „Zd „Zd „Zddd „Zd„Zd„ZedƒZd„Zd„Zd„Z d„Z!d„Z"d„Z#d„Z$d„Z%d„Z&d„Z'ej(dej)ƒZ*ej(dƒZ+d„Z,d„Z-d„Z.d „Z/d!„Z0d"„Z1d#„Z2d$„Z3dS(%u# Standard chiron fetcher functions iÿÿÿÿ(tprint_functiontunicode_literalsN(tchoice(tetreei(tcsvtencodinguUTF-8cCs4tj|dtƒ}tj|jtƒ}||fS(u!Fetch a URL and return parsed XMLtstream(trequeststgettTrueRt fromstringtcontenttparser(turltresponsetxml((sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pytfetch_and_parse_xml#scs‡fd†}|S(u  Return a fetcher for a bugzilla instance >>> url, n = fetch_bugzilla("https://bugzilla.redhat.com")("123456") >>> url u'https://bugzilla.redhat.com/show_bug.cgi?id=123456' >>> n 'System with Syntax - S635MP motherboard will not install' csOdˆ|f}t|ƒ\}}|jdƒ}|rA||fS|dfSdS(uInner Bugzilla fetcheru%s/show_bug.cgi?id=%su0string(//span[@id="short_desc_nonedit_display"])N(RtxpathtNone(tticketR Rtdummy_responsettitle(tbase_url(sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pytbugzilla_fetcher9s  ((RR((RsZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pytfetch_bugzilla/s  cs‡fd†}|S(u¶ Return a fetcher for a Trac instance >>> url, n = fetch_trac("https://debathena.mit.edu/trac")("123") >>> url u'https://debathena.mit.edu/trac/ticket/123' >>> n u'debathena-ssl-certificates should include a CRL' >>> url, n = fetch_trac("https://scripts.mit.edu/trac")("123") >>> url u'https://scripts.mit.edu/trac/ticket/123' >>> n u'scripts-remove works poorly with non-Athena accounts' cs}dˆ|f}tj|dƒ}|jdkrotj|jjdƒƒ}t|ƒ}||jddƒfS|dfSdS(uInner Trac fetcheru %s/ticket/%su ?format=csviÈu usummaryN( RRt status_codeRt DictReaderttexttsplittnextR(RR Rtreadertrow(R(sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pyt trac_fetcherSs ((RR ((RsZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pyt fetch_tracDs csIˆdkrˆ‰nˆdkr0tjƒ‰n‡‡‡fd†}|S(u¸ Return a fetcher for a Jira instance >>> fetch_jira("https://issues.apache.org")("ZOOKEEPER-1234") (u'https://issues.apache.org/jira/browse/ZOOKEEPER-1234', u'basic cleanup in LearnerHandler') Some Jira instances may use different hostnames for interactive usage than for API usage, for example due to using client certs for API authentication and some other mechanism for normal authentication. When the URLs differ, set the api_url option as well as the initial "url" parameter. You can also supply a custom `requests.Session` object with custom cookies, client certs, or other parameters needed to authenticate. For example: >>> def make_fetch_jira(): ... r = requests.Session() ... r.verify = "/path/to/ca.pem" ... r.cert = ("/path/to/public.crt", "/path/to/private.key") ... return fetch_jira("https://jira.example.com", api_url="https://api.example.com", req=r) csjdˆ|f}dˆ|f}ˆj|ƒ}y||jƒddfSWntk re|dfSXdS(uInner Jira fetcheru*%s/jira/rest/api/2/issue/%s?fields=summaryu%s/jira/browse/%sufieldsusummaryN(RtjsontKeyErrorR(Rtfull_apithtml_urltresp(tapi_urltreqR (sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pytfetch{s N(RRtSession(R R'R(R)((R'R(R sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pyt fetch_jira`s     cs‡‡fd†}|S(u· Return a fetcher for a Github instance >>> fetch_github("sipb", "chiron")("2") (u'https://github.com/sipb/chiron/issues/2', u'Teach debothena about its bugtracker') cscdˆˆ|f}tj|ƒ}y"|jƒd|jƒdfSWntk r^|dfSXdS(uInner GitHub fetcheru,https://api.github.com/repos/%s/%s/issues/%suhtml_urlutitleN(RRR"R#R(RR R(trepotuser(sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pyR)s " ((R-R,R)((R,R-sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pyt fetch_githubˆscCsVd|f}t|ƒ\}}|jdƒ}|sF|jdƒ}n||pRdfS(u RFC fetcher >>> fetch_rfc("1234") (u'https://datatracker.ietf.org/doc/html/rfc1234', 'Tunneling IPX Traffic through IP Networks') >>> fetch_rfc("9300") (u'https://datatracker.ietf.org/doc/html/rfc9300', 'The Locator/ID Separation Protocol (LISP)') u+https://datatracker.ietf.org/doc/html/rfc%sustring(//span[@class="h1"])ustring(//h1[@id="title"])N(RRR(tnumberR RRR((sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pyt fetch_rfc›s uhttps://bugzilla.redhat.comcCs‘t|ƒ\}}td||fƒ|r:|d|fSd|}t|ƒ\}}|jdƒ}|rƒ|d|jƒdfS|dfSdS(u´ CVE fetcher >>> url, n = fetch_cve("CVE-2015-1234") RHBZ url='https://bugzilla.redhat.com/show_bug.cgi?id=CVE-2015-1234' title='CVE-2015-1234 chromium-browser: buffer overflow via race condition in GPU' >>> url u'https://bugzilla.redhat.com/show_bug.cgi?id=CVE-2015-1234' >>> n u'[RHBZ] CVE-2015-1234 chromium-browser: buffer overflow via race condition in GPU' >>> url, n = fetch_cve("CVE-1999-0012") RHBZ url='https://bugzilla.redhat.com/show_bug.cgi?id=CVE-1999-0012' title='None' >>> url u'http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-1999-0012' >>> n u'\nSome web servers under Microsoft Windows allow remote attackers to bypass access restrictions for files with long file names.\n' >>> fetch_cve("CVE-1999-9000") RHBZ url='https://bugzilla.redhat.com/show_bug.cgi?id=CVE-1999-9000' title='None' (u'http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-1999-9000', None) uRHBZ url='%s' title='%s'u[RHBZ] u0http://cve.mitre.org/cgi-bin/cvename.cgi?name=%su/string(//tr[th="Description"]/following::tr[1])u N(tfetch_cve_rhbztprintRRtstripR(RR RRR((sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pyt fetch_cve­s cCsId|}t|ƒ\}}|jdƒ}|r;||fS|dfSdS(u¢ scripts.mit.edu FAQ fetcher >>> fetch_scripts_faq("136") (u'http://scripts.mit.edu/faq/136', u'Is scripts.mit.edu appropriate for my\xa0site?') uhttp://scripts.mit.edu/faq/%su!string(//h3[@class="storytitle"])N(RRR(RR RRR((sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pytfetch_scripts_faqÐs   cCsZd|}tj|ƒ}y"|jƒd|jƒdfSWntk rU|dfSXdS(u‰ Launchpad fetcher >>> fetch_launchpad("123456") (u'https://bugs.launchpad.net/bugs/123456', u'podcast crashes amarok') u$http://api.launchpad.net/1.0/bugs/%suweb_linkutitleN(RRR"R#R(RR R((sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pytfetch_launchpadßs  " cs‡fd†}|S(u÷ Debbugs (Debian bugtracker) fetcher >>> url, n = fetch_debbugs("https://bugs.debian.org")("123456") >>> url u'https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=123456' >>> n 'ITP: eazel-engine -- Crux theme for GTK+' csOdˆ|f}t|ƒ\}}|jdƒ}|rA||fS|dfSdS(uInner Debbugs fetcheru%s/cgi-bin/bugreport.cgi?bug=%su&normalize-space(//h1/child::text()[2])N(RRR(RR RRR(R(sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pytdebbugs_fetcher÷s  ((RR7((RsZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pyt fetch_debbugsís  cCs}d|f}t|ƒ\}}d}|j|ƒ}|jdƒpFd}td|fƒ|rm|d}n|}||fS(u´ Debian Security Advisories fetcher >>> fetch_dsa("DSA-1234") -> DSA URLs in page: [] (u'https://security-tracker.debian.org/tracker/DSA-1234', 'ruby1.6') u.https://security-tracker.debian.org/tracker/%suL//a[text()="Debian"]/@href[starts-with(.,"http://www.debian.org/security/")]u&string(//tr[td/b="Description"]/td[2])u -> DSA URLs in page: %siN(RRRR2(R/R RRt dsa_xpathtdsa_urlsRtdsa_url((sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pyt fetch_dsas  cCsÙd}tj|dƒ}x³|jjdƒD]Ÿ}|dd!dkr,|jdƒdd !\}}ySt|ƒt|ƒkr³|d |d j|jd ƒdjdƒd ƒffSWqËtk rÇqËXq,q,W|dfS(u^ Pokemon fetcher (by Pokedex number) [broken] >>> url, n = fetch_pokemon("123") >>> url u'https://bulbapedia.bulbagarden.net/wiki/List_of_Pok%C3%A9mon_by_National_Pok%C3%A9dex_number' >>> n Previously returned u'Scyther (Bug, Flying)' but unfortunately Bulbapedia now seems to have bot-blocking from Cloudflare. u\https://bulbapedia.bulbagarden.net/wiki/List_of_Pok%C3%A9mon_by_National_Pok%C3%A9dex_numberu ?action=rawu iiu{{rdex|u|iiu%s (%s)u, u}iN(RRRRtinttjoint ValueErrorR(RR Rtlinetnumtname((sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pyt fetch_pokemons ;  cCsRd|f}t|ƒ\}}|jdƒ}|rD||jƒfS|dfSdS(uÆ MIT class fetcher >>> url, n = fetch_mit_class("6.1810") >>> url u'http://student.mit.edu/catalog/search.cgi?search=6.1810' >>> n '6.1810 Operating System Engineering' u3http://student.mit.edu/catalog/search.cgi?search=%su string(//h3)N(RRR3R(RR RRR((sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pytfetch_mit_class0s cCsSd|f}t|ƒ\}}|jdƒ}|rC|jƒ}n||pOdfS(uª whats fetcher (MIT SIPB acronym database) >>> fetch_whats("SIPB") (u'https://stuff.mit.edu/cgi/whats.cgi?SIPB', 'Student Information Processing Board') u&https://stuff.mit.edu/cgi/whats.cgi?%sustring(//dl/dd)N(RRR3R(twhatsR RRR((sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pyt fetch_whatsBs  c Cs†d}t|ƒ\}}t|jdƒƒjjƒ}tddddddd d gƒ}ttj|ƒƒ}|d |||ffS( u@Generate a random fake Debathena package name and file to divertu9http://debathena.mit.edu/trac/wiki/PackageNamesWeDidntUseuid("content")//liu/etcu/binu/usr/binu/sbinu /usr/sbinu /dev/mapperu /etc/defaultu/var/runu%s should divert %s/%s(RRRRR3tostlistdir(RR RRtpackaget directorytfilename((sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pytundebathena_funPscCsVd}d d|fd f}tj|d|ƒ}d}d |j|f}|j|fS( uÁ Bible fetcher using esvapi.org v2 #>>> fetch_bible_esvapi2("John 4:8") #(u'http://www.esvapi.org/v2/rest/passageQuery?key=TEST&passage=John+4%3A8&output-format=plain-text', u'\n=======================================================\nJohn 4:8\n [8](For his disciples had gone away into the city to buy food.) (ESV)\n(From The Holy Bible, English Standard Version. See http://www.crosswaybibles.org and http://www.esvapi.org/.)') u*http://www.esvapi.org/v2/rest/passageQueryukeyuTESTupassageu output-formatu plain-texttparamsun(From The Holy Bible, English Standard Version. See http://www.crosswaybibles.org and http://www.esvapi.org/.)u %s %s(ukeyuTEST(u output-formatu plain-text(RRRR (tverseR RMRt copyright_msgR((sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pytfetch_bible_esvapi2Zs u.(?P(\d+ )?[0-9a-z]+) (?P[0-9:-]+)u [ ]* [ ]*cs‡fd†}|S(uz Bible fetcher, using ibibles.net >>> fetch_ibibles("niv")("mat 5:3-4") #doctest:+NORMALIZE_WHITESPACE,+ELLIPSIS (u'https://ibibles.net/quote.php?niv-mat/5:3-4', u'"Blessed are the poor in spirit, ..., for they will be comforted.') >>> fetch_ibibles("kjv")("Matthew 5:3") #doctest:+NORMALIZE_WHITESPACE (u'https://ibibles.net/quote.php?kjv-Matthew/5:3', u'Blessed are the poor in spirit: for theirs is the kingdom of heaven.') >>> fetch_ibibles("niv")("1 Timothy 3:9") #doctest:+NORMALIZE_WHITESPACE (u'https://ibibles.net/quote.php?niv-1 Timothy/3:9', u'They must keep hold of the deep truths of the faith with a clear conscience.') >>> fetch_ibibles("niv")("1ti 3:9") #doctest:+NORMALIZE_WHITESPACE (u'https://ibibles.net/quote.php?niv-1ti/3:9', u'They must keep hold of the deep truths of the faith with a clear conscience.') cs“tj|ƒ}|sdSˆ|jdƒ|jdƒf}d|}t|ƒ\}}|jdƒ}tjtddj |ƒj ƒƒ}||fS( uInner ibibles fetcherubookuverseu&https://ibibles.net/quote.php?%s-%s/%su //body/text()u uN(NN( tBIBLE_REtmatchRtgroupRRtretsubt STRIP_NL_RER>R3(t full_verset parsed_verseRMR RRtbodyR(t translation(sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pytibibles_fetchers! $((RZR[((RZsZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pyt fetch_ibiblesms cCs[d|f}t|ƒ\}}|jdƒ}|rM|jdkrM||fS|dfSdS(uj XKCD fetcher >>> fetch_xkcd("123") (u'http://xkcd.com/123/', 'xkcd: Centrifugal Force') uhttp://xkcd.com/%s/ustring(//title)iÈN(RRRR(tcomicR RRR((sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pyt fetch_xkcdŽs   cCs‰d|f}t|ƒ\}}|jdƒ}td||jfƒ|r{|jdkr{||dtt|dƒƒfS|dfSdS(uÏ Unicode fetcher (number->char) This would work with a browser, but we seem to be blocked by Cloudflare now, so Chiron's Unicode support seems to be broken. So, uh, test that it hasn't resumed working. >>> url, n = fetch_unicode("2603") Unicode: 'Unicode Character 'SNOWMAN' (U+2603)' '200' >>> url u'https://www.fileformat.info/info/unicode/char/2603/index.htm' >>> n u"Unicode Character 'SNOWMAN' (U+2603): \u2603" u:https://www.fileformat.info/info/unicode/char/%s/index.htmustring(//title)uUnicode: '%s' '%s'iÈu: iN(RRR2RtunichrR=R(t codepointR RRR((sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pyt fetch_unicodes !cCswtt|ƒdƒ}d|f}t|ƒ\}}|jdƒ}|rb|jdkrb||fS|d|ffSdS(uç Unicode fetcher (char->number) >>> url, n = fetch_unicode_char("ሴ") >>> url u'https://www.fileformat.info/info/unicode/char/1234/index.htm' >>> n "Unicode Character 'ETHIOPIC SYLLABLE SEE' (U+1234)" uxu:https://www.fileformat.info/info/unicode/char/%s/index.htmustring(//title)iÈuU+%sN(tformattordRRR(t characterR`R RRR((sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pytfetch_unicode_charµs   cCs‰d|f}t|ƒ\}}|jdƒ}|jdƒ}|r{|jdkr{|rkd||f}n|}||fS|dfSdS(uñ Airport fetcher (code->location) >>> url, n = fetch_airport("BOS") >>> url u'http://www.gcmap.com/airport/BOS' >>> n u'Boston, Massachusetts, United States (General Edward Lawrence Logan International Airport)' uhttp://www.gcmap.com/airport/%su.string(//meta[@name="geo.placename"]/@content)ustring(//td[@class="fn org"])iÈu%s (%s)N(RRRR(tcodeR RRtplaceRBR((sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pyt fetch_airportÈs  cCsdS(u1Fetcher to reduce misdirected Guild office combosuNO COMBOS OVER ZEPHYRu DO @b(NOT) ASK FOR OR SEND THE OFFICE COMBO OVER ZEPHYR, EVEN PERSONAL ZEPHYR. Instead, look in /mit/assassin/Office. If you don't have access, ask to be added.(uNO COMBOS OVER ZEPHYRu DO @b(NOT) ASK FOR OR SEND THE OFFICE COMBO OVER ZEPHYR, EVEN PERSONAL ZEPHYR. Instead, look in /mit/assassin/Office. If you don't have access, ask to be added.((t_ticket((sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pytdeal_with_assassinâscCsdS(u"Science" ASCII art fetcheruSCIENCE!uÔ ____ ____ ___ _____ _ _ ____ _____ / ___| / ___|_ _| ____| \ | |/ ___| ____| \___ \| | | || _| | \| | | | _| ___) | |___ | || |___| |\ | |___| |___ |____/ \____|___|_____|_| \_|\____|_____| (uSCIENCE!uÔ ____ ____ ___ _____ _ _ ____ _____ / ___| / ___|_ _| ____| \ | |/ ___| ____| \___ \| | | || _| | \| | | | _| ___) | |___ | || |___| |\ | |___| |___ |____/ \____|___|_____|_| \_|\____|_____| ((Ri((sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pytinvoke_scienceêscCs |dfS(u*"Debothena" ASCII art fetcher, for testinguÝ â•ºâ”³â”“â”â”╸â”┓ â”â”┓╺┳╸╻ â•»â”â”╸â”┓╻â”â”┓ ┃┃┣╸ ┣┻┓┃ ┃ ┃ ┣â”┫┣╸ ┃┗┫┣â”┫ ╺┻┛┗â”╸┗â”┛┗â”â”› ╹ ╹ ╹┗â”╸╹ ╹╹ ╹ ((R((sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pytinvoke_debothenaõs(4t__doc__t __future__RRRGRTtsystrandomRtlxmlRRt version_infotmajort backportsRR_t NameErrortexctchrt HTMLParserR RRR!RR+R.R0R1R4R5R6R8R<RCRDRFRLRPtcompilet IGNORECASERQRVR\R^RaReRhRjRkRl(((sZ/afs/sipb.mit.edu/project/chiron/chiron-scripts-venv/src/chiron-bot/chiron_bot/fetchers.pytsT        (    #         !