#!/usr/bin/python __version__='$Rev: 616 $' __release__='2' import sys, os, time, re, itertools, smtplib import mitsfs if 'dex' in locals(): del dex dex = None program = 'dexhamster' review = False def main(args): global dex try: import psyco psyco.full() except ImportError: pass dsn = None if len(args) > 2: print 'usage: dexhamster [dsn]' return if len(args) == 2: dsn = args[1] mitsfs.banner(program, __release__, __version__) sys.stdout.write('Connecting to dex...') sys.stdout.flush() dex = mitsfs.dexdb(client='dexhamster', dsn=dsn or mitsfs.database_dsn) print 'done. (%s)' % dex.filename if dex.recoverable(): print dex.filename,'has recoverable data.' if mitsfs.read('Recover? ').strip().lower()[0] == 'y': print 'recovering...' dex.recover() else: dex.norecover() print 'not recovering.' starmenu = [('B', 'Back to Other Menu', None), ('?', 'Check for dissociated roles', checkdis), ('K', 'Key a member', key), ('D', 'De-key a member', dekey), ('W', 'Who are keyholders', keylist), ('C', 'List committees', cttelist), ('A', 'Add a keyholder to a committee', ctteadd), ('R', 'Remove a keyholder from a committee', ctteremove), ('Q', 'Quit', quithamster),] othermenu = [('B', 'Back to Main Menu', None), ('A', 'change codes in a shelfdex Arc', arc), ('X', 'boXing shelfdex arcs', boxing), ('H', 'generate a Hassledex', hassledex), ('C', 'check the dex consistency', dexck), ('O', 'edit bOok', book), ('T', 'edit Title', title), ('S', 'edit Series', series), ('W', 'Withdraw books', withdraw), ('M', 'Membership', membership), ('*', '*-Chamber menu', lambda line: mitsfs.menu(starmenu, line)), ('Q', 'Quit', quithamster)] menu = [('N', 'New Entry', newentry), ('C', 'Code Change', editcodes), ('P', 'Put entry into series', lambda line: editfield('series', line)), ('T', 'Title Change', lambda line: editfield('titles', line)), ('A', 'Author Change', lambda line: editfield('authors', line)), ('R', 'Review mode toggle', reviewtoggle), ('G', 'Grep for pattern', grep), ('O', 'Other (The Dread Menu Miscellaneous)', lambda line: mitsfs.menu(othermenu, line)), ('F', 'File a bug', filebug), ('Q', 'Quit', quithamster)] mitsfs.menu(menu) dex.unlock() def save(line): #dex.save() print print ' *BING*' print def reviewtoggle(line): global review review = not review print 'Review mode is',review and 'ON' or 'OFF' def editfield(field, line): pmap={ 'authors': ('Author: ', [munge_author, munge_field], [validate_field]), 'titles': ('Title: ', [munge_field], [validate_field, validate_title, validate_title_or_series]), 'series': ('Series: ', [munge_field, munge_series], [validate_field, validate_series, validate_title_or_series]), } assert field in pmap book = mitsfs.specify(dex) if not book: return print 'selected',book prompt, munge, validate = pmap[field] value = readvalidate(prompt, dex.indices[field].iterkeys, preload = str(getattr(book, field)), munge = munge, validate = validate, history = field) if not value: return if value == '-': value = '' new = mitsfs.dexline(book, **{field: value}) print 'now', new dex.replace(book, new) def convertcodes(codes): try: codes = mitsfs.shelfcodes(codes) except mitsfs.InvalidShelfcode, e: print e return None return codes def editcodes(line): if review: print 'WARNING REVIEW MODE IS ON' codes = None while True: book = mitsfs.specify(dex) if not book: return print 'selected',book if line: codes = convertcodes(line) if not codes: line = '' if not codes: codes = readvalidate('New codes: ', lambda: (i + ':' for i in itertools.chain(mitsfs.codes.iterkeys(), (i for i in book.codes if i not in mitsfs.codes))), validate = [validate_shelfcodes], history = 'codes') codes = convertcodes(codes) if not codes: return if int(codes) > 0: # check only if we are not rearranging deckchairs # the new state of things newcodes = book.codes + codes # only check the codes we're increasing codeslist = [(i, mitsfs.splitcode(i)[1]) for i in codes] hassleset = set([mitsfs.conf.hassle[codebase] for (code, codebase) in codeslist if codes[code] > 0 and codebase in mitsfs.conf.hassle]) # count up the books hassle = [(consider, keep, sum(newcodes[i] for i in consider)) for (consider, keep) in hassleset] # check to see if any of them violate our constraints hassle = [(consider, keep, count) for (consider, keep, count) in hassle if count > keep] if hassle: print codes,'results in',newcodes,'which seems like a lot' if not mitsfs.readyes('Are you sure you want to do that? '): return changecodes(book, codes) if not line: return def changecodes(book, codes): oldcodes = book.codes both = oldcodes + codes lost = False negs = [(code, count) for (code, count) in both.items() if count < 0] if negs: print 'Change would result in',mitsfs.shelfcodes(negs),'(%s)' % both print '(not doing it)' else: new = mitsfs.dexline(book, codes=codes) if not both and mitsfs.readyes('We will no longer have any copies, add to lostdex? '): lost = True dex.add(new, review, lost) print 'now', dex[new] if review and int(codes) > 0: reviewdex_add(new) def validate_field(field): cchars = False bchars = '' for c in field: if ord(c) < ord(' ') and not cchars: print 'No control characters. Tabs either.' cchars = True if c in '<>{}^\\': if c not in bchars: bchars += c if bchars: s = len(bchars) != 1 and 's' or '' print 'Illegal character'+s+':',bchars return not cchars and not bchars def validate_title_or_series(field): for c in field: if c in '[]': if not mitsfs.readyes('Do you really want those brackets? '): return False break if re.match(r'^(?:A|AN|THE) ', field): if not mitsfs.readyes('Do you really want to start with an article? '): return False return True def validate_title(field): if len(field.split('=')) > 2: print 'Only one placement title is allowed.' return False return True def validate_series(field): # a series name should not itself start with "@" if field[:2] == '@@': print 'May not have multiple leading @s.' return False if '|@' in field: print 'May only be @ first series' return False # random @s in the name are allowed ("b@nking") but likely mistakes if '@' in field[1:]: if not mitsfs.readyes("Do you really want an '@' as part of the series name? "): return False # better not have multiple #s, or any after |s if re.match(r'#.*#', field): print 'Only one #, please' return False if re.match(r'\|.*#', field): print '#s only in the first series, please' return False # check they didn't put in a shelfcode by mistake if field: try: mitsfs.shelfcodes(field) if not mitsfs.readyes('That looks like a shelfcode. Did you mean that? '): return False except mitsfs.InvalidShelfcode: pass if '=' in field: if not mitsfs.readyes("Do you really want an '=' as part of the series name?"): return False return True def validate_shelfcodes(field): if not field.strip(): return True try: mitsfs.shelfcodes(field) except mitsfs.InvalidShelfcode, e: print e return False return bool(field.strip()) def munge_series(field): # no spaces in "#1,2,3" part while True: newfield = re.sub(r'( [0-9#,]+) (?=[0-9#,]*(\Z|\|))', r'\1', field) if newfield == field: break else: field = newfield return field def munge_field(field): field = field.strip() field = re.sub(r'\s+', ' ', field) field = re.sub(r'\s([=|,])', r'\1', field) field = re.sub(r'([=|,])\s', r'\1', field) field = re.sub(r',(\S)', r', \1', field) return field def munge_author(author): return re.sub(r'\.(?![ \.,|]|\Z)', '. ', author) def readvalidate(prompt, callback = None, preload = None, munge = [munge_field], validate = [validate_field], history = None): if preload is None: result = '' else: result = preload while True: result = mitsfs.read(prompt, callback, result, history).upper().strip() if not result: return result # blank always validates for munger in munge: result = munger(result) for validater in validate: if not validater(result): break # ... so falls back around the while loop else: break # actually breaks the while loop return result def newentry(line): if line.strip().upper() == 'R': reviewthis = True else: reviewthis = review if reviewthis: print 'WARNING REVIEW MODE IS ON' author = readvalidate('Author: ', dex.indices.authors.iterkeys, munge = [munge_field, munge_author], history = 'authors') if not author: return title = readvalidate('Title: ', dex.indices.titles.iterkeys, validate = [validate_field, validate_title, validate_title_or_series], history = 'titles') if not title: return tl='<'.join([author, title, '', '']) if tl in dex: print print "* That's not new! We have",dex[tl].codes print return series = readvalidate('Series: ', dex.indices.series.iterkeys, munge = [munge_field, munge_series], validate = [validate_field, validate_series, validate_title_or_series], history = 'series') code = readvalidate('Code: ', lambda: (i+':' for i in mitsfs.codes.iterkeys()), validate = [validate_shelfcodes], history = 'codes') line = mitsfs.dexline('<'.join([author, title, series, code])) if line.codes: print 'entering ',line dex.add(line, reviewthis) newdex_add(line) if reviewthis: reviewdex_add(line) else: print 'No codes, not entering', line def mon(): return time.strftime('%b').lower() def reviewdex_add(book): foodex_add('review-' + mon(), book, recycle=True) def newdex_add(book): foodex_add('newdex-' + mon(), book, recycle=True) def lostdex_add(book): foodex_add('lostdex', book, zerok=True) def foodex_add(dexname, book, recycle=False, zerok = False): if not hasattr(dex, 'db'): filename = os.path.join(mitsfs.dexbase, dexname) if recycle: try: st = os.stat(filename) # file exists if (time.time()-st.st_mtime) > 40*86400: #older than 40 days os.unlink(filename) except OSError: # file does not exist; Proceed. pass foodex = mitsfs.dex(filename, zerok = zerok) foodex.add(book) foodex.save(filename) def motd(*args): mitsfs.motd() def filebug(line): smtpserver = 'localhost' to = 'libcomm-bugs@mit.edu' fro = '%s@mit.edu' % os.environ['USER'] desc = mitsfs.read('Short description: ') print body = mitsfs.readlines() # assemble the e-mail message report = [ 'To: %s' % to, 'From: %s' % fro, 'Subject: %s' % desc, ''] report.extend(body) report.append('') print print '---BUG REPORT---' msg = "\n".join(report) print msg if mitsfs.readyes('Send this report? [yN] '): session = smtplib.SMTP(smtpserver) smtpresult = session.sendmail([fro], [to], msg) if smtpresult: errstr = "" for recip in smtpresult.keys(): errstr = """Could not delivery mail to: %s Server said: %s %s %s""" % (recip, smtpresult[recip][0], smtpresult[recip][1], errstr) raise smtplib.SMTPException, errstr else: print 'Report sent.' def quithamster(line): if dex.modified: print print '* There is unsaved data.' print if (not dex.modified) or mitsfs.readyes('really? '): raise EOFError() def lessiter(iterator): pager = os.environ.get('PAGER', 'less') #os.environ.setdefault('LESS', '-eMX') #just, no. os.environ['LESS'] = '-eMX' try: out = os.popen(pager, 'w') for i in iterator: print >>out, i out.close() except IOError: pass def grep(pattern): if not pattern: pattern = mitsfs.read('pattern? ', history='grep') try: if pattern: while pattern[-1] == '\\': print 'Removing presumably spurious trailing \\.' pattern=pattern[:-1] lessiter(dex.grep(pattern)) except mitsfs.InvalidShelfcode, e: print 'In shelfcode query:',e except mitsfs.DataError, e: print 'While querying', e def validate_shelfcode(code): if not code.strip(): return True try: at, code, double = mitsfs.splitcode(code) if at: print 'No @s' return False except mitsfs.InvalidShelfcode: return False return True def arc(line): sourcecode = readvalidate('Source code: ', dex.indices.codes.iterkeys, validate = [validate_shelfcode], #, lambda code: len(dex.indices['codes'].get(code, [])) > 0], history = 'codes') if not sourcecode: return print 'extracting shelfcode' books = list(mitsfs.dexline(i) for i in dex.indices.codes[sourcecode]) print 'sorting extract...', sys.stdout.flush() try: books.sort(key=lambda v: v.sortkey()) except KeyError, e: print e return print 'done' mydex=mitsfs.dex(books) predicate = lambda book: sourcecode in book.codes print 'First book' start = mitsfs.specify(mydex, books[0], predicate) if not start: return print 'selected',start print 'Last book' finish = mitsfs.specify(mydex, books[-1], predicate) if not finish: return print 'selected',finish destcode = readvalidate('Destination code: ', mitsfs.codes.iterkeys, validate = [validate_shelfcode], history = 'codes') if not destcode: return starti = books.index(mitsfs.dexline(start)) finishi = books.index(mitsfs.dexline(finish)) for i in books[starti:finishi+1]: count = i.codes[sourcecode] changecodes(i, mitsfs.shelfcodes({sourcecode: -count, destcode: count})) def boxing(line): sourcecode = readvalidate('Source code: ', dex.indices['codes'].iterkeys, validate = [validate_shelfcode, lambda code: (len(dex.indices['codes'].get(code, [])) > 0) if code.strip() else True], history = 'codes') if not sourcecode: return sourcecode = sourcecode.upper() boxdex = mitsfs.boxDex() print 'sorting datadex...', sys.stdout.flush() try: dex.sortcode(sourcecode) except KeyError, e: print e return print 'done' description = '' while True: predicate = lambda book: sourcecode in book.codes if not dex.indices['codes'][sourcecode]: print 'All done, bye bye!' break print 'First book' start = mitsfs.specify(dex, dex.indices['codes'][sourcecode][0], predicate) if not start: break print 'selected',start print 'Last book' finish = mitsfs.specify(dex, None, predicate) if not finish: finish = dex.indices['codes'][sourcecode][-1] print 'selected',finish description = mitsfs.read('Box description: ', preload=description, history='box') if not description: break box = boxdex.newbox('KBX', sourcecode, description) print box destcode = box.shelfcode starti = dex.indices['codes'][sourcecode].index(start) finishi = dex.indices['codes'][sourcecode].index(finish) for i in dex.indices['codes'][sourcecode][starti:finishi+1]: count = i.codes[sourcecode] changecodes(i, mitsfs.shelfcodes({sourcecode: -count, destcode: count})) def hassledex(line): while True: codes = [i for i in re.split(',\s*|\s+,?\s*', line.upper()) if i] badcodes = [i for i in codes if i not in mitsfs.codes] if badcodes: print 'Bad shelfcodes:',' '.join(badcodes) else: unhassle = [i for i in codes if i not in mitsfs.conf.hassle] if unhassle: print "I don't know how to hassle:",' '.join(unhassle) elif codes: break line = mitsfs.read('Shelfcodes to hassle: ', mitsfs.codes.iterkeys, preload=line, history='codes').strip() if not line: return hassleset = set([mitsfs.conf.hassle[i] for i in codes]) candidates = [(consider, keep, [code for code in consider if code in codes]) for (consider, keep) in hassleset] underflow = 0 hassledex = mitsfs.dex() for book in dex: for consider, keep, target in candidates: goal = sum(book.codes[i] for i in consider) - keep possibles = sum(book.codes[i] for i in target) if goal > possibles: underflow += goal - possibles goal = possibles if goal < 1: continue pull = mitsfs.shelfcodes() codes = list(target) while int(pull) < goal: code = codes[0] codes = codes[1:] + codes[:1] if (book.codes - pull)[code] > 0: pull[code] += 1 if pull: hassledex.add(mitsfs.dexline(book, codes=pull)) stats = hassledex.stats() print 'Resulting hassledex contains:',', '.join('%s:%d' % (code, count) for (code, count) in sorted(stats.items())) print 'Underflow of',underflow if not stats or not mitsfs.readyes('Build hasslecomm run? '): return codes = list(stats) boxdex = mitsfs.boxDex() box = boxdex.newbox('HASSLE', codes[0], 'HassleComm run built '+mitsfs.timestamp()) codemap = {codes[0]: box.shelfcode} boxcode = box.boxcode for code in codes[:1]: box = boxdex.addcode(boxcode, code) codemap[code] = box.shelfcode print 'Building %s:' % boxcode for book in hassledex: dex.add(mitsfs.dexline(book, codes=-book.codes)) dex.add(mitsfs.dexline(book, codes=mitsfs.shelfcodes((codemap[code], count) for (code, count) in book.codes.items()))) print 'done:',', '.join('%s:%d' % (code, len(dex.indices['codes'][code])) for code in sorted(codemap.values())) def dexck(line): print "Not yet" def membership(line): mb = mitsfs.membook(dex) name_or_initials = mitsfs.read('Name or initials? ', preload=line, complete=mb.complete_name_or_initials).strip() if not name_or_initials: return possibles = mb.get(name_or_initials) member = None if len(possibles) == 0: print 'Creating...' if not name_or_initials: member = mitsfs.member(dex) if len(name_or_initials) < 4: member = mitsfs.member(dex, initials = name_or_initials) else: member = mitsfs.member(dex, name = name_or_initials) elif len(possibles) == 1: member = possibles[0] else: for i, member in enumerate(possibles): print 0, 'New entry' print i+1, member.initials, member.name n = mitsfs.readnumber('? ', 0, len(possibles) + 1, 'select') if n is None: return if n == 0: member = mitsfs.member(dex) member = possibles[n] if not member: return # I don't think this can happen at this point, but.... while True: fields = ['initials', 'name'] kf = dict((f[0].upper(), f) for f in fields) t = [] unfilled = False for f in fields: v = getattr(member, f) k = f[0].upper() if v is not None: t += [(k+'.',f.title(),'', v)] else: t += [(k+'.',f.title(),'*')] unfilled = True t += [()] keys = kf.keys() if not member.id: if not unfilled: t += [('C.','Create')] else: t += [('C.','Create','*','(there are unfilled fields)')] keys.append('C') else: t += [(' created %s by %s with %s' % (member.created, member.created_by, member.created_with),), ('modified %s by %s with %s' % (member.modified, member.modified_by, member.modified_with),), ()] t += [('X.','eXit')] keys.append('X') print if member.new: print 'Editing new member',member.initials or member.name or '' else: print 'Editing member',member.initials print print mitsfs.tabulate(t) what = mitsfs.read('action: ', lambda: keys, history = 'menu').upper().strip() if not what: continue if what == 'X': if member.new and not mitsfs.readyes('Are you sure you want to exit without saving? '): continue break elif member.new and what == 'C': if unfilled: print 'Please fill out the field marked with a *' else: member.create() print 'Created.' elif what in kf: f = kf[what] try: val = mitsfs.read(f.title()+'? ', preload = getattr(member, f) or '', history='memberfield') except KeyboardInterrupt: continue if val: setattr(member, f, val) else: print 'Unknown option',what def series(line): series = None while series is None: name = mitsfs.read('Series Name? ', preload=line, complete=dex.indices.series.complete).strip() if not name: return series = dex.series(name) if not series: print 'No such series.' while True: fields = ['name', 'comment'] kf = dict((f[0].upper(), f) for f in fields) t = [] for f in fields: v = getattr(series, f) k = f[0].upper() if v is not None: t += [(k+'.',f.title(),'', v)] else: t += [(k+'.',f.title(),'*')] t += [()] keys = kf.keys() t += [(' created %s by %s with %s' % (series.created, series.created_by, series.created_with),), ('modified %s by %s with %s' % (series.modified, series.modified_by, series.modified_with),), (),] count = len(series) if count: if count == 1: counts = '' else: counts = 's' t += [('L.', 'List series (%d title%s)' % (count, counts))] keys.append('L') else: t += [('','No titles in series')] t += [(), ('X.','eXit')] keys.append('X') print print 'Editing series',series.name print print mitsfs.tabulate(t) what = mitsfs.read('action: ', lambda: keys, history = 'menu').upper().strip() if not what: continue if what == 'X': break elif len(series) and what == 'L': lessiter(series) elif what in kf: f = kf[what] try: val = mitsfs.read(f.title()+'? ', preload = getattr(series, f) or '', history='seriesfield') except KeyboardInterrupt: continue if val: setattr(series, f, val) else: print 'Unknown option',what def book(line): book = mitsfs.specify_book(dex) if book is None: return while True: fields = ['title', 'shelfcode', 'visible', 'doublecrap', 'review', 'withdrawn', 'comment'] kf = dict((f[0].upper(), f) for f in fields) t = [] for f in fields: v = getattr(book, f) k = f[0].upper() if v is not None: t += [(k+'.',f.title(),'', str(v))] else: t += [(k+'.',f.title(),'*')] t += [()] keys = kf.keys() t += [(' created %s by %s with %s' % (book.created, book.created_by, book.created_with),), ('modified %s by %s with %s' % (book.modified, book.modified_by, book.modified_with),), (),] t += [(), ('X.','eXit')] keys.append('X') print print 'Editing book',book print print mitsfs.tabulate(t) what = mitsfs.read('action: ', lambda: keys, history = 'menu').upper().strip() if not what: continue if what == 'X': break elif what in kf: f = kf[what] try: if f == 'title': val = mitsfs.specify(dex, book.title) else: val = mitsfs.read(f.title()+'? ', preload = getattr(book, f) or '', history='bookfield') except KeyboardInterrupt: continue if val: setattr(book, f, val) else: print 'Unknown option',what def withdraw(line): while True: print print 'Book to withdraw ->' book = mitsfs.specify_book(dex) if not book: break if book.withdrawn: print book, 'is already withdrawn' continue book.withdrawn = True print book, ': withdrawn' def title(line): title = mitsfs.specify(dex) if title is None: return while True: fields = ['lang', 'lost', 'comment'] kf = dict((f[0].upper(), f) for f in fields) t = [] for f in fields: v = getattr(title, f) k = f[0].upper() if v is not None: t += [(k+'.',f.title(),'', str(v))] else: t += [(k+'.',f.title(),'*')] t += [()] keys = kf.keys() t += [(' created %s by %s with %s' % (title.created, title.created_by, title.created_with),), ('modified %s by %s with %s' % (title.modified, title.modified_by, title.modified_with),), (),] t += [(), ('X.','eXit')] keys.append('X') print print 'Editing title',title print print mitsfs.tabulate(t) what = mitsfs.read('action: ', lambda: keys, history = 'menu').upper().strip() if not what: continue if what == 'X': break elif what in kf: f = kf[what] try: val = mitsfs.read(f.title()+'? ', preload = getattr(title, f) or '', history='titlefield') except KeyboardInterrupt: continue if val: setattr(title, f, val) else: print 'Unknown option',what def checkdis(line): print 'Dissociated roles (key them or get the speaker to postgres to remove them)' print ' '.join(mitsfs.star_dissociated(dex)) def key(line): m = mitsfs.membook(dex) key = mitsfs.read('Member? ',preload=line, complete=m.complete_initials).strip() if not key: return mem = m[key] if not mem: print 'Unknown member' return print 'Keying', mem.name role = mitsfs.read('Kerberos name? ') if not role: return mitsfs.star_key(dex, key, role) def dekey(line): m = mitsfs.membook(dex) key = mitsfs.read('Keyholder? ', preload=line, callback=lambda: mitsfs.star_keys(dex)) if not key: return mem = m[key] if not mem: print 'Unknown member' return print 'Dekeying', mem.name if not mitsfs.readyes('Are you sure? '): return cttes = mitsfs.star_dekey(dex, key) if cttes: print 'Was on',' '.join(cttes) def maybeprettylist(x): if not x: return '' return '(%s)' % ', '.join(x) def keylist(line): print for key in mitsfs.star_keys(dex): print key, maybeprettylist(mitsfs.star_mem_ctte(dex, key)) print def cttelist(line): print for ctte in mitsfs.star_cttes(dex): print ctte, maybeprettylist(mitsfs.star_ctte_mem(dex, ctte)) print def ctteadd(line): print 'Adding...' key = mitsfs.read('Keyholder? ', preload=line, callback=lambda: mitsfs.star_keys(dex)).upper().strip() if not key: return ctte = mitsfs.read('Committee? ', callback=lambda: mitsfs.star_cttes(dex) + ['*chamber']).lower().strip() if not ctte: return mitsfs.star_ctte_add(dex, key, ctte) print key, maybeprettylist(mitsfs.star_mem_ctte(dex, key)) print ctte, maybeprettylist(mitsfs.star_ctte_mem(dex, key)) def ctteremove(line): print 'Removing...' key = mitsfs.read('Keyholder? ', preload=line, callback=lambda: mitsfs.star_keys(dex)).upper().strip() if not key: return ctte = mitsfs.read('Committee? ', callback=lambda: mitsfs.star_cttes(dex) + ['*chamber']).lower().strip() if not ctte: return mitsfs.star_ctte_remove(dex, key, ctte) print key, maybeprettylist(mitsfs.star_mem_ctte(dex, key)) print ctte, maybeprettylist(mitsfs.star_ctte_mem(dex, key)) class Quit(Exception): pass if __name__ == '__main__': main(sys.argv)