from __future__ import with_statement import sys, os, pwd, tempfile, threading, shutil, warnings, random, string import pysvn from bazbase import structure from bazbase.db import TransactionAborted from bazbase.flavors import FLAVORS from bazbase.benchmark import benchmarking from bazbase import custom as basecust from bazjunk.path import makedirs from bazyaml import format from .share import svndriven from . import custom state = threading.local() def checkouts_dir(): return "/tmp/%s.%s.bazki/checkouts" % (pwd.getpwuid(os.getuid())[0], basecust.APP_NAME) def locked_checkouts_dir(): return "/tmp/%s.%s.bazki/locked_checkouts" % (pwd.getpwuid(os.getuid())[0], basecust.APP_NAME) def lock_one_of(dirs): d = random.choice(dirs) nd = os.path.join(locked_checkouts_dir(), d) os.rename(os.path.join(checkouts_dir(), d), nd) return nd def update(client, tempd, rev): if client.info(tempd).revision.number != rev.number: client.update(tempd, revision=rev) def new_checkout_dir(): nd = os.path.join(locked_checkouts_dir(), ''.join(random.choice(string.lowercase) for x in xrange(32))) os.makedirs(nd) return nd def start_checkout(client, revision, repopath): try: while True: nd = lock_one_of(os.listdir(checkouts_dir())) if os.path.exists(os.path.join(nd, '.svn', 'entries')): break except (IndexError, OSError): nd = new_checkout_dir() thread = threading.Thread( target=lambda cl, tempd, rev: cl.checkout(repopath, tempd, revision=rev), args=(client, nd, revision)) else: thread = threading.Thread(target=update, args=(client, nd, revision)) if thread is not None: thread.start() return nd, thread def return_checkout(dir): makedirs(checkouts_dir()) os.rename(dir, os.path.join(checkouts_dir(), os.path.basename(dir))) def prefetch_checkout(): try: client = pysvn.Client() dirs = os.listdir(checkouts_dir()) nd = None if len(dirs) > 2: nd = lock_one_of(dirs) if os.path.exists(os.path.join(nd, '.svn', 'entries')): client.update(nd) else: # Don't return it, checkout something new. nd = None if nd is None: nd = new_checkout_dir() client.checkout('file:///'+os.path.abspath(custom.REPOSITORY), nd) return_checkout(nd) except OSError: # If we lost a race, we don't care. pass def clear_checkouts(): if os.path.exists(checkouts_dir()): shutil.rmtree(checkouts_dir()) if os.path.exists(locked_checkouts_dir()): shutil.rmtree(locked_checkouts_dir()) def _mkdir_p(path): bits = path.split('/') pathpnt = '' for b in bits: pathpnt += b+'/' if not os.path.exists(pathpnt): state.client.mkdir(pathpnt,"") # helper func def _setaddfile(path,val): #print path dir,fil = path.rsplit('/',1) _mkdir_p(dir) with open(path,'w') as fil: fil.write(val+'\n') try: state.client.add(path) except pysvn.ClientError: pass class InvalidState(Exception): pass class LockFailed(Exception): pass def _elementpath(element,parent=None,symlink=False,orgmode=None): above = [] if orgmode is None: orgmode = element.get_orgmode() while True: if len(above) == 0: if element.has_children() or orgmode == u'toplevel': above = [element.ename,element.ename+'.yaml'] else: above = [element.ename+'.yaml'] else: above = [element.ename] + above if parent is not None: element = parent parent = None else: element = element.get_parent() if element is None or orgmode == u'toplevel': break elif element is False: return None orgmode = element.get_orgmode() return os.path.join(*above) def notify(dct): if pysvn.wc_notify_state.conflicted in (dct['content_state'], dct['prop_state']): state.conflict = True def begin(): username = custom.get_username() if username: username = username.encode('utf-8') else: username = 'nobody' state.client = pysvn.Client() state.client.callback_notify = notify state.client.set_default_username(username) state.deltas = {} state.lock_state = False state.hostname = os.uname()[1] state.conflict = False repopath = 'file:///'+os.path.abspath(custom.REPOSITORY) revision = state.client.info2(repopath, recurse=False)[0][1]['rev'] state.tempd, state.checkout_thread = start_checkout(state.client, revision, repopath) return revision.number def setprop(element, pname, val, format=None): path = _elementpath(element) assert isinstance(val, str), repr(val) prop = structure.get_prop(pname) dtf = pname in element.get_final_map() if prop.default == val and dtf: state.deltas.setdefault(path, {}).setdefault(element.ename, {})[pname] = None else: if FLAVORS[prop.flavor].binary: assert format is not None val = (format, val) else: val = unicode(val, 'utf-8') state.deltas.setdefault(path, {}).setdefault(element.ename, {})[pname] = val def delete(element,pname): path = _elementpath(element) state.deltas.setdefault(path,{}).setdefault(element.ename,{})[pname]=None def esetattr(element, attr, oldval, val, bootstrapping=False): if attr == 'parent': assert element is not None assert isinstance(val, structure.Element) or val is None,val.__class__ olddelts = {} if oldval == val and not bootstrapping: return if oldval is not None: oldpath = _elementpath(element,parent=oldval) if (oldpath in state.deltas and element.ename in state.deltas[oldpath]): olddelts = state.deltas[oldpath][element.ename] newpath = _elementpath(element,parent=val) if newpath.endswith('.list') and (oldval is None or not oldpath.endswith('.list')): if oldval is not None: edelete(element) state.deltas.setdefault(newpath,{}).setdefault('parentage',{}).setdefault(val,{})[element.ename] = olddelts elif (oldval is not None and oldpath.endswith('.list') and not newpath.endswith('.list')): do_something() elif (oldval is not None and oldpath.endswith('.list') and newpath.endswith('.list')): do_something_else() else: # Just a simple listless move assert newpath not in state.deltas,(element,newpath,state.deltas[newpath]) if val is not None and not val.has_children() and val.get_orgmode() != 'toplevel': # Transform from foo.yaml to foo/foo.yaml oldparpath = os.path.dirname(newpath)+'.yaml' newparpath = os.path.dirname(newpath)+'/'+val.ename +'.yaml' state.checkout_thread.join() state.client.mkdir(state.tempd+'/' +os.path.dirname(newpath),"") state.client.move(state.tempd+'/'+oldparpath, state.tempd+'/'+newparpath,force=True) assert newparpath not in state.deltas if oldparpath in state.deltas: state.deltas[newparpath] = state.deltas[oldparpath] del state.deltas[oldparpath] if oldval is None or bootstrapping: state.checkout_thread.join() _setaddfile(state.tempd+'/'+newpath,'') elif oldpath != newpath: state.checkout_thread.join() if oldpath.endswith("%s/%s.yaml" % (element.ename, element.ename)): assert newpath.endswith("%s/%s.yaml" % (element.ename, element.ename)) oldpath = os.path.dirname(oldpath) newpath = os.path.dirname(newpath) if len(element.get_parent().get_children()) == 1: # We're the last child oldparpath = os.path.join(os.path.dirname(path), element.get_parent().ename+'.yaml') newparpath = os.path.join(os.path.dirname( os.path.dirname(path)), element.get_parent().ename+'.yaml') state.client.move(oldparpath, newparpath, force=True) state.client.remove([os.path.dirname(oldparpath)]) state.client.move(state.tempd+'/'+oldpath, state.tempd+'/'+newpath,force=True) # Make a symlink for toplevel elements if element.get_orgmode() == 'toplevel': state.checkout_thread.join() if element.get_parent() != val: oldsymlink = _elementpath(element,symlink=True) state.client.remove(state.tempd+'/'+oldsymlink) newsymlink = _elementpath(element,parent=val,symlink=True) if os.path.exists(state.tempd+'/'+newsymlink): os.unlink(state.tempd+'/'+newsymlink) dontadd=True else: dontadd=False os.symlink('../'*newsymlink.count('/')+newpath, state.tempd+'/'+newsymlink) if not dontadd: state.client.add(state.tempd+'/'+newsymlink) elif attr == 'ename': if element.has_children(): fil2 = os.path.dirname(_elementpath(element)) fil1 = os.path.join(os.path.dirname(fil2), oldval) else: fil2 = _elementpath(element) fil1 = os.path.join(os.path.dirname(fil2), oldval+'.yaml') assert fil1 != fil2, (fil1, fil2) state.checkout_thread.join() state.client.move(state.tempd + '/' + fil1, state.tempd + '/' + fil2, force=True) elif attr == 'orgmode': # TODO(xavid): Handle orgmode! assert False, "orgmode" was = _elementpath(element) will = _elementpath(element,orgmode=val) if was != will: assert will not in state.deltas if was in state.deltas: state.deltas[will] = state.deltas[was] del state.deltas[was] state.checkout_thread.join() if element.get_orgmode() == 'list': assert False elif val == 'list': assert False elif val == 'toplevel': assert element.get_orgmode() == 'normal' if was.endswith('.def'): # Move to toplevel and make symlink state.client.move(state.tempd+'/'+os.path.dirname(was), state.tempd+'/'+os.path.dirname(will), force=True) for k in state.deltas.keys(): if k.startswith(os.path.dirname(was)+'/'): new = k[len(os.path.dirname(was)):] new = os.path.dirname(will)+new assert new not in state.deltas state.deltas[new] = state.deltas[k] del state.deltas[k] else: assert was.endswith('.elm') # Make dir, move, make symlink state.client.mkdir(state.tempd+'/'+os.path.dirname(will), "") state.client.move(state.tempd+'/'+was, state.tempd+'/'+will,force=True) # Symlink sympath = _elementpath(element,orgmode=val,symlink=True) os.symlink('../'*sympath.count('/')+os.path.dirname(will), state.tempd+'/'+sympath) state.client.add(state.tempd+'/'+sympath) elif val == 'normal': assert element.get_orgmode() == 'toplevel' if will.endswith('.def'): # Move from toplevel and nix symlink state.client.move(state.tempd+'/'+os.path.dirname(was), state.tempd+'/'+os.path.dirname(will), force=True) for k in state.deltas.keys(): if k.startswith(os.path.dirname(was)+'/'): new = k[len(os.path.dirname(was)):] new = os.path.dirname(will)+new assert new not in state.deltas state.deltas[new] = state.deltas[k] del state.deltas[k] else: assert will.endswith('.elm') # Move, rmdir, nix symlink state.client.move(state.tempd+'/'+was, state.tempd+'/'+will,force=True) state.client.remove([state.tempd+'/'+os.path.dirname(was)]) # Nix symlink sympath = _elementpath(element,symlink=True) state.client.remove(state.tempd+'/'+sympath) else: raise InvalidState("Invalid orgmode '%s'!" % val) else: raise Exception('Unknown eattr %s!'%attr) def edelete(element): rpath = _elementpath(element) state.checkout_thread.join() path = os.path.join(state.tempd,rpath) if path.endswith('%s/%s.yaml' % (element.ename, element.ename)): raise InvalidState("Removing a node with children!") elif path.endswith('.list'): if os.path.basename(path) == element.ename+'.list': assert not element.has_children() state.client.remove([path],force=True) else: state.deltas.setdefault(path,{})[element.ename] = None if rpath in state.deltas and element.ename in state.deltas[rpath]: del state.deltas[rpath][element.ename] return # skip deltanukage else: state.client.remove([path], force=True) if len(element.get_parent().get_children()) == 1: # We're the last child oldparpath = os.path.join(os.path.dirname(path), element.get_parent().ename+'.yaml') newparpath = os.path.join(os.path.dirname(os.path.dirname(path)), element.get_parent().ename+'.yaml') state.client.move(oldparpath, newparpath, force=True) state.client.remove([os.path.dirname(oldparpath)]) if rpath in state.deltas: del state.deltas[rpath] def psetattr(prop,attr,val): assert attr in ('flavor','default','comment','pname','visible') state.checkout_thread.join() if attr == 'pname': fil1 = 'props/%s.yaml' % (prop.name) fil2 = 'props/%s.yaml' % (val) assert fil2 not in state.deltas if os.path.exists(state.tempd+'/'+fil1): state.client.move(state.tempd+'/'+fil1, state.tempd+'/'+fil2,force=True) if fil1 in state.deltas: assert len(state.deltas[fil1]) == 1, state.deltas[fil1] assert prop.name in state.deltas[fil1], state.deltas[fil1] state.deltas[fil2] = {val: state.deltas[fil1][prop.name]} del state.deltas[fil1] # We need to edit all elements that contain this prop. for e in prop.containing_elements(): old_pv = e.get_propval(val) assert old_pv, (e, e.get_propval(val), e.get_propval(prop.name)) setprop(e, val, old_pv.value, old_pv.format) delete(e, prop.name) else: _setaddfile(os.path.join(state.tempd,path),'') else: path = 'props/%s.yaml' % (prop.name) if not isinstance(val, unicode): val = unicode(str(val), 'utf-8') state.deltas.setdefault(path,{}).setdefault(prop.name,{})[unicode( attr, 'utf-8')] = val if not os.path.exists(state.tempd+'/'+path): _setaddfile(os.path.join(state.tempd,path),'') def pdelete(prop): state.checkout_thread.join() fil = 'props/%s.yaml' % (prop.name) state.client.remove([state.tempd+'/'+fil], force=True) if fil in state.deltas: del state.deltas[fil] class parse_handler(object): def __init__(self,subs,path): self.subs = subs self.dir = state.tempd+'/'+os.path.dirname(path) self.noprop = False def prop(self, e, p, contents): assert not self.noprop if e in self.subs and p in self.subs[e]: ret = self.subs[e][p] del self.subs[e][p] return ret else: return contents def start_element(self,e,parent=None): if e in self.subs and self.subs[e] is None: return False else: return True def end_element(self, e): pass def get_new_properties(self,e): self.noprop = True # Things can be None if they're new and defaulting, say for a new # element. return dict((p, self.subs[e][p]) for p in self.subs[e] if self.subs[e][p] is not None) def get_new_children(self,e): if e in self.parentage: return self.parentage[e] else: return [] def get_include(self, path): path = os.path.join(self.dir, path) with open(path, 'rb') as fil: return fil.read() def set_include(self, path, val, new): path = os.path.join(self.dir, path) if val is not None: if new: cnt = 1 # Start numbering at 2 origpath = path while os.path.exists(path): cnt += 1 path = ('.%s.' % cnt).join(origpath.rsplit('.', 1)) _setaddfile(path, val) else: assert not new if os.path.exists(path): state.client.remove([path], force=True) return os.path.basename(path) def commit(): assert not svndriven() with benchmarking('waiting for checkout'): state.checkout_thread.join() try: # Do the runaround of state.deltas for path in state.deltas: inf = open(state.tempd+'/'+path) # We replace the old file with a new file with the same path, # for ease of processing os.unlink(state.tempd+'/'+path) outf = open(state.tempd+'/'+path,mode='w') elms = state.deltas[path] if path.endswith('.yaml'): assert len(elms) == 1 format.parse(state.tempd+'/'+path, parse_handler(state.deltas[path],path),inf,outf) inf.close() outf.close() # -xavid: too slow for a rare-case fix ##update here to make sure no transactions are in progress and to merge ##harmless changes #state.client.update(state.tempd) #if state.conflict: # raise TransactionAborted("Subversion conflict!") except: state.revert_thread = threading.Thread(target=do_revert, args=(state.client,state.tempd)) state.revert_thread.start() raise else: log_message = "Web Edit: " + custom.get_commit_message() # commit changes revision = state.client.checkin(state.tempd, log_message=log_message) return_checkout(state.tempd) if revision is None: return None else: return revision.number def do_revert(client,tempd): client.revert(tempd, recurse=True) # Remove unversioned files. for status in client.status(tempd, get_all=False): if not status.is_versioned: p = os.path.join(tempd, status.path) if os.path.isdir(p): shutil.rmtree(p) else: os.unlink(p) else: raise Exception("Reverted client has file in unknown status: %s" % status) return_checkout(tempd) def abort(): assert not svndriven() try: state.checkout_thread.join() except AttributeError: pass try: state.revert_thread = threading.Thread(target=do_revert, args=(state.client,state.tempd)) except AttributeError: pass else: state.revert_thread.start() def get_revision(): client = pysvn.Client() # TODO(xavid): unify with impl above repopath = 'file:///'+os.path.abspath(custom.REPOSITORY) revision = client.info2(repopath, recurse=False)[0][1]['rev'] return revision.number def update_auth(): from bazbase import conversion with open(os.path.join(custom.REPOSITORY, 'conf', 'passwd'), 'w') as fil: fil.write('### The name and password for each user follow, one account per line.\n') fil.write('\n') fil.write('[users]\n') for admin in (structure.get_element(basecust.EDITOR_ANCESTOR) .get_descendants()): try: username = admin.get_prop(u'username') password = conversion.render(admin, u'password') except KeyError: pass else: if username and password: fil.write('%s = %s\n' % (username, password))