import sys, itertools from tokens import * import creole class Symbol(object): pass SELFENV = Symbol() class inner_macro_it: def __init__(self, macroname, it): self.macroname = macroname self.it = it self.closed = False self.macro_stack = [] def __iter__(self): return self def next(self): endmacroname = None t = self.it.next() if t.style == MACRO: if creole.debug: print >>sys.stderr, 'imi', self.macroname, t.op, t.arg, if creole.debug: print >>sys.stderr, self.macro_stack if t.op == END: if t.arg == self.macroname and not self.macro_stack: self.closed = t raise StopIteration elif (len(self.macro_stack) > 0 and self.macro_stack[-1] == t.arg): self.macro_stack.pop() elif t.op == START: self.macro_stack.append(t.arg[0]) return t def macro_it(macro_func, error_func, macroname, arglist, it): imit = inner_macro_it(macroname, it) mit = macro_func(macroname, arglist, imit) for i in mit: yield i if not imit.closed: for i in imit: pass if not imit.closed: error_func() if arglist is not None: yield Entity(ERROR, "Unclosed macro <<%s %s>>!" % (macroname, arglist)) else: yield Entity(ERROR, "Unclosed macro '%s'!" % (macroname)) else: yield imit.closed for i in it: yield i # Returns default display, {'url': url, 'style': css class, other metadata} def default_link_func(h, sty): return h, {'url': h, 'style': 'external' if '://' in h else 'internal'} class Parser(object): def __init__(self, format, macro_func, link_func=default_link_func, error_func=lambda:None): self.format = format self.macro_func = macro_func self.link_func = link_func self.error_func = error_func def parse(self, markup, format=None, link_func=None): assert isinstance(markup, basestring), markup return self.render(creole.tokenize(markup, self.error_func), format=format, link_func=link_func) def iparse(self, markup, format=None, link_func=None): return self.irender(creole.tokenize(markup, self.error_func), format=format, link_func=link_func) def render(self, tokens, format=None, link_func=None): return u''.join(self.irender(tokens, format=format, link_func=link_func)) END_ENTITY_MACRO = Op('END_ENTITY_MACRO') def irender(self, tokens, format=None, link_func=None): if format is None: format = self.format() if link_func is None: link_func = self.link_func it = iter(tokens) env_stack = [] start_depth = 0 macro_env_stack = [] env_closed = False start_stack = [] while True: try: t = it.next() except StopIteration: if creole.debug: print >>sys.stderr, StopIteration, env_stack, start_stack # TODO(xavid): unify with duplicate code below while start_stack: oldsty, oldenv = start_stack.pop() if oldenv is not None: assert env_stack == [[False, None]], (oldsty, env_stack) env_stack = oldenv for s in format.end(oldsty, None): yield s while env_stack: env, envarg = env_stack.pop() if env is False: env_stack = saved_env elif env is True: pass else: assert env, repr(env) for s in format.end(env, envarg): yield s break if creole.debug: print >>sys.stderr, t assert isinstance(t, Token), (repr(it), macroname, repr(t)) sty = t.style is_macro_close = ((t.op == END and sty == MACRO) or t.op is self.END_ENTITY_MACRO) is_nonmacro_close = (t.op == END and sty != MACRO and start_stack and sty == start_stack[-1][0] and start_stack[-1][1] is not None) if t.op == END and sty != MACRO: if creole.debug: print >>sys.stderr, 'nmc?', sty, start_stack can_clear_env = True #(( #(not macro_env_stack # or (is_macro_close # and len(macro_env_stack) == 1) # or (env_stack and env_stack[-1][0] == PARAGRAPH) # ) and #start_depth == 0) # or (sty == ENV_BREAK and t.arg is True)) print >>sys.stderr, 11, env_closed, start_stack, t.op, t.arg text_when_closed = (env_closed or not start_stack ) and t.op == TEXT and t.arg.strip() # Should we implicitly close something, like a list item, here? print >>sys.stderr, 'CHECK', t.op, start_stack, t if start_stack and ((t.op == START and not start_stack[-1][0].link and sty and sty.env is not True and not sty.inline and sty != MACRO) or (sty == ENV_BREAK)): # TODO(xavid): duplicate of below where explicit ends oldsty, oldenv = start_stack.pop() print 'fwee!', oldsty, t if oldenv is not None: assert env_stack == [[False, None]], (oldsty, env_stack) env_stack = oldenv start_depth -= 1 for s in format.end(oldsty, None): yield s if ((sty and t.op != END and sty.env is not False and (sty.env is not None or can_clear_env)) #or (macro_env_stack and is_macro_close) or (env_stack and env_stack[-1][0] is not False and is_nonmacro_close) or (env_stack and text_when_closed)): if is_macro_close: if creole.debug: print >>sys.stderr, 'mac_close', macro_env_stack dest_env, arg = macro_env_stack.pop() elif is_nonmacro_close: if creole.debug: print >>sys.stderr, 'nmclose', env_stack dest_env = False arg = None elif text_when_closed: if creole.debug: print >>sys.stderr, 'text_when_closed', t dest_env = PARAGRAPH arg = None else: dest_env = sty.env arg = t.arg if t.op == END: env_closed = True else: env_closed = False incr_done = False if (env_stack and (can_clear_env or is_nonmacro_close or (dest_env is not True and dest_env.group and env_stack[-1][0] is not True and env_stack[-1][0] is not False and dest_env.group == env_stack[-1][0].group) or is_macro_close)): if dest_env != env_stack[-1][0] and ( not dest_env or dest_env is True or not dest_env.group or env_stack[-1][0] is True or (dest_env is not True and env_stack[-1][0] is not True and dest_env.group != env_stack[-1][0].group)): if creole.debug: print >>sys.stderr, ( "break", env_stack, "to", dest_env, "via", sty, arg, start_depth, can_clear_env) clear = False while env_stack: if env_stack[-1][0] == dest_env: break env, envarg = env_stack.pop() if env is not True and envarg is not SELFENV: for s in format.end(env, envarg): yield s else: clear = True if creole.debug: print >>sys.stderr, 'cleared to', env_stack if sty == ENV_BREAK: # We closed the environment. if creole.debug: print >>sys.stderr, "clear EB=>", clear t.arg = clear elif clear: if creole.debug: print >>sys.stderr, "clear so ENV_BREAK", t for s in format.entity(ENV_BREAK, True): yield s elif (dest_env and dest_env is not True and dest_env.group and dest_env.group == env_stack[-1][0].group): top_env, top_arg = env_stack[-1] if creole.debug: print >>sys.stderr, "trans", dest_env, if creole.debug: print >>sys.stderr, top_arg, arg if top_arg > arg: for x in xrange(top_arg, arg, -1): env, envarg = env_stack.pop() assert envarg == x, (envarg, x) for s in format.end(env, envarg): yield s elif top_arg < arg: for x in xrange(top_arg + 1, arg + 1): env_stack.append([dest_env, x]) for s in format.start(dest_env, x): yield s elif top_env != dest_env: env_stack.pop() for s in format.end(top_env, top_arg): yield s env_stack.append([dest_env, arg]) for s in format.start(dest_env, arg): yield s incr_done = True if sty == ENV_BREAK: if creole.debug: print >>sys.stderr, "oother EB case" else: if sty == ENV_BREAK: if creole.debug: print >>sys.stderr, "other EB case" else: if creole.debug: print >>sys.stderr, "start env", sty, env_stack if not env_stack and sty == ENV_BREAK: t.arg = True # If we had something incompatable, we'd have cleared it out # above. print 'ES', env_stack, dest_env if (env_stack and env_stack[-1][0] is True and dest_env is True and t.op == END): if creole.debug: print >>sys.stderr, "pop True" env_stack.pop() elif (dest_env and not incr_done and (not env_stack or env_stack[-1][0] != dest_env)): if dest_env is not True and dest_env.group: for x in xrange(arg): env_stack.append([dest_env, x + 1]) for s in format.start(dest_env, x + 1): yield s else: env_stack.append([dest_env, arg]) if dest_env is not True: for s in format.start(dest_env, arg): yield s print 'ES!', env_stack elif not env_stack and sty != ENV_BREAK and sty != MACRO and t.op != END: if t.op == TEXT and not t.arg.strip(): continue if creole.debug: print >>sys.stderr, "start", PARAGRAPH, "due to", sty for s in format.start(PARAGRAPH): yield s env_stack.append([PARAGRAPH, None]) elif sty == ENV_BREAK: if creole.debug: print >>sys.stderr, 'C', sty, env_stack, can_clear_env, macro_env_stack, is_macro_close, start_stack if t.op is self.END_ENTITY_MACRO: continue elif t.op == START: if sty == MACRO: macroname, arglist = t.arg if env_stack and env_stack[-1][0] != PARAGRAPH: macro_env_stack.append(env_stack[-1]) assert macroname is not None, t.arg assert macroname.strip(), (macroname, t.arg) it = macro_it(self.macro_func, self.error_func, macroname, arglist, it) else: if (env_stack and env_stack[-1][0] != PARAGRAPH): if creole.debug: print >>sys.stderr, 'Saving env_stack', env_stack, t saved_env = env_stack env_stack = [[False, None]] if sty.selfenv: env_stack.append([sty, SELFENV]) else: saved_env = None start_stack.append([sty, saved_env]) start_depth += 1 print 'start_stack', start_stack, start_depth if sty.link: display, metadata = link_func(t.arg, sty) for s in format.start(sty, metadata): yield s else: for s in format.start(sty, t.arg): yield s elif t.op == END: if sty == MACRO: pass else: assert start_stack and start_stack[-1][0] == sty, (sty, start_stack) oldsty, oldenv = start_stack.pop() if oldenv is not None: assert env_stack == [[False, None]], (sty, env_stack) env_stack = oldenv start_depth -= 1 if sty.link: display, metadata = link_func(t.arg, sty) for s in format.end(sty, metadata): yield s else: for s in format.end(sty, t.arg): yield s elif t.op == ENTITY: if sty == MACRO: macroname, arglist = t.arg if env_stack and env_stack[-1][0] != PARAGRAPH: macro_env_stack.append(env_stack[-1]) it = itertools.chain(self.macro_func(macroname, arglist), [Token(self.END_ENTITY_MACRO)], it) elif sty == ENV_BREAK: # Environment closing handled above; arg may have been set # then. for s in format.entity(sty, t.arg): yield s elif sty.link: assert isinstance(t.arg, unicode), repr(t.arg) display, metadata = link_func(t.arg, sty) assert isinstance(display, unicode) for s in format.start(sty, metadata): yield s for s in format.text(display): yield s for s in format.end(sty, metadata): yield s else: for s in format.entity(sty, t.arg): yield s elif t.op == TEXT: for s in format.text(t.arg): yield s elif t.op == LITERAL: yield t.arg else: assert False, t if __name__ == '__main__': print [dir(t) for t in Parser().tokenize('This* is a **bold** [[Plan|plan]] at ~** http://plan.com/.')]