diff -r 3a2bd70c01df -r b4a9a3122abb backend.py --- a/backend.py Wed Aug 29 23:57:58 2018 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,840 +0,0 @@ -# 2.5.8 backend - -# written by VB. - -import re, codecs -import fileinput -import sys, traceback, exceptions, os -from xml.sax.saxutils import escape, quoteattr -from copy import deepcopy -from glob import glob -from pyPEG import code, parse, parseLine, u, Symbol -from yml2 import ymlCStyle, comment, _inner - -ymlFunc, pointers, pythonFunc = {}, {}, {} -in_ns = u"" -operator = [] -included = u"" -includePath = [] -emitlinenumbers = False -encoding = "utf-8" - -first = True -enable_tracing = False - -ymlFunc["decl"] = "#error" -ymlFunc["define"] = "#error" -ymlFunc["operator"] = "#error" - -def clearAll(): - global ymlFunc, pointers, pythonFunc, in_ns, operator, included - ymlFunc, pointers, pythonFunc = {}, {}, {} - in_ns = u"" - operator = [] - included = u"" - -lq = re.compile(r"\|(\>*)(.*)") -sq = re.compile(r"(\d*)\>(.*)") -ts = re.compile(r'(\|\|(?P\>*)\s*\n(?P.*?)\n(?P\s*)\|\|)|("""(?P.*?)""")|(\>\>(?P.*?)\>\>)', re.S) -tq = re.compile(r"(\]|\<)\s*(.*)") -bq = re.compile(r"\`(.*?)\`", re.S) -bqq = re.compile(r"\s*\`\`(.*)") -all = re.compile(r".*", re.S) - -line = 1 - -def pointer(name): - try: - return u(pointers[name[1:]]) - except: - if name == "*_trace_info": - return u'""' - if included: - raise LookupError(u"in " + included + u":" + u(line) + u": pointer " + name) - else: - raise LookupError(u"in " + u(line) + u": pointer " + name) - -def evalPython(expr): - try: - result = eval(u(expr), pythonFunc) - if type(result) is str: - return codecs.decode(result, encoding) - else: - return result - except: - name, parm, tb = sys.exc_info() - msg = u"in python expression: " + u(parm) - if name is exceptions.SyntaxError: - tbl = traceback.format_exception(name, parm, tb) - msg += u"\n" + tbl[-3] + tbl[-2] - else: - msg += u": " + expr + u"\n" - if included: - raise name(u"in " + included + u":" + u(line) + u": " + msg) - else: - raise name(u"in " + u(line) + u": " + msg) - -def execPython(script): - try: - if type(script) is unicode: - exec script in pythonFunc - else: - exec codecs.decode(script, encoding) in pythonFunc - except: - name, parm, tb = sys.exc_info() - msg = u"in python script: " + u(parm) - if name is exceptions.SyntaxError: - tbl = traceback.format_exception(name, parm, tb) - msg += u"\n" + tbl[-3] + tbl[-2] - else: - msg += u": " + expr + u"\n" - if included: - raise name(u"in " + included + u":" + u(line) + u": " + msg) - else: - raise name(u"in " + u(line) + u": " + msg) - -def textOut(text): - if not text: - return u"" - if type(text) is not unicode: - text = codecs.decode(text, encoding) - text = text.replace(r'\"', r'\\"') - text = u'u"""' + text.replace('"', r'\"') + u'"""' - try: - textFunc = ymlFunc["text"] - parms = ['text', ('parm', [text])] - c, result = textFunc(parms) - if c: - if type(textFunc.alias) is unicode: - result += u"" - else: - result += u"" - return result - except: - return escape(eval(text)) - -def strRepl(text): - if not text: - return u"" - if type(text) is not unicode: - text = codecs.decode(text, encoding) - text = text.replace(r'\"', r'\\"') - text = u'u"""' + text.replace('"', r'\"') + u'"""' - if type(text) is unicode: - return escape(eval(text)) - -def applyMacros(macros, text): - result = text - for key, value in macros.iteritems(): - result = result.replace(key, value) - return result - -class YF: - def __init__(self, name): - self.name = name - self.parms = [] - self.descends = [] - self.values = {} - self.content = None - self.pointers = {} - self.macros = {} - if in_ns: - self.alias = in_ns + u":" + name.replace("_", "-") - else: - self.alias = name.replace("_", "-") - pythonFunc["yml_" + name] = self - if emitlinenumbers: - self.values["yml:declared"] = u(line) - - def copy(self, newName): - yf = YF(newName) - yf.parms.extend(self.parms) - yf.descends.extend(self.descends) - yf.values = self.values.copy() - yf.content = self.content - yf.pointers = self.pointers.copy() - yf.macros = self.macros.copy() - yf.alias = self.alias - return yf - - def patch(self, second): - self.parms.extend(second.parms) - self.descends.extend(second.descends) - self.values.update(second.values) - if second.content: - self.content = second.content - self.pointers.update(second.pointers) - self.macros.update(second.macros) - - def __call__(self, called_with, hasContent = False, avoidTag = False): - global pointers - parms = [] - vals = {} - if self.pointers: - pointers.update(self.pointers) - - for data in called_with: - if type(data) is tuple or type(data) is Symbol: - if data[0] == "parm": - l = data[1] - parm = l[0] - if parm[0] == "*": - parm = pointer(parm) - if len(l) == 1: - if type(parm) is tuple or type(parm) is Symbol: - if parm[0] == "pyExp": - val = evalPython(parm[1][0]) - parms.append(val) - else: - parms.append(evalPython((parm))) - else: - if type(parm) is tuple or type(parm) is Symbol: - if parm[0] == "pyExp": - parm = evalPython(parm[1][0]) - val = l[1] - if type(val) is tuple or type(val) is Symbol: - if val[0] == "pyExp": - val = evalPython(val[1][0]) - if val[0] == "*": - val = pointer(val) - if u(val)[0] == '"' or u(val)[0] == "'": - vals[parm] = evalPython(u(val)) - else: - vals[parm] = u(val) - elif data[0] == "content": - hasContent = True - - if enable_tracing: - text = u(parms) + u", " + u(vals) - pointers["_trace_info"] = u'"' + u(line) + u": " + u(self.name) + u" " + text.replace(u'"', u'#') + u'"' - - if emitlinenumbers: - global first - if first: - vals["xmlns:yml"] = u"http://fdik.org/yml" - first = False - vals["yml:called"] = u(line) - return self.xml(parms, vals, hasContent, avoidTag) - - def addParm(self, parm): - if parm[0] == "%": - for i in range(len(self.parms)): - if self.parms[i][0] != "%": - self.parms.insert(i, parm) - return - self.parms.append(parm) - - def addDescend(self, desc): - if desc[0] == "+" or desc[0] == "@": - self.descends.append(desc[1:]) - else: - self.descends.append(desc) - - def addValue(self, parm, value): - if type(value) is str or type(value) is unicode: - if value[0] != "'" and value[0] != '"': - self.values[parm] = u(value) - else: - self.values[parm] = u(evalPython(value)) - else: - self.values[parm] = u(evalPython(u(value))) - - def xml(self, callParms, callValues, hasContent, avoidTag = False): - global pointers - extraContent = u"" - if self.content: - hasContent = True - resultParms = self.values.copy() - macros = self.macros.copy() - toDelete = resultParms.keys() - for key in toDelete: - if key[0] == "*": - del resultParms[key] - for key, value in callValues.iteritems(): - if key[0] == "%": - macros[key] = value - else: - resultParms[key] = value - i = 0 - for cp in callParms: - if i < len(self.parms): - if self.parms[i][0] == "*": - cp = u(cp) - if "'" in cp: - pointers[self.parms[i][1:]] = u'"' + cp + u'"' - else: - pointers[self.parms[i][1:]] = u"'" + cp + u"'" - elif self.parms[i][0] == "%": - macros[self.parms[i]] = u(cp) - else: - resultParms[self.parms[i]] = cp - else: - extraContent += u(cp) - hasContent = True - i += 1 - result = u"" - for p, v in resultParms.iteritems(): - if p[0] == "'" or p[0] == '"': - p = eval(p) - result += u" "+ p + u"=" + quoteattr(applyMacros(macros, u(v))) - if hasContent: - if avoidTag: - return True, strRepl(extraContent) - else: - return True, u"<" + self.alias + result + u">" + strRepl(extraContent) - else: - if avoidTag: - return False, u"" - else: - return False, u"<" + self.alias + result + u"/>" - -def replaceContent(tree, subtree): - n = 0 - while n < len(tree): - obj = tree[n] - if obj[0] == "func": - l = obj[1] - if l[0] == "content": - d = 1 - if subtree: - for el in subtree: - tree.insert(n+d, el) - d += 1 - del tree[n] - n += d - else: - try: - if l[-1][0] == "content": - replaceContent(l[-1][1], subtree) - except: pass - elif obj[0] == "funclist": - replaceContent(obj[1], subtree) - n += 1 - return tree - -def executeCmd(text): - if type(text) is not unicode: - text = codecs.decode(text, encoding) - for (regex, pattern) in operator: - match = re.search(regex, text) - while match: - cmd = pattern - opt = match.groups() - for i in range(len(opt)): - cmd = cmd.replace(u"%" + u(i+1), opt[i]) - text = text[:match.start()] + u"`" + cmd + u"`"+ text[match.end():] - match = re.search(regex, text) - - result = u"" - m = re.search(bq, text) - while text and m: - cmd = m.group(1) - head = textOut(text[:m.start()]) - text = text[m.end():] - try: - r, rest = parseLine(cmd, _inner, [], True, comment) - if rest: raise SyntaxError(cmd) - except SyntaxError: - if included: - raise SyntaxError(u"in " + included + u":" + u(line) + u": syntax error in executing command: " + cmd.strip()) - else: - raise SyntaxError(u"in " + u(line) + u": syntax error in executing command: " + cmd.strip()) - inner = _finish(r) - result += head + inner - m = re.search(bq, text) - result += textOut(text) - - return result - -def codegen(obj): - global in_ns, pointers, line, included - ctype = obj[0] - - if type(obj) is code: - return obj - - try: - if ctype.line: line = ctype.line - except: pass - - if ctype == "empty": - return code(u"") - - if ctype == "in_ns": - in_ns = obj[1][0] - subtree = obj[1] - for sel in subtree: - codegen(sel) - in_ns = u"" - return code(u"") - - elif ctype == "decl": - name = u"" - for data in obj[1]: - if type(data) is unicode or type(data) is str: - name = data - try: - yf = ymlFunc[name] - yf.alias - except: - ymlFunc[name] = YF(name) - yf = ymlFunc[name] - if in_ns: - yf.alias = in_ns + u":" + name - if not enable_tracing: - if in_ns == "xsl" and (name == "debug" or name=="assert" or name[:7]=="_trace_"): - yf.alias = "-" - yf.addParm("skip1") - yf.addParm("skip2") - break - elif type(data) is tuple or type(data) is Symbol: - if data[0] == "base": - base = data[1][0] - try: - yf = ymlFunc[name] = ymlFunc[base].copy(name) - except KeyError: - if included: - raise KeyError(u"in " + included + u":" + u(line) + u": " + base + u" as base for " + name) - else: - raise KeyError(u"in " + u(line) + u": " + base + u" as base for " + name) - elif data[0] == "shape": - shape = ymlFunc[data[1]] - try: - yf = ymlFunc[name] - yf.patch(shape) - except KeyError: - if included: - raise KeyError(u"in " + included + u":" + u(line) + u": " + base + u" as shape for " + name) - else: - raise KeyError(u"in " + u(line) + u": " + base + u" as shape for " + name) - elif data[0] == "descend": - yf.addDescend(data[1]) - elif data[0] == "declParm": - l = data[1] - parmName = l[0] - if len(l)==1: - yf.addParm(parmName) - else: - value = l[1] - if parmName[0] != "%": - yf.addValue(parmName, value) - if parmName[0] == "*": - yf.pointers[parmName[1:]] = value - yf.addParm(parmName) - elif parmName[0] == "%": - if type(value) is unicode or type(value) is str: - yf.macros[parmName] = u(evalPython(value)) - else: - yf.macros[parmName] = u(evalPython(u(value))) - yf.addParm(parmName) - elif data[0] == "alias": - if in_ns: - yf.alias = in_ns + u":" + data[1][0] - else: - yf.alias = data[1][0] - elif data[0] == "content": - yf.content = data[1] - - return code(u"") - - elif ctype == "funclist": - result = u"" - for f in obj[1]: - result += codegen(f) - return code(result) - - elif ctype == "parentheses": - if len(obj[1]): - return codegen(('func', ['_parentheses', ('content', [obj[1][0]])])) - else: - return u"" - - elif ctype == "fparm": - if len(obj[1]): - return codegen(('func', ['_parm', ('content', [obj[1][0]])])) - else: - return u"" - - elif ctype == "generic": - return codegen(('func', ['_generic', ('content', [obj[1][0]])])) - - elif ctype == "xbase": - return codegen(('func', ['_base', ('content', [obj[1][0]])])) - - elif ctype == "func": - avoidTag = False - name = obj[1][0] - - if name == "decl": - if ymlFunc[name] == "#error": - if included: - raise SyntaxError(u"in " + included + u":" + u(line) + u": syntax error in decl statement") - else: - raise SyntaxError(u"in " + u(line) + u": syntax error in decl statement") - if name == "define" or name == "operator": - if ymlFunc[name] == "#error": - if included: - raise SyntaxError(u"in " + included + u":" + u(line) + u": syntax error in define statement") - else: - raise SyntaxError(u"in " + u(line) + u": syntax error in define statement") - - if name[0] == "&": - avoidTag = True - name = name[1:] - hasContent = False - - if len(name) > 2: - if name[0:2] == "**": - return code(eval('u'+pointer(name[1:]))) - - if name[0] == "*": - name = eval(pointer(name)) - if name[0] == "&": - avoidTag = True - name = name[1:] - - try: - ymlFunc[name] - except: - try: - ymlFunc["_"] - return codegen(('func', ['_', ('content', [('funclist', [obj])])])) - except: - ymlFunc[name] = YF(name) - - if ymlFunc[name].alias == "-": avoidTag = True - - to_add = [] - if len(ymlFunc[name].descends): - if obj[1][-1][0] != 'content': - if included: - raise KeyError(u"in " + included + u":" + u(line) + u": " + name + u" has descending attributes, but no descendants are following") - else: - raise KeyError(u"in " + u(line) + u": " + name + u" has descending attributes, but no descendants are following") - - def first_func(obj): - if type(obj) is tuple or type(obj) is Symbol: - if obj[0] == 'func': - return obj - elif obj[0] == 'funclist': - return first_func(obj[1]) - elif obj[0] == 'content': - return first_func(obj[1]) - else: - return None - elif type(obj) == list: - for e in obj: - f = first_func(e) - if f: return f - return None - - def copy_without_first_func(o, found = False): - c = [] - for obj in o: - if found: - c.append(obj) - else: - if obj[0] == 'func': - if obj[1][-1][0] == 'content': - c.extend( obj[1][-1][1] ) - found = True - else: - c.append( ( obj[0], copy_without_first_func(obj[1], False ) ) ) - return c - - def get_parms(obj): - result = [] - for e in obj[1]: - if type(e) is tuple or type(e) is Symbol: - if e[0] == "parm": - result.append( e ) - return result - - try: - add_params = get_parms(obj) - for e in obj[1][-1][1]: - c = e[1] - for dname in ymlFunc[name].descends: - f, c = first_func(c), copy_without_first_func(c) - if dname[0] == "*": - pointers[dname[1:]] = "'" + f[1][0] + "'" - else: - add_params.append( ('parm', [dname, u"'" + f[1][0] + u"'"]) ) - try: - add_params.extend( get_parms(f) ) - except: pass - - new_things = [ e[1][0] ] - new_things.extend( add_params ) - new_things.append( ('content', c) ) - - to_add.append( ('func', new_things ) ) - except: - if included: - raise KeyError(u"in " + included + u":" + u(line) + u": " + name + u" has descending attributes, and too less descendants are following") - else: - raise KeyError(u"in " + u(line) + u": " + name + u" has descending attributes, and too less descendants are following") - - if not to_add: - to_add = ( obj, ) - - complete = u"" - - for obj in to_add: - subtree = None - try: - if obj[1][-1][0] == "content": - subtree = obj[1][-1][1] - except: pass - - if ymlFunc[name].content: - hasContent = True - treetemplate = deepcopy(ymlFunc[name].content) - subtree = replaceContent(treetemplate, subtree) - - if subtree: - hasContent = True - - hasContent, result = ymlFunc[name](obj[1], hasContent, avoidTag) - - if subtree: - for sel in subtree: - result += codegen(sel) - - if hasContent and not(avoidTag): - result += u"" - - complete += result - - return code(complete) - - elif ctype == "textsection": - result = u'' - ll = obj[1].splitlines() - space = len(ll[-1]) - 2 - for l in ll[1:-1]: - m = re.match(bqq, l) - if m: - cmd = m.group(1) - try: - r, x = parseLine(cmd, _inner, [], True, comment) - if x: raise SyntaxError(cmd) - result += _finish(r) - except SyntaxError: - if included: - raise SyntaxError(u"in " + included + u":" + u(line) + u": syntax error in executing command: " + cmd.strip()) - else: - raise SyntaxError(u"in " + u(line) + u": syntax error in executing command: " + cmd.strip()) - else: - result += codegen(Symbol(u'lineQuote', u'| ' + l[space:])) - return code(result) - - elif ctype == "textsectionu": - result = u'' - ll = obj[1].splitlines() - space = len(ll[-1]) - 2 - for l in ll[1:-1]: - m = re.match(bqq, l) - if m: - cmd = m.group(1) - try: - r, x = parseLine(cmd, _inner, [], True, comment) - if x: raise SyntaxError(cmd) - result += _finish(r) - except SyntaxError: - if included: - raise SyntaxError(u"in " + included + u":" + u(line) + u": syntax error in executing command: " + cmd.strip()) - else: - raise SyntaxError(u"in " + u(line) + u": syntax error in executing command: " + cmd.strip()) - else: - if result != u'': result += u' ' - result += codegen(Symbol(u'quote', [u'> ' + l[space:]])) - return code(result) - - elif ctype == "lineQuote" or ctype == "quote": - m, text, base, inds = None, u"", 0, 0 - - if ctype == "lineQuote": - text = obj[1] - m = lq.match(text) - if m: - inds = len(m.group(1)) - text = m.group(2)[1:] - else: inds = 0 - elif ctype == "quote": - inds = -1 - text = obj[1][0] - m = sq.match(text) - if m: - if m.group(1): - inds = int(m.group(1)) - text = m.group(2)[1:] - else: - if type(text) is unicode or type(text) is str: - text = u(evalPython(text)) - - ind = u"" - if inds > -1: - try: - cmd = evalPython(u"indent(" + u(inds) + u")") - result, rest = parseLine(u(cmd), _inner, [], True, comment) - if rest: - raise SyntaxError() - ind = _finish(result) - except: pass - - if ctype == "lineQuote": text += u"\n" - - hasTextFunc = False - try: - ymlFunc["text"] - hasTextFunc = True - except: pass - - text = executeCmd(text) - return code(ind + text) - - elif ctype == "tagQuote": - m = tq.match(obj[1]) - if m.group(1) == "<": - return code(u"<" + m.group(2)) - else: - return code(m.group(2)) - - elif ctype == "operator": - operator.append((re.compile(evalPython(obj[1][0])), obj[1][1])) - return code(u"") - - elif ctype == "constant": - name = obj[1][0] - if name[0] == "*": - name = name[1:] - value = obj[1][1] - pointers[name] = value - return code(u"") - - elif ctype == "include": - reverse = False - ktext, kxml = False, False - for arg in obj[1]: - if type(arg) is tuple or type(arg) is Symbol: - if arg[0] == "reverse": - reverse = True - elif arg[0] == "ktext": - ktext = True - elif arg[0] == "kxml": - kxml = True - elif type(arg) is unicode or type(arg) is str: - filemask = arg - - if filemask[0] == '/' or filemask[0] == '.': - files = sorted(glob(filemask)) - else: - files = [] - for directory in includePath: - path = os.path.join(directory, filemask) - files.extend(sorted(glob(path))) - - if files and reverse: - files = files[-1::-1] - - if not(files): - if included: - raise IOError(u"in " + included + ":" + u(line) + u": include file(s) '" + filemask + u"' not found") - else: - raise IOError(u"in " + u(line) + u": include file(s) '" + filemask + u"' not found") - - includeFile = fileinput.input(files, mode="rU", openhook=fileinput.hook_encoded(encoding)) - _included = included - if ktext or kxml: - text = u"" - for line in includeFile: - included = includeFile.filename() - if kxml: - if (not line[:6] == '