From a2d1343fd856b30ebe1f59bf50e4fbf0c6c261bd Mon Sep 17 00:00:00 2001 From: CensoredUsername Date: Tue, 20 Feb 2024 23:48:27 +0100 Subject: [PATCH] Transition decompiler to python 3. --- decompiler/__init__.py | 40 +++++++++++++------------- decompiler/astdump.py | 49 +++++++++++++++++++++++--------- decompiler/atldecompiler.py | 5 +--- decompiler/sl2decompiler.py | 7 ++--- decompiler/testcasedecompiler.py | 3 +- decompiler/translate.py | 4 +-- decompiler/util.py | 23 +++++++-------- setup.py | 2 +- unrpyc.py | 21 +++++++------- 9 files changed, 87 insertions(+), 67 deletions(-) diff --git a/decompiler/__init__.py b/decompiler/__init__.py index 658fb53a..f9fecb86 100644 --- a/decompiler/__init__.py +++ b/decompiler/__init__.py @@ -18,23 +18,21 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from __future__ import unicode_literals -from util import DecompilerBase, First, WordConcatenator, reconstruct_paraminfo, \ - reconstruct_arginfo, string_escape, split_logical_lines, Dispatcher, \ - say_get_code, OptionBase +from .util import DecompilerBase, First, WordConcatenator, reconstruct_paraminfo, \ + reconstruct_arginfo, string_escape, split_logical_lines, Dispatcher, \ + say_get_code, OptionBase from operator import itemgetter -from StringIO import StringIO +from io import StringIO -import magic -magic.fake_package(b"renpy") +from . import magic +magic.fake_package("renpy") import renpy -import sl2decompiler -import testcasedecompiler -import atldecompiler -import codegen -import astdump +from . import sl2decompiler +from . import testcasedecompiler +from . import atldecompiler +from . import astdump __all__ = ["astdump", "codegen", "magic", "sl2decompiler", "testcasedecompiler", "translate", "util", "Options", "pprint", "Decompiler"] @@ -147,7 +145,8 @@ def print_imspec(self, imspec): if len(imspec[6]) > 0: words.append("behind %s" % ', '.join(imspec[6])) - if isinstance(imspec[4], unicode): + # todo: this check probably doesn't work in ren'py 8 + if isinstance(imspec[4], str): words.append("onlayer %s" % imspec[4]) if imspec[5] is not None: @@ -226,7 +225,8 @@ def print_scene(self, ast): self.write("scene") if ast.imspec is None: - if isinstance(ast.layer, unicode): + # todo: this check probably doesn't work in ren'py 8 + if isinstance(ast.layer, str): self.write(" onlayer %s" % ast.layer) needs_space = True else: @@ -385,7 +385,7 @@ def print_if(self, ast): for i, (condition, block) in enumerate(ast.entries): # The non-Unicode string "True" is the condition for else:. # todo: this probably isn't true anymore for 8.0/7.5 and upwards. - if (i + 1) == len(ast.entries) and not isinstance(condition, unicode): + if (i + 1) == len(ast.entries) and not isinstance(condition, str): self.indent() self.write("else:") else: @@ -524,7 +524,8 @@ def print_menu_item(self, label, condition, block, arguments): self.write(reconstruct_arginfo(arguments)) if block is not None: - if isinstance(condition, unicode): + # todo: this check probably doesn't work in ren'py 8 + if isinstance(condition, str): self.write(" if %s" % condition) self.write(":") self.print_nodes(block, 1) @@ -559,8 +560,9 @@ def print_menu(self, ast): # if the condition is a unicode subclass with a "linenumber" attribute it was script. # If it isn't ren'py used to insert a "True" string. This string used to be of type str - # but nowadays it's of time unicode, just not of type PyExpr - if isinstance(condition, unicode) and hasattr(condition, "linenumber"): + # but nowadays it's of type unicode, just not of type PyExpr + # todo: this check probably doesn't work in ren'py 8 + if isinstance(condition, str) and hasattr(condition, "linenumber"): if self.say_inside_menu is not None and condition.linenumber > self.linenumber + 1: # The easy case: we know the line number that the menu item is on, because the condition tells us # So we put the say statement here if there's room for it, or don't if there's not @@ -720,7 +722,7 @@ def print_style(self, ast): if ast.variant.linenumber not in keywords: keywords[ast.variant.linenumber] = WordConcatenator(False) keywords[ast.variant.linenumber].append("variant %s" % ast.variant) - for key, value in ast.properties.iteritems(): + for key, value in ast.properties.items(): if value.linenumber not in keywords: keywords[value.linenumber] = WordConcatenator(False) keywords[value.linenumber].append("%s %s" % (key, value)) diff --git a/decompiler/astdump.py b/decompiler/astdump.py index f05add43..553fda81 100644 --- a/decompiler/astdump.py +++ b/decompiler/astdump.py @@ -18,8 +18,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from __future__ import unicode_literals - import sys import inspect import ast as py_ast @@ -40,7 +38,7 @@ class AstDumper(object): MAP_CLOSE = {list: ']', tuple: ')', set: '})', frozenset: '})'} def __init__(self, out_file=None, no_pyexpr=False, - comparable=False, indentation=u' '): + comparable=False, indentation=" "): self.indentation = indentation self.out_file = out_file or sys.stdout self.comparable = comparable @@ -70,9 +68,11 @@ def print_ast(self, ast): self.print_pyexpr(ast) elif isinstance(ast, dict): self.print_dict(ast) - elif isinstance(ast, (str, unicode)): + elif isinstance(ast, str): self.print_string(ast) - elif isinstance(ast, (int, long, bool)) or ast is None: + elif isinstance(ast, (bytes, bytearray)): + self.print_bytes(ast) + elif isinstance(ast, (int, bool)) or ast is None: self.print_other(ast) elif inspect.isclass(ast): self.print_class(ast) @@ -135,7 +135,7 @@ def should_print_key(self, ast, key): ast.col_offset = 0 # TODO maybe make this match? elif key == 'name' and type(ast.name) == tuple: name = ast.name[0] - if isinstance(name, unicode): + if isinstance(name, str): name = name.encode('utf-8') ast.name = (name.split(b'/')[-1], 0, 0) elif key == 'location' and type(ast.location) == tuple: @@ -232,16 +232,37 @@ def print_class(self, ast): def print_string(self, ast): # prints the representation of a string. If there are newlines in this string, # it will print it as a docstring. + if '\n' in ast: + astlist = ast.split('\n') + self.p('"""') + self.p(self.escape_string(astlist.pop(0))) + for i, item in enumerate(astlist): + self.p('\n') + self.p(self.escape_string(item)) + self.p('"""') + self.ind() + + else: + self.p(repr(ast)) + + def print_bytes(self, ast): + # prints the representation of a bytes object. If there are newlines in this string, + # it will print it as a docstring. + is_bytearray = isinstance(ast, bytearray) + if b'\n' in ast: astlist = ast.split(b'\n') - if isinstance(ast, unicode): - self.p('u') + if is_bytearray: + self.p('bytearray(') + self.p('b') self.p('"""') self.p(self.escape_string(astlist.pop(0))) for i, item in enumerate(astlist): self.p('\n') self.p(self.escape_string(item)) self.p('"""') + if is_bytearray: + self.p(')') self.ind() else: @@ -249,10 +270,12 @@ def print_string(self, ast): def escape_string(self, string): # essentially the representation of a string without the surrounding quotes - if isinstance(string, unicode): - return repr(string)[2:-1] - elif isinstance(string, str): + if isinstance(string, str): return repr(string)[1:-1] + elif isinstance(string, bytes): + return repr(string)[2:-1] + elif isinstance(string, bytearray): + return repr(bytes(string))[2:-1] else: return string @@ -266,10 +289,10 @@ def ind(self, diff_indent=0, ast=None): # shouldn't indent in case there's only one or zero objects in this object to print if ast is None or len(ast) > 1: self.indent += diff_indent - self.p(u'\n' + self.indentation * self.indent) + self.p('\n' + self.indentation * self.indent) def p(self, string): # write the string to the stream - string = unicode(string) + string = str(string) self.linenumber += string.count('\n') self.out_file.write(string) diff --git a/decompiler/atldecompiler.py b/decompiler/atldecompiler.py index 4fdf931c..8d7165e1 100644 --- a/decompiler/atldecompiler.py +++ b/decompiler/atldecompiler.py @@ -1,7 +1,4 @@ - - -from __future__ import unicode_literals -from util import DecompilerBase, WordConcatenator, Dispatcher +from .util import DecompilerBase, WordConcatenator, Dispatcher import renpy diff --git a/decompiler/sl2decompiler.py b/decompiler/sl2decompiler.py index b51fc7ed..07956050 100644 --- a/decompiler/sl2decompiler.py +++ b/decompiler/sl2decompiler.py @@ -18,14 +18,13 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from __future__ import unicode_literals import sys from operator import itemgetter -from util import DecompilerBase, First, reconstruct_paraminfo, \ - reconstruct_arginfo, split_logical_lines, Dispatcher +from .util import DecompilerBase, First, reconstruct_paraminfo, \ + reconstruct_arginfo, split_logical_lines, Dispatcher -import atldecompiler +from . import atldecompiler from renpy import ui, sl2 from renpy.ast import PyExpr diff --git a/decompiler/testcasedecompiler.py b/decompiler/testcasedecompiler.py index ae8e49aa..2b2723b7 100644 --- a/decompiler/testcasedecompiler.py +++ b/decompiler/testcasedecompiler.py @@ -18,8 +18,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from __future__ import unicode_literals -from util import DecompilerBase, split_logical_lines, Dispatcher, string_escape +from .util import DecompilerBase, split_logical_lines, Dispatcher, string_escape from renpy.test import testast # Main API diff --git a/decompiler/translate.py b/decompiler/translate.py index 872993b1..9cad91cd 100644 --- a/decompiler/translate.py +++ b/decompiler/translate.py @@ -18,14 +18,14 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from util import say_get_code +from .util import say_get_code import renpy import hashlib import re from copy import copy -class Translator(object): +class Translator: def __init__(self, language, saving_translations=False): self.language = language self.saving_translations = saving_translations diff --git a/decompiler/util.py b/decompiler/util.py index 1388d206..61579b91 100644 --- a/decompiler/util.py +++ b/decompiler/util.py @@ -1,15 +1,14 @@ -from __future__ import unicode_literals import sys import re -from StringIO import StringIO +from io import StringIO from contextlib import contextmanager -class OptionBase(object): +class OptionBase: def __init__(self, indentation=" ", printlock=None): self.indentation = indentation self.printlock = printlock -class DecompilerBase(object): +class DecompilerBase: def __init__(self, out_file=None, options=OptionBase()): # the file object that the decompiler outputs to self.out_file = out_file or sys.stdout @@ -62,7 +61,7 @@ def write(self, string): """ Shorthand method for writing `string` to the file """ - string = unicode(string) + string = str(string) self.linenumber += string.count('\n') self.skip_indent_until_write = False self.out_file.write(string) @@ -105,7 +104,7 @@ def rollback_state(self, state): def advance_to_line(self, linenumber): # If there was anything that we wanted to do as soon as we found a blank line, # try to do it now. - self.blank_line_queue = filter(lambda m: m(linenumber), self.blank_line_queue) + self.blank_line_queue = [m for m in self.blank_line_queue if m(linenumber)] if self.linenumber < linenumber: # Stop one line short, since the call to indent() will advance the last line. # Note that if self.linenumber == linenumber - 1, this will write the empty string. @@ -178,7 +177,7 @@ def print_unknown(self, ast): def print_node(self, ast): raise NotImplementedError() -class First(object): +class First: # An often used pattern is that on the first item # of a loop something special has to be done. This class # provides an easy object which on the first access @@ -298,7 +297,7 @@ def simple_expression_guard(s): # Some things we deal with are supposed to be parsed by # ren'py's Lexer.simple_expression but actually cannot # be parsed by it. figure out if this is the case - # a slightly more naive approach woudl be to check + # a slightly more naive approach would be to check # for spaces in it and surround it with () if necessary # but we're not naive s = s.strip() @@ -311,7 +310,7 @@ def simple_expression_guard(s): def split_logical_lines(s): return Lexer(s).split_logical_lines() -class Lexer(object): +class Lexer: # special lexer for simple_expressions the ren'py way # false negatives aren't dangerous. but false positives are def __init__(self, string): @@ -483,18 +482,18 @@ def __init__(self, needs_space, reorderable=False): self.reorderable = reorderable def append(self, *args): - self.words.extend(filter(None, args)) + self.words.extend(i for i in args if i) def join(self): if not self.words: return '' if self.reorderable and self.words[-1][-1] == ' ': - for i in xrange(len(self.words) - 1, -1, -1): + for i in range(len(self.words) - 1, -1, -1): if self.words[i][-1] != ' ': self.words.append(self.words.pop(i)) break last_word = self.words[-1] - self.words = map(lambda x: x[:-1] if x[-1] == ' ' else x, self.words[:-1]) + self.words = [x[:-1] if x[-1] == ' ' else x for x in self.words[:-1]] self.words.append(last_word) rv = (' ' if self.needs_space else '') + ' '.join(self.words) self.needs_space = rv[-1] != ' ' diff --git a/setup.py b/setup.py index 9e786522..0e4ed69c 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 from setuptools import setup def readme(): diff --git a/unrpyc.py b/unrpyc.py index 93214d75..f4bae7b2 100755 --- a/unrpyc.py +++ b/unrpyc.py @@ -27,6 +27,7 @@ import itertools import traceback import struct +import zlib from operator import itemgetter try: @@ -53,10 +54,10 @@ def release(self): SPECIAL_CLASSES = [set, frozenset] @SPECIAL_CLASSES.append -class PyExpr(magic.FakeStrict, unicode): +class PyExpr(magic.FakeStrict, str): __module__ = "renpy.ast" def __new__(cls, s, filename, linenumber, py=None): - self = unicode.__new__(cls, s) + self = str.__new__(cls, s) self.filename = filename self.linenumber = linenumber self.py = py @@ -64,9 +65,9 @@ def __new__(cls, s, filename, linenumber, py=None): def __getnewargs__(self): if self.py is not None: - return unicode(self), self.filename, self.linenumber, self.py + return str(self), self.filename, self.linenumber, self.py else: - return unicode(self), self.filename, self.linenumber + return str(self), self.filename, self.linenumber @SPECIAL_CLASSES.append class PyCode(magic.FakeStrict): @@ -126,7 +127,7 @@ def read_ast_from_file(in_file): raw_contents = in_file.read() # ren'py 8 only uses RPYC 2 files - if not raw_contents.startswith("RENPY RPC2"): + if not raw_contents.startswith(b"RENPY RPC2"): raise Exception("This isn't a normal rpyc file") # parse the archive structure @@ -140,7 +141,7 @@ def read_ast_from_file(in_file): chunks[slot] = raw_contents[start: start + length] - raw_contents = chunks[1].decode('zlib') + raw_contents = zlib.decompress(chunks[1]) # import pickletools # with open("huh.txt", "wb") as f: @@ -268,7 +269,7 @@ def main(): help="instead of decompiling, pretty print the ast to a file") parser.add_argument('-p', '--processes', dest='processes', action='store', type=int, - choices=range(1, cc_num), default=cc_num - 1 if cc_num > 2 else 1, + choices=list(range(1, cc_num)), default=cc_num - 1 if cc_num > 2 else 1, help="use the specified number or processes to decompile." "Defaults to the amount of hw threads available minus one, disabled when muliprocessing is unavailable.") @@ -333,7 +334,7 @@ def glob_or_complain(s): if not retval: print("File not found: " + s) return retval - filesAndDirs = map(glob_or_complain, args.file) + filesAndDirs = [glob_or_complain(i) for i in args.file] # Concatenate lists filesAndDirs = list(itertools.chain(*filesAndDirs)) @@ -352,7 +353,7 @@ def glob_or_complain(s): print("No script files to decompile.") return - files = map(lambda x: (args, x, path.getsize(x)), files) + files = [(args, x, path.getsize(x)) for x in files] processes = int(args.processes) if processes > 1: # If a big file starts near the end, there could be a long time with @@ -363,7 +364,7 @@ def glob_or_complain(s): else: # Decompile in the order Ren'Py loads in files.sort(key=itemgetter(1)) - results = map(worker, files) + results = list(map(worker, files)) if args.write_translation_file: print("Writing translations to %s..." % args.write_translation_file)