--explain mode

README.md

@@ -10,7 +10,7 @@ pip3 install -r requirements.txt
 You *need* **python3.8** to run Panoramix. Yes, there was no way around it.
 
 ```
-python3.8 panoramix.py address [func_name] [--verbose|--silent]
+python3.8 panoramix.py address [func_name] [--verbose|--silent|--explain]
 ```
 
 e.g.
@@ -23,11 +23,18 @@ or
 python3.8 panoramix.py kitties
 ```
 
-
 Output goes to two places:
 - `console`
 - ***`cache_pan/`*** directory - .pan, .json, .asm files
 
+If you want to see how Panoramix works under the hood, try the `--explain` mode:
+
+```
+python3.8 panoramix.py kitties paused --explain
+python3.8 panoramix.py kitties pause --explain
+python3.8 panoramix.py kitties tokenMetadata --explain
+```
+
 ### Optional parameters:
 
 func_name -- name of the function to decompile (note: storage names won't be discovered in this mode)
@@ -82,4 +89,4 @@ But if you manage to figure out a way to do it without Tilde (and maintain reada
 
 # How Panoramix works
 
-See the source code comments, starting with panoramix.py. Also, those slides.
+See the source code comments, starting with panoramix.py. Also, those slides[tbd].

pano/function.py

@@ -7,7 +7,7 @@
 
 from utils.helpers import EasyCopy, opcode, find_f_list, find_f, replace_f
 
-from pano.prettify import prettify, pprint_logic
+from pano.prettify import prettify, pprint_logic, explain_text
 
 from core.arithmetic import simplify_bool
 
@@ -314,8 +314,12 @@ def find_returns(exp):
             else:
                 return []
 
+        exp_text = []
+
         self.returns = find_f_list(self.trace, find_returns)
 
+        exp_text.append(('possible return values', prettify(self.returns)))
+
         first = self.trace[0]
 
         if opcode(first) == 'if' and simplify_bool(first[1]) == 'callvalue'   \
@@ -329,11 +333,15 @@ def find_returns(exp):
         else:
             self.payable = True
 
+        exp_text.append(('payable', self.payable))
+
         self.read_only = True
         for op in ['store', 'selfdestruct', 'call', 'delegatecall', 'codecall', 'create']:
             if f"'{op}'" in str(self.trace):
                 self.read_only = False
 
+        exp_text.append(('read_only', self.read_only))
+
 
         '''
             const func detection
@@ -355,6 +363,9 @@ def find_returns(exp):
         else:
             self.const = None
 
+        if self.const:
+            exp_text.append(('const', self.const))
+
         '''
             getter detection
         '''
@@ -408,4 +419,9 @@ def l2(x):
                 else:
                     pass
 
+        if self.getter:
+            exp_text.append((f'getter for', prettify(self.getter)))
+
+        explain_text('function traits', exp_text)
+
         return self

pano/prettify.py

@@ -24,6 +24,8 @@
 
 from pano.loader import Loader
 
+import sys
+
 logger = logging.getLogger(__name__)
 
 '''
@@ -38,6 +40,33 @@
 '''
 
 
+
+prev_trace = None
+def explain(title, trace):
+    global prev_trace
+
+    if '--explain' not in sys.argv:
+        return
+
+    if trace == prev_trace:
+        return
+
+    print('\n'+C.green_back+f" {title}: "+C.end+'\n')
+    pprint_trace(trace)
+    prev_trace = trace
+
+def explain_text(title, params):
+    global prev_trace
+
+    if '--explain' not in sys.argv:
+        return
+
+    print('\n'+C.blue_back+f" {title}: "+C.end+'\n')
+
+    for name, val in params:
+        print(f' {C.gray}{name}{C.end}: {val}')
+    print()
+
 def make_ast(trace):
     def store_to_set(line):
         if line ~ ('store', :size, :off, :idx, :val):
@@ -485,10 +514,10 @@ def pretty_line(r, add_color=True):
             yield "       args {}".format(', '.join(fparams))
 
     elif r ~ ('label', :name, :setvars):
-        yield COLOR_GREEN + f'label {str(name)} setvars: {str(setvars)}' + ENDC
+        yield COLOR_GREEN + f'loop {str(name)} setvars: {str(setvars)}' + ENDC
 
     elif r ~ ('goto', *rest):
-        yield COLOR_GREEN + f'goto {str(rest)}'+ENDC
+        yield COLOR_GREEN + f'continue {str(rest)}'+ENDC
 
     elif r ~ ('continue', :jd, :setvars):
         for v in setvars:

pano/simplify.py

@@ -7,7 +7,7 @@
 
 from core.memloc import range_overlaps, splits_mem, fill_mem, memloc_overwrite, split_setmem, apply_mask_to_range, split_store
 
-from utils.helpers import rewrite_trace_multiline, opcode, cached, walk_trace, to_exp2, replace, find_op_list
+from utils.helpers import is_array, C, rewrite_trace_multiline, opcode, cached, walk_trace, to_exp2, replace, find_op_list
 from utils.helpers import contains, find_f_set, find_f_list, rewrite_trace, rewrite_trace_full, replace, replace_f, replace_f_stop, rewrite_trace_ifs
 
 from core.algebra import simplify, calc_max, add_ge_zero, minus_op, sub_op, flatten_adds, max_to_add, divisible_bytes, _max_op, div_op
@@ -27,7 +27,7 @@
 
 from .rewriter import postprocess_exp, postprocess_trace, rewrite_string_stores
 
-from pano.prettify import pretty_repr
+from pano.prettify import pretty_repr, explain
 
 import sys
 
@@ -98,18 +98,42 @@ def simplify_trace(trace):
         # between every stage here to see changes that happen to the code.
 
         trace = replace_f(trace, simplify_exp)
+        explain(f'simplify expressions', trace)
+
         trace = cleanup_vars(trace)
+        explain('cleanup variables', trace)
+
         trace = cleanup_mems(trace)
+        explain(f'cleanup mems', trace)
+
         trace = rewrite_trace(trace, split_setmem)
         trace = rewrite_trace_full(trace, split_store)
+        explain('split setmems & storages', trace)
+
         trace = cleanup_vars(trace)
+        explain('cleanup vars', trace)
+
         trace = replace_f(trace, simplify_exp)
+        explain(f'simplify expressions', trace)
+
         trace = cleanup_mul_1(trace)
+        explain(f'simplify expressions', trace)
+
         trace = cleanup_msize(trace)
+        explain(f'calculate msize', trace)
+
         trace = replace_bytes_or_string_length(trace)
+        explain(f'replace storage with length', trace)
+
         trace = cleanup_conds(trace)
+        explain(f'cleanup unused ifs', trace)
+
         trace = rewrite_trace(trace, loop_to_setmem)
+        explain(f'convert loops to setmems', trace)
+
         trace = propagate_storage_in_loops(trace)
+        explain('move loop indexes outside of loops', trace)
+
 
         # there is a logic to this ordering, but it would take a long
         # time to explain. if you play with it, just run through bulk_compare.py
@@ -124,14 +148,17 @@ def simplify_trace(trace):
 
     trace = replace_f(trace, postprocess_exp)
     trace = replace_f(trace, postprocess_exp)
-
     trace = rewrite_trace_ifs(trace, postprocess_trace)
 
     trace = rewrite_trace_multiline(trace, rewrite_string_stores, 3)
+    explain('using heuristics to clean up some things', trace)
+
     trace = cleanup_mems(trace)
     trace = cleanup_mems(trace)
     trace = cleanup_mems(trace)
     trace = cleanup_conds(trace)
+    explain('final setmem/condition cleanup', trace)
+
 
     def fix_storages(exp):
         if exp ~ ('storage', :size, int:off, :loc) and off < 0:
@@ -140,9 +167,12 @@ def fix_storages(exp):
 
     trace = replace_f(trace, fix_storages)
     trace = cleanup_conds(trace)
+    explain('cleaning up storages slightly', trace)
+
 
-    logger.debug('readability')
     trace = readability(trace)
+    explain('adding nicer variable names', trace)
+
 
     trace = cleanup_mul_1(trace)
 

pano/vm.py

@@ -15,7 +15,7 @@
 
 from pano.prettify import pprint_trace
 
-from utils.helpers import precompiled, precompiled_var_names
+from utils.helpers import precompiled, precompiled_var_names, C
 
 import sys
 
@@ -212,7 +212,6 @@ def run(self, start, history = {}, condition = None, re_run = False):
         func_node = Node(vm=self, start=start, safe=True, stack=[])
         trace = [('setmem', ('range', 0x40, 32), 0x60), ('jump', func_node, 'safe', tuple())]
 
-
         root = Node(vm=self, trace=trace, start=start, safe=True, stack=[])
         func_node.set_prev(root)
 
@@ -367,6 +366,12 @@ def handle_jumps(self, trace, line, condition):
         i, op = line[0], line[1]
         stack = self.stack
 
+        if '--explain' in sys.argv and op in ('jump', 'jumpi', 'selfdestruct', 'stop', 'return', 'invalid', 'assert_fail', 'revert'):
+            trace.append(C.asm(f'       {stack}'))
+            trace.append('')
+            trace.append(f'[{line[0]}] {C.asm(op)}')
+
+
         if op == 'jump':
             target = stack.pop()
 
@@ -451,18 +456,19 @@ def trace_extend(l):
 
         previous_len = stack.len()
 
-        if logging.getLogger().isEnabledFor(logging.DEBUG):
-            trace(str(stack))
+        if '--verbose' in sys.argv or '--explain' in sys.argv:
+            trace(C.asm('       '+str(stack)))
+            trace('')
 
             if "push" not in op and "dup" not in op and "swap" not in op:
-                trace('[{}] {}',line[0],op)
+                trace('[{}] {}',line[0],C.asm(op))
             else:
                 if type(line[2]) == str:
-                    trace('[{}] {} {}',line[0],op," ”"+line[2]+"”")
+                    trace('[{}] {} {}',line[0],C.asm(op),C.asm(" ”"+line[2]+"”"))
                 elif line[2] > 0x1000000000:
-                    trace('[{}] {} {}',line[0],op,hex(line[2]))
+                    trace('[{}] {} {}',line[0],C.asm(op),C.asm(hex(line[2])))
                 else:                
-                    trace('[{}] {} {}',line[0],op,str(line[2]))
+                    trace('[{}] {} {}',line[0],C.asm(op),C.asm(str(line[2])))
 
         assert op not in ['jump', 'jumpi', 'revert', 'return', 'stop', 'jumpdest']
 

pano/whiles.py

@@ -7,7 +7,7 @@
 
 from core.memloc import range_overlaps, splits_mem, fill_mem, memloc_overwrite, split_setmem, apply_mask_to_range, split_store
 
-from utils.helpers import rewrite_trace_multiline, opcode, cached, walk_trace, to_exp2, replace, find_op_list
+from utils.helpers import C, rewrite_trace_multiline, opcode, cached, walk_trace, to_exp2, replace, find_op_list
 from utils.helpers import contains, find_f_set, find_f_list, rewrite_trace, rewrite_trace_full, replace, replace_f, replace_f_stop, rewrite_trace_ifs
 
 from core.algebra import simplify, calc_max, add_ge_zero, minus_op, sub_op, flatten_adds, max_to_add, divisible_bytes, _max_op, div_op
@@ -25,7 +25,7 @@
 
 from core.masks import to_mask, to_neg_mask
 
-from pano.prettify import pretty_repr
+from pano.prettify import pretty_repr, explain
 
 from pano.simplify import simplify_trace
 
@@ -42,9 +42,9 @@
 
 def make_whiles(trace): 
 
-    logger.info('making')
     trace = make(trace)
-    logger.info('cleaning up jumpdests')
+
+    explain('Loops -> whiles', trace)
 
     # clean up jumpdests
     trace = rewrite_trace(trace, lambda line: [] if line ~ ('jumpdest', ...) else [line])

panoramix.py

@@ -21,6 +21,10 @@
 import random
 import traceback
 
+import pano.folder as folder
+
+from utils.helpers import rewrite_trace
+
 import coloredlogs
 import logging
 
@@ -55,7 +59,7 @@
 from pano.function import Function
 from pano.vm import VM
 from pano.contract import Contract
-from pano.prettify import pprint_trace, pretty_type, pprint_repr
+from pano.prettify import pprint_trace, pretty_type, pprint_repr, explain
 
 VER = '4 Oct 2019'
 
@@ -241,14 +245,27 @@ def decompile(this_addr, only_func_name=None):
             @timeout_decorator.timeout(30, use_signals=True) # 180 used in production
             def dec():
                 trace = VM(loader).run(target)
+                explain('Initial decompiled trace', trace[1:])
+
+                if '--explain' in sys.argv:
+                    trace = rewrite_trace(trace, lambda line: [] if type(line) == str else [line])
+                    explain('Without assembly', trace)
+
                 trace = make_whiles(trace)
+                explain('final', trace)
+
+                if '--explain' in sys.argv:
+                    explain('folded', folder.fold(trace))
+
+
                 return trace
 
             trace = dec()
 
             functions[hash] = Function(hash, trace)
 
 
+
         except Exception as e:
             problems[hash] = fname