Migrate payments, orders & infos to classes

* Create `BaseClass` class + general `Payments` class
* Integrate new `Paypal` class
* Add classes `Orders` & `Infos`
* Add invoice helpers for extracting IDs & dates
* Move CSV-related options to class props
* Harmonize `invoice2number` nomenclature

knv_cli/__init__.py

@@ -1,8 +1,8 @@
 from .algorithms.contacts import get_contacts
 from .algorithms.matching import Matching
 from .algorithms.ranking import get_ranking
-from .processors.paypal import process_payments
-from .processors.shopkonfigurator import process_orders, process_infos
+from .processors.paypal import Paypal
+from .processors.shopkonfigurator import Orders, Infos
 from .utils import dedupe, group_data
 
 __all__ = [
@@ -12,9 +12,9 @@
     'get_contacts',
 
     # Processors
-    'process_payments',
-    'process_orders',
-    'process_infos',
+    'Paypal',
+    'Orders',
+    'Infos',
 
     # Utilities
     'dedupe',

knv_cli/database.py

@@ -7,10 +7,10 @@
 from shutil import move
 from zipfile import ZipFile
 
-from .processors.paypal import process_payments
-from .processors.shopkonfigurator import process_orders, process_infos
-from .utils import load_csv, load_json, dump_json
-from .utils import build_path, dedupe, group_data
+from .processors.paypal import Paypal
+from .processors.shopkonfigurator import Orders, Infos
+from .utils import load_json, dump_json
+from .utils import build_path, dedupe, group_data, invoice2number
 
 
 class Database:
@@ -35,25 +35,15 @@ def import_payments(self) -> None:
         import_files = build_path(self.config.import_dir, self.config.payment_regex)
 
         # Generate payment data by ..
-        # (1) .. fetching their content
-        import_data = load_csv(import_files, 'utf-8', ',')
+        # (1) .. extracting information from import files
+        handler = Paypal()
+        handler.load_csv(import_files)
 
-        # (2) .. removing duplicates
-        # (3) .. extracting information
-        import_data, _ = process_payments(dedupe(import_data))
-
-        # Load database files
-        db_files = build_path(self.config.payment_dir)
-        payments = load_json(db_files)
-
-        # Compare existing & imported data if database was built before ..
-        payments = self.merge_data(payments, import_data, 'Transaktion')
-
-        # Sort payments by date
-        payments.sort(key=itemgetter('Datum'))
+        # (2) .. merging with existing data
+        handler.load_json(build_path(self.config.payment_dir))
 
         # Split payments per-month & export them
-        for code, data in group_data(payments).items():
+        for code, data in group_data(handler.payments()).items():
             dump_json(data, join(self.config.payment_dir, code + '.json'))
 
 
@@ -62,52 +52,33 @@ def import_orders(self) -> None:
         import_files = build_path(self.config.import_dir, self.config.order_regex)
 
         # Generate order data by ..
-        # (1) .. fetching their content
-        import_data = load_csv(import_files)
-
-        # (2) .. removing duplicates
-        # (3) .. extracting information
-        import_data = process_orders(dedupe(import_data))
-
-        # Load database files
-        db_files = build_path(self.config.order_dir)
-        orders = load_json(db_files)
+        # (1) .. extracting information from import files
+        handler = Orders()
+        handler.load_csv(import_files)
 
-        # Compare existing & imported data if database was built before ..
-        orders = self.merge_data(orders, import_data, 'ID')
+        # (2) .. merging with existing data
 
-        # Sort orders by date
-        orders.sort(key=itemgetter('Datum'))
+        handler.load_json(build_path(self.config.order_dir))
 
         # Split orders per-month & export them
-        for code, data in group_data(orders).items():
+        for code, data in group_data(handler.orders()).items():
             dump_json(data, join(self.config.order_dir, code + '.json'))
 
 
     def import_infos(self) -> None:
         # Select info files to be imported
         import_files = build_path(self.config.import_dir, self.config.info_regex)
 
-        # Generate info data by ..
-        # (1) .. fetching their content
-        import_data = load_csv(import_files)
-
-        # (2) .. removing duplicates
-        # (3) .. extracting information
-        import_data = process_infos(dedupe(import_data))
-
-        # Load database files
-        db_files = build_path(self.config.info_dir)
-        infos = load_json(db_files)
-
-        # Compare existing & imported data if database was built before ..
-        infos = self.merge_data(infos, import_data, 'ID')
+        # Generate order data by ..
+        # (1) .. extracting information from import files
+        handler = Infos()
+        handler.load_csv(import_files)
 
-        # Sort infos by date
-        infos.sort(key=itemgetter('Datum'))
+        # (2) .. merging with existing data
+        handler.load_json(build_path(self.config.info_dir))
 
         # Split infos per-month & export them
-        for code, data in group_data(infos).items():
+        for code, data in group_data(handler.infos()).items():
             dump_json(data, join(self.config.info_dir, code + '.json'))
 
 
@@ -117,14 +88,14 @@ def import_invoices(self) -> None:
 
         # Check invoices currently in database
         invoices = build_path(self.config.invoice_dir, '*.pdf')
-        invoices = [basename(invoice) for invoice in invoices]
+        invoices = {invoice2number(invoice): invoice for invoice in invoices}
 
         for invoice_file in invoice_files:
             try:
                 with ZipFile(invoice_file) as archive:
                     for zipped_invoice in archive.namelist():
                         # Import only invoices not already in database
-                        if not zipped_invoice in invoices:
+                        if not invoice2number(zipped_invoice) in invoices:
                             archive.extract(zipped_invoice, self.config.invoice_dir)
 
             except:

knv_cli/processors/base.py

@@ -0,0 +1,84 @@
+import json
+
+from abc import ABCMeta, abstractmethod
+from hashlib import md5
+
+from pandas import concat, read_csv
+
+
+class BaseClass(metaclass=ABCMeta):
+    # Props
+    data = None
+    identifier = None
+
+    # CSV options
+    encoding='iso-8859-1'
+    delimiter=';'
+    skiprows=None
+
+
+    def load_csv(self, csv_files) -> None:
+        try:
+            df = concat(map(lambda file: read_csv(
+                file,
+                sep=self.delimiter,
+                encoding=self.encoding,
+                low_memory=False,
+                skiprows=self.skiprows
+            ), csv_files))
+
+        except ValueError:
+            return []
+
+        self.load_data(self.process_data(df.to_dict('records')))
+
+
+    def load_json(self, json_files) -> None:
+        data = []
+
+        for json_file in json_files:
+            try:
+                with open(json_file, 'r') as file:
+                    data.extend(json.load(file))
+
+            except json.decoder.JSONDecodeError:
+                raise Exception
+
+            except FileNotFoundError:
+                pass
+
+        self.load_data(data)
+
+
+    def load_data(self, data: list) -> None:
+        if self.data:
+            # Permit only unique entries, either by ..
+            if self.identifier is not None:
+                # .. (1) using a unique identifier
+                codes = {item[self.identifier] for item in self.data}
+
+                # Merge only data not already in database
+                for item in data:
+                    if item[self.identifier] not in codes:
+                        codes.add(item[self.identifier])
+                        self.data.append(item)
+
+            else:
+                # (2) .. hashing the whole item
+                codes = set()
+
+                for item in data:
+                    hash_digest = md5(str(item).encode('utf-8')).hexdigest()
+
+                    if hash_digest not in codes:
+                        codes.add(hash_digest)
+                        self.data.append(item)
+
+        # .. otherwise, start from scratch
+        else:
+            self.data = data
+
+
+    @abstractmethod
+    def process_data(self, data: list) -> list:
+        pass

knv_cli/processors/payments.py

@@ -0,0 +1,26 @@
+from abc import abstractmethod
+from operator import itemgetter
+
+from .base import BaseClass
+
+class Payments(BaseClass):
+    # Props
+    _blocked_payments = []
+
+
+    def process_data(self, data: list) -> list:
+        return self.process_payments(data)
+
+
+    @abstractmethod
+    def process_payments(self, data: list) -> tuple:
+        pass
+
+
+    def payments(self):
+        # Sort payments by date
+        return sorted(self.data, key=itemgetter('Datum'))
+
+
+    def blocked_payments(self):
+        return sorted(self.blocked_payments, key=itemgetter('Datum'))

knv_cli/processors/paypal.py

@@ -1,47 +1,56 @@
-# ~*~ coding=utf-8 ~*~
-
-# PAYPAL™
-# This module contains functions for processing 'Aktivitäten'
+# This module contains a class for processing & working with
+# 'Aktivitäten', as exported from PayPal™
 # See https://www.paypal.com/de/smarthelp/article/FAQ1007
 
 from .helpers import convert_number, convert_date
+from .payments import Payments
 
 
-# Processes 'Download*.CSV' files
-def process_payments(data) -> list:
-    codes = set()
-    payments = []
-    blocked_payments = []
+class Paypal(Payments):
+    # Props
+    identifier = 'Transaktion'
 
-    for item in data:
-        # Skip withdrawals
-        if item['Brutto'][:1] == '-':
-            continue
+    # CSV options
+    encoding='utf-8'
+    delimiter=','
 
-        # Assign identifier
-        code = item['Transaktionscode']
 
-        payment = {}
+    def process_payments(self, data) -> list:
+        '''
+        Processes 'Download*.CSV' files
+        '''
+        codes = set()
+        payments = []
 
-        payment['ID'] = 'nicht zugeordnet'
-        payment['Transaktion'] = code
-        payment['Datum'] = convert_date(item['Datum'])
-        payment['Vorgang'] = 'nicht zugeordnet'
-        payment['Name'] = item['Name']
-        payment['Email'] = item['Absender E-Mail-Adresse']
-        payment['Brutto'] = convert_number(item['Brutto'])
-        payment['Gebühr'] = convert_number(item['Gebühr'])
-        payment['Netto'] = convert_number(item['Netto'])
-        payment['Währung'] = item['Währung']
+        for item in data:
+            # Skip withdrawals
+            if item['Brutto'][:1] == '-':
+                continue
 
-        if code not in codes:
-            codes.add(code)
+            # Assign identifier
+            code = item['Transaktionscode']
 
-            # Sort out regular payments
-            if item['Typ'] == 'Allgemeine Zahlung':
-                blocked_payments.append(payment)
-                continue
+            payment = {}
+
+            payment['ID'] = 'nicht zugeordnet'
+            payment['Transaktion'] = code
+            payment['Datum'] = convert_date(item['Datum'])
+            payment['Vorgang'] = 'nicht zugeordnet'
+            payment['Name'] = item['Name']
+            payment['Email'] = item['Absender E-Mail-Adresse']
+            payment['Brutto'] = convert_number(item['Brutto'])
+            payment['Gebühr'] = convert_number(item['Gebühr'])
+            payment['Netto'] = convert_number(item['Netto'])
+            payment['Währung'] = item['Währung']
+
+            if code not in codes:
+                codes.add(code)
+
+                # Sort out regular payments
+                if item['Typ'] == 'Allgemeine Zahlung':
+                    self._blocked_payments.append(payment)
+                    continue
 
-            payments.append(payment)
+                payments.append(payment)
 
-    return (payments, blocked_payments)
+        return payments