Add payment gateway 'Volksbank'

* Add global option 'VKN'
* Add payment gateway 'Volksbank'

README.md

@@ -26,6 +26,7 @@ Adjusting most options to suit your needs is straightforward, global config is s
 
 ```ini
 [DEFAULT]
+vkn = 12345    # 'Verkehrsnummer'
 verbose = off  # Enable verbose mode
 
 [directories]
@@ -37,7 +38,9 @@ export_dir = ./dist                # generated spreadsheets & graphs
 [regexes]
 # (1) .. exported by PayPal™
 payment_regex = Download*.CSV
-# (2) .. exported by Shopkonfigurator
+# (2) .. exported by Volksbank
+volksbank_regex = Umsaetze_*_*.csv
+# (3) .. exported by Shopkonfigurator
 order_regex = Orders_*.csv
 info_regex = OrdersInfo_*.csv
 invoice_regex = *_Invoices_TimeFrom*_TimeTo*.pdf

knv_cli/cli.py

@@ -23,16 +23,20 @@
 @click.group()
 @pass_config
 @click.option('-v', '--verbose', is_flag=True, default=None, help='Activate verbose mode.')
+@click.option('--vkn', help='KNV "Verkehrsnummer".')
 @click.option('--data-dir', type=clickpath, help='Custom database directory.')
 @click.option('--import-dir', type=clickpath, help='Custom import directory.')
 @click.option('--export-dir', type=clickpath, help='Custom export directory.')
-def cli(config, verbose, data_dir, import_dir, export_dir):
+def cli(config, verbose, vkn, data_dir, import_dir, export_dir):
     """CLI utility for handling data exported from KNV & pcbis.de"""
 
     # Apply CLI options
     if verbose is not None:
         config.verbose = verbose
 
+    if vkn is not None:
+        config.VKN = vkn
+
     if data_dir is not None:
         config.data_dir = data_dir
 

knv_cli/config.py

@@ -13,6 +13,7 @@ def __init__(self):
         # Provide sensible defaults
         config = SafeConfigParser()
         config['DEFAULT'] = {
+            'vkn': '12345',
             'verbose': 'off',
         }
 
@@ -23,7 +24,6 @@ def __init__(self):
         }
 
         config['regexes'] = {
-            'payment_regex': 'Download*.CSV',
             'order_regex': 'Orders_*.csv',
             'info_regex': 'OrdersInfo_*.csv',
             'invoice_regex': '*_Invoices_TimeFrom*_TimeTo*.zip',

knv_cli/database.py

@@ -1,13 +1,14 @@
 # ~*~ coding=utf-8 ~*~
 
 
-from os import remove
-from os.path import basename, join
+from os import getcwd, remove
+from os.path import basename, isfile, join
 from operator import itemgetter
 from shutil import move
 from zipfile import ZipFile
 
 from .processors.paypal import Paypal
+from .processors.volksbank import Volksbank
 from .processors.shopkonfigurator import Orders, Infos
 from .utils import load_json, dump_json
 from .utils import build_path, dedupe, group_data, invoice2number
@@ -22,7 +23,11 @@ def __init__(self, config: dict) -> None:
     # GENERAL methods
 
     def flush(self) -> None:
-        files = build_path(self.config.payment_dir) + build_path(self.config.order_dir) + build_path(self.config.info_dir)
+        files = build_path(join(self.config.payment_dir, 'paypal'))
+        files += build_path(join(self.config.payment_dir, 'volksbank'))
+        files += build_path(self.config.payment_dir)
+        files += build_path(self.config.order_dir)
+        files += build_path(self.config.info_dir)
 
         for file in files:
             remove(file)
@@ -31,20 +36,33 @@ def flush(self) -> None:
     # IMPORT methods
 
     def import_payments(self) -> None:
-        # Select payment files to be imported
-        import_files = build_path(self.config.import_dir, self.config.payment_regex)
+        # Prepare payment gateways
+        gateways = {
+            'paypal': Paypal,
+            'volksbank': Volksbank,
+        }
 
-        # Generate payment data by ..
-        # (1) .. extracting information from import files
-        handler = Paypal()
-        handler.load_csv(import_files)
+        for identifier, gateway in gateways.items():
+            # Initialize payment gateway handler
+            handler = gateway()
 
-        # (2) .. merging with existing data
-        handler.load_json(build_path(self.config.payment_dir))
+            # Apply VKN & blocklist CLI options
+            handler.VKN = self.config.vkn
+            handler.blocklist = self.config.blocklist
+
+            # Select payment files to be imported
+            import_files = build_path(self.config.import_dir, handler.regex)
+
+            # Generate payment data by ..
+            # (1) .. extracting information from import files
+            handler.load_csv(import_files)
+
+            # (2) .. merging with existing data
+            handler.load_json(build_path(join(self.config.payment_dir, identifier)))
 
-        # Split payments per-month & export them
-        for code, data in group_data(handler.payments()).items():
-            dump_json(data, join(self.config.payment_dir, code + '.json'))
+            # Split payments per-month & export them
+            for code, data in group_data(handler.payments()).items():
+                dump_json(data, join(self.config.payment_dir, identifier, code + '.json'))
 
 
     def import_orders(self) -> None:
@@ -57,7 +75,6 @@ def import_orders(self) -> None:
         handler.load_csv(import_files)
 
         # (2) .. merging with existing data
-
         handler.load_json(build_path(self.config.order_dir))
 
         # Split orders per-month & export them
@@ -100,21 +117,3 @@ def import_invoices(self) -> None:
 
             except:
                 raise Exception
-
-
-    def merge_data(self, data, import_data: list, identifier: str) -> list:
-        if data:
-            # Populate set with identifiers
-            codes = {item[identifier] for item in data}
-
-            # Merge only data not already in database
-            for item in import_data:
-                if item[identifier] not in codes:
-                    codes.add(item[identifier])
-                    data.append(item)
-
-        # .. otherwise, start from scratch
-        else:
-            data = import_data
-
-        return data

knv_cli/processors/payments.py

@@ -7,6 +7,10 @@ class Payments(BaseClass):
     # Props
     _blocked_payments = []
 
+    # Class-specific
+    VKN = None
+    blocklist = []
+
 
     def process_data(self, data: list) -> list:
         return self.process_payments(data)

knv_cli/processors/paypal.py

@@ -9,6 +9,7 @@
 class Paypal(Payments):
     # Props
     identifier = 'Transaktion'
+    regex = 'Download*.CSV'
 
     # CSV options
     encoding='utf-8'

knv_cli/processors/volksbank.py

@@ -0,0 +1,120 @@
+# This module contains a class for processing & working with
+# 'Umsätze', as exported from Volksbank
+# TODO: Add link about CSV export in Volksbank online banking
+
+
+from re import findall, fullmatch, split
+from string import punctuation
+
+from .helpers import convert_number, convert_date
+from .payments import Payments
+
+
+class Volksbank(Payments):
+    # Props
+    regex = 'Umsaetze_*_*.csv'
+
+    # CSV options
+    skiprows=12
+
+
+    def process_payments(self, data) -> list:
+        payments = []
+
+        for item in data:
+            # Skip all payments not related to customers
+            # (1) Skip withdrawals
+            if item[' '] == 'S':
+                continue
+
+            # (2) Skip opening & closing balance
+            if item['Kundenreferenz'] == 'Anfangssaldo' or item['Kundenreferenz'] == 'Endsaldo':
+                continue
+
+            reference = split('\n', item['Vorgang/Verwendungszweck'])
+
+            # (3) Skip balancing debits
+            if reference[0].lower() == 'abschluss':
+                continue
+
+            # Prepare reference string
+            reference = ''.join(reference[1:])
+
+            payment = {}
+
+            payment['ID'] = 'nicht zugeordnet'
+            payment['Datum'] = convert_date(item['Buchungstag'])
+            payment['Vorgang'] = 'nicht zugeordnet'
+            payment['Name'] = item['Empfänger/Zahlungspflichtiger']
+            payment['Betrag'] = convert_number(item['Umsatz'])
+            payment['Währung'] = item['Währung']
+            payment['Rohdaten'] = item['Vorgang/Verwendungszweck']
+
+            # Skip blocked payers by ..
+            is_blocklisted = False
+
+            # (1) .. iterating over them & blocklisting current payment based on ..
+            for entity in self.blocklist:
+                # (a) .. name of the payer
+                if entity.lower() in payment['Name'].lower():
+                    is_blocklisted = True
+                    break
+
+                # (b) .. payment reference
+                if entity.lower() in reference.lower():
+                    is_blocklisted = True
+                    break
+
+            # (3) .. skipping (but saving) blocklisted payments
+            if is_blocklisted:
+                self._blocked_payments.append(payment)
+                continue
+
+            # Extract order identifiers
+            order_candidates = []
+
+            for line in reference.split(' '):
+                # Skip VKN
+                if line == self.VKN:
+                    continue
+
+                # Match strings preceeded by VKN & hypen (definite hit)
+                if line.split('-')[0] == self.VKN:
+                    order_candidates.append(line)
+
+                # Otherwise, try matching 6 random digits ..
+                else:
+                    order_candidate = fullmatch(r"\d{6}", line)
+
+                    # .. so unless it's the VKN by itself ..
+                    if order_candidate and order_candidate[0] != self.VKN:
+                        # .. we got a hit
+                        order_candidates.append(self.VKN + '-' + order_candidate[0])
+
+            if order_candidates:
+                payment['ID'] = order_candidates
+
+            # Prepare reference string for further investigation
+            reference = reference.replace('/', ' ').translate(str.maketrans('', '', punctuation))
+
+            # Extract invoice numbers, matching ..
+            # (1) .. '20' + 9 random digits
+            # (2) .. '9' + 11 random digits
+            pattern = r"(R?[2][0]\d{9}|[9]\d{11})"
+            invoice_candidates = findall(pattern, reference)
+
+            if not invoice_candidates:
+                # Remove whitespace & try again
+                reference = reference.replace(' ', '')
+                invoice_candidates = findall(pattern, reference)
+
+            if invoice_candidates:
+                payment['Vorgang'] = invoice_candidates
+
+            if payment['ID'] == 'nicht zugeordnet' and payment['Vorgang'] == 'nicht zugeordnet':
+                self._blocked_payments.append(payment)
+                continue
+
+            payments.append(payment)
+
+        return payments