From 8f7d3ff722781d988362416b4b80f899fbff42b6 Mon Sep 17 00:00:00 2001 From: Martin Folkers Date: Fri, 21 May 2021 17:29:24 +0200 Subject: [PATCH] Add payment gateway 'Volksbank' * Add global option 'VKN' * Add payment gateway 'Volksbank' --- README.md | 5 +- knv_cli/cli.py | 6 +- knv_cli/config.py | 2 +- knv_cli/database.py | 65 +++++++++-------- knv_cli/processors/payments.py | 4 ++ knv_cli/processors/paypal.py | 1 + knv_cli/processors/volksbank.py | 120 ++++++++++++++++++++++++++++++++ 7 files changed, 167 insertions(+), 36 deletions(-) create mode 100644 knv_cli/processors/volksbank.py diff --git a/README.md b/README.md index 03c4bc6..5d2be85 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ Adjusting most options to suit your needs is straightforward, global config is s ```ini [DEFAULT] +vkn = 12345 # 'Verkehrsnummer' verbose = off # Enable verbose mode [directories] @@ -37,7 +38,9 @@ export_dir = ./dist # generated spreadsheets & graphs [regexes] # (1) .. exported by PayPal™ payment_regex = Download*.CSV -# (2) .. exported by Shopkonfigurator +# (2) .. exported by Volksbank +volksbank_regex = Umsaetze_*_*.csv +# (3) .. exported by Shopkonfigurator order_regex = Orders_*.csv info_regex = OrdersInfo_*.csv invoice_regex = *_Invoices_TimeFrom*_TimeTo*.pdf diff --git a/knv_cli/cli.py b/knv_cli/cli.py index d34e762..ac29292 100755 --- a/knv_cli/cli.py +++ b/knv_cli/cli.py @@ -23,16 +23,20 @@ @click.group() @pass_config @click.option('-v', '--verbose', is_flag=True, default=None, help='Activate verbose mode.') +@click.option('--vkn', help='KNV "Verkehrsnummer".') @click.option('--data-dir', type=clickpath, help='Custom database directory.') @click.option('--import-dir', type=clickpath, help='Custom import directory.') @click.option('--export-dir', type=clickpath, help='Custom export directory.') -def cli(config, verbose, data_dir, import_dir, export_dir): +def cli(config, verbose, vkn, data_dir, import_dir, export_dir): """CLI utility for handling data exported from KNV & pcbis.de""" # Apply CLI options if verbose is not None: config.verbose = verbose + if vkn is not None: + config.VKN = vkn + if data_dir is not None: config.data_dir = data_dir diff --git a/knv_cli/config.py b/knv_cli/config.py index ea4bb0a..90ff51c 100644 --- a/knv_cli/config.py +++ b/knv_cli/config.py @@ -13,6 +13,7 @@ def __init__(self): # Provide sensible defaults config = SafeConfigParser() config['DEFAULT'] = { + 'vkn': '12345', 'verbose': 'off', } @@ -23,7 +24,6 @@ def __init__(self): } config['regexes'] = { - 'payment_regex': 'Download*.CSV', 'order_regex': 'Orders_*.csv', 'info_regex': 'OrdersInfo_*.csv', 'invoice_regex': '*_Invoices_TimeFrom*_TimeTo*.zip', diff --git a/knv_cli/database.py b/knv_cli/database.py index 9b00a75..1e27fc6 100644 --- a/knv_cli/database.py +++ b/knv_cli/database.py @@ -1,13 +1,14 @@ # ~*~ coding=utf-8 ~*~ -from os import remove -from os.path import basename, join +from os import getcwd, remove +from os.path import basename, isfile, join from operator import itemgetter from shutil import move from zipfile import ZipFile from .processors.paypal import Paypal +from .processors.volksbank import Volksbank from .processors.shopkonfigurator import Orders, Infos from .utils import load_json, dump_json from .utils import build_path, dedupe, group_data, invoice2number @@ -22,7 +23,11 @@ def __init__(self, config: dict) -> None: # GENERAL methods def flush(self) -> None: - files = build_path(self.config.payment_dir) + build_path(self.config.order_dir) + build_path(self.config.info_dir) + files = build_path(join(self.config.payment_dir, 'paypal')) + files += build_path(join(self.config.payment_dir, 'volksbank')) + files += build_path(self.config.payment_dir) + files += build_path(self.config.order_dir) + files += build_path(self.config.info_dir) for file in files: remove(file) @@ -31,20 +36,33 @@ def flush(self) -> None: # IMPORT methods def import_payments(self) -> None: - # Select payment files to be imported - import_files = build_path(self.config.import_dir, self.config.payment_regex) + # Prepare payment gateways + gateways = { + 'paypal': Paypal, + 'volksbank': Volksbank, + } - # Generate payment data by .. - # (1) .. extracting information from import files - handler = Paypal() - handler.load_csv(import_files) + for identifier, gateway in gateways.items(): + # Initialize payment gateway handler + handler = gateway() - # (2) .. merging with existing data - handler.load_json(build_path(self.config.payment_dir)) + # Apply VKN & blocklist CLI options + handler.VKN = self.config.vkn + handler.blocklist = self.config.blocklist + + # Select payment files to be imported + import_files = build_path(self.config.import_dir, handler.regex) + + # Generate payment data by .. + # (1) .. extracting information from import files + handler.load_csv(import_files) + + # (2) .. merging with existing data + handler.load_json(build_path(join(self.config.payment_dir, identifier))) - # Split payments per-month & export them - for code, data in group_data(handler.payments()).items(): - dump_json(data, join(self.config.payment_dir, code + '.json')) + # Split payments per-month & export them + for code, data in group_data(handler.payments()).items(): + dump_json(data, join(self.config.payment_dir, identifier, code + '.json')) def import_orders(self) -> None: @@ -57,7 +75,6 @@ def import_orders(self) -> None: handler.load_csv(import_files) # (2) .. merging with existing data - handler.load_json(build_path(self.config.order_dir)) # Split orders per-month & export them @@ -100,21 +117,3 @@ def import_invoices(self) -> None: except: raise Exception - - - def merge_data(self, data, import_data: list, identifier: str) -> list: - if data: - # Populate set with identifiers - codes = {item[identifier] for item in data} - - # Merge only data not already in database - for item in import_data: - if item[identifier] not in codes: - codes.add(item[identifier]) - data.append(item) - - # .. otherwise, start from scratch - else: - data = import_data - - return data diff --git a/knv_cli/processors/payments.py b/knv_cli/processors/payments.py index 8af7a93..3a664c3 100644 --- a/knv_cli/processors/payments.py +++ b/knv_cli/processors/payments.py @@ -7,6 +7,10 @@ class Payments(BaseClass): # Props _blocked_payments = [] + # Class-specific + VKN = None + blocklist = [] + def process_data(self, data: list) -> list: return self.process_payments(data) diff --git a/knv_cli/processors/paypal.py b/knv_cli/processors/paypal.py index e62535f..c97583c 100644 --- a/knv_cli/processors/paypal.py +++ b/knv_cli/processors/paypal.py @@ -9,6 +9,7 @@ class Paypal(Payments): # Props identifier = 'Transaktion' + regex = 'Download*.CSV' # CSV options encoding='utf-8' diff --git a/knv_cli/processors/volksbank.py b/knv_cli/processors/volksbank.py new file mode 100644 index 0000000..3d7c922 --- /dev/null +++ b/knv_cli/processors/volksbank.py @@ -0,0 +1,120 @@ +# This module contains a class for processing & working with +# 'Umsätze', as exported from Volksbank +# TODO: Add link about CSV export in Volksbank online banking + + +from re import findall, fullmatch, split +from string import punctuation + +from .helpers import convert_number, convert_date +from .payments import Payments + + +class Volksbank(Payments): + # Props + regex = 'Umsaetze_*_*.csv' + + # CSV options + skiprows=12 + + + def process_payments(self, data) -> list: + payments = [] + + for item in data: + # Skip all payments not related to customers + # (1) Skip withdrawals + if item[' '] == 'S': + continue + + # (2) Skip opening & closing balance + if item['Kundenreferenz'] == 'Anfangssaldo' or item['Kundenreferenz'] == 'Endsaldo': + continue + + reference = split('\n', item['Vorgang/Verwendungszweck']) + + # (3) Skip balancing debits + if reference[0].lower() == 'abschluss': + continue + + # Prepare reference string + reference = ''.join(reference[1:]) + + payment = {} + + payment['ID'] = 'nicht zugeordnet' + payment['Datum'] = convert_date(item['Buchungstag']) + payment['Vorgang'] = 'nicht zugeordnet' + payment['Name'] = item['Empfänger/Zahlungspflichtiger'] + payment['Betrag'] = convert_number(item['Umsatz']) + payment['Währung'] = item['Währung'] + payment['Rohdaten'] = item['Vorgang/Verwendungszweck'] + + # Skip blocked payers by .. + is_blocklisted = False + + # (1) .. iterating over them & blocklisting current payment based on .. + for entity in self.blocklist: + # (a) .. name of the payer + if entity.lower() in payment['Name'].lower(): + is_blocklisted = True + break + + # (b) .. payment reference + if entity.lower() in reference.lower(): + is_blocklisted = True + break + + # (3) .. skipping (but saving) blocklisted payments + if is_blocklisted: + self._blocked_payments.append(payment) + continue + + # Extract order identifiers + order_candidates = [] + + for line in reference.split(' '): + # Skip VKN + if line == self.VKN: + continue + + # Match strings preceeded by VKN & hypen (definite hit) + if line.split('-')[0] == self.VKN: + order_candidates.append(line) + + # Otherwise, try matching 6 random digits .. + else: + order_candidate = fullmatch(r"\d{6}", line) + + # .. so unless it's the VKN by itself .. + if order_candidate and order_candidate[0] != self.VKN: + # .. we got a hit + order_candidates.append(self.VKN + '-' + order_candidate[0]) + + if order_candidates: + payment['ID'] = order_candidates + + # Prepare reference string for further investigation + reference = reference.replace('/', ' ').translate(str.maketrans('', '', punctuation)) + + # Extract invoice numbers, matching .. + # (1) .. '20' + 9 random digits + # (2) .. '9' + 11 random digits + pattern = r"(R?[2][0]\d{9}|[9]\d{11})" + invoice_candidates = findall(pattern, reference) + + if not invoice_candidates: + # Remove whitespace & try again + reference = reference.replace(' ', '') + invoice_candidates = findall(pattern, reference) + + if invoice_candidates: + payment['Vorgang'] = invoice_candidates + + if payment['ID'] == 'nicht zugeordnet' and payment['Vorgang'] == 'nicht zugeordnet': + self._blocked_payments.append(payment) + continue + + payments.append(payment) + + return payments