From 22f49c440f9fb831c4ca14458f0a93ed584ca814 Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Fri, 6 Oct 2017 17:15:57 +0200
Subject: [PATCH 01/40] capture is not neccessary final

---
 payments/core.py                 | 6 +++---
 payments/cybersource/__init__.py | 2 +-
 payments/dummy/__init__.py       | 2 +-
 payments/models.py               | 4 ++--
 payments/paypal/__init__.py      | 4 ++--
 payments/stripe/__init__.py      | 2 +-
 payments/stripe/test_stripe.py   | 2 +-
 7 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/payments/core.py b/payments/core.py
index e81bfab6e..118e83cb0 100644
--- a/payments/core.py
+++ b/payments/core.py
@@ -26,8 +26,8 @@ def get_base_url():
     """
     Returns host url according to project settings. Protocol is chosen by
     checking PAYMENT_USES_SSL variable.
-    If PAYMENT_HOST is not specified, gets domain from Sites. 
-    Otherwise checks if it's callable and returns it's result. If it's not a 
+    If PAYMENT_HOST is not specified, gets domain from Sites.
+    Otherwise checks if it's callable and returns it's result. If it's not a
     callable treats it as domain.
     """
     protocol = 'https' if PAYMENT_USES_SSL else 'http'
@@ -92,7 +92,7 @@ def get_return_url(self, payment, extra_data=None):
             return url + '?' + qs
         return url
 
-    def capture(self, payment, amount=None):
+    def capture(self, payment, amount=None, final=True):
         raise NotImplementedError()
 
     def release(self, payment):
diff --git a/payments/cybersource/__init__.py b/payments/cybersource/__init__.py
index 6b119ebf2..40125796a 100644
--- a/payments/cybersource/__init__.py
+++ b/payments/cybersource/__init__.py
@@ -161,7 +161,7 @@ def charge(self, payment, data):
             self._set_proper_payment_status_from_reason_code(
                 payment, response.reasonCode)
 
-    def capture(self, payment, amount=None):
+    def capture(self, payment, amount=None, final=True):
         if amount is None:
             amount = payment.total
         params = self._prepare_capture(payment, amount=amount)
diff --git a/payments/dummy/__init__.py b/payments/dummy/__init__.py
index 0a693c701..1d1788be5 100644
--- a/payments/dummy/__init__.py
+++ b/payments/dummy/__init__.py
@@ -63,7 +63,7 @@ def process_data(self, payment, request):
             return HttpResponseRedirect(payment.get_success_url())
         return HttpResponseRedirect(payment.get_failure_url())
 
-    def capture(self, payment, amount=None):
+    def capture(self, payment, amount=None, final=True):
         payment.change_status(PaymentStatus.CONFIRMED)
         return amount
 
diff --git a/payments/models.py b/payments/models.py
index 2cbd8206a..e9f44ec25 100644
--- a/payments/models.py
+++ b/payments/models.py
@@ -133,12 +133,12 @@ def get_success_url(self):
     def get_process_url(self):
         return reverse('process_payment', kwargs={'token': self.token})
 
-    def capture(self, amount=None):
+    def capture(self, amount=None, final=True):
         if self.status != PaymentStatus.PREAUTH:
             raise ValueError(
                 'Only pre-authorized payments can be captured.')
         provider = provider_factory(self.variant)
-        amount = provider.capture(self, amount)
+        amount = provider.capture(self, amount, final)
         if amount:
             self.captured_amount = amount
             self.change_status(PaymentStatus.CONFIRMED)
diff --git a/payments/paypal/__init__.py b/payments/paypal/__init__.py
index fbeb2421e..8b82a49b2 100644
--- a/payments/paypal/__init__.py
+++ b/payments/paypal/__init__.py
@@ -252,13 +252,13 @@ def get_amount_data(self, payment, amount=None):
             'total': str(amount.quantize(
                 CENTS, rounding=ROUND_HALF_UP))}
 
-    def capture(self, payment, amount=None):
+    def capture(self, payment, amount=None, final=True):
         if amount is None:
             amount = payment.total
         amount_data = self.get_amount_data(payment, amount)
         capture_data = {
             'amount': amount_data,
-            'is_final_capture': True
+            'is_final_capture': final
         }
         links = self._get_links(payment)
         url = links['capture']['href']
diff --git a/payments/stripe/__init__.py b/payments/stripe/__init__.py
index f4a7fb5c5..a1b6faaa3 100644
--- a/payments/stripe/__init__.py
+++ b/payments/stripe/__init__.py
@@ -31,7 +31,7 @@ def get_form(self, payment, data=None):
             raise RedirectNeeded(payment.get_success_url())
         return form
 
-    def capture(self, payment, amount=None):
+    def capture(self, payment, amount=None, final=True):
         amount = int((amount or payment.total) * 100)
         charge = stripe.Charge.retrieve(payment.transaction_id)
         try:
diff --git a/payments/stripe/test_stripe.py b/payments/stripe/test_stripe.py
index dd08ccfde..9a67db297 100644
--- a/payments/stripe/test_stripe.py
+++ b/payments/stripe/test_stripe.py
@@ -35,7 +35,7 @@ def change_fraud_status(self, status, message='', commit=True):
         self.fraud_status = status
         self.fraud_message = message
 
-    def capture(self, amount=None):
+    def capture(self, amount=None, final=True):
         amount = amount or self.total
         self.captured_amount = amount
         self.change_status(PaymentStatus.CONFIRMED)

From 5ab5c681fdb7cd53de348d9f5b69f85e52ae4262 Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Fri, 6 Oct 2017 17:35:56 +0200
Subject: [PATCH 02/40] improve documentation, fix logic and add return values

---
 payments/core.py      |  3 +++
 payments/models.py    | 21 ++++++++++++++-------
 payments/test_core.py |  6 ++++--
 3 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/payments/core.py b/payments/core.py
index 118e83cb0..493a7c680 100644
--- a/payments/core.py
+++ b/payments/core.py
@@ -93,12 +93,15 @@ def get_return_url(self, payment, extra_data=None):
         return url
 
     def capture(self, payment, amount=None, final=True):
+        ''' Capture a fraction of the total amount of a payment. Return amount captured or None '''
         raise NotImplementedError()
 
     def release(self, payment):
+        ''' Annilates captured payment '''
         raise NotImplementedError()
 
     def refund(self, payment, amount=None):
+        ''' Refund payment, return amount which was refunded '''
         raise NotImplementedError()
 
 
diff --git a/payments/models.py b/payments/models.py
index e9f44ec25..559e50c36 100644
--- a/payments/models.py
+++ b/payments/models.py
@@ -134,16 +134,20 @@ def get_process_url(self):
         return reverse('process_payment', kwargs={'token': self.token})
 
     def capture(self, amount=None, final=True):
+        ''' Capture a fraction of the total amount of a payment. Return amount captured or None '''
         if self.status != PaymentStatus.PREAUTH:
             raise ValueError(
                 'Only pre-authorized payments can be captured.')
         provider = provider_factory(self.variant)
         amount = provider.capture(self, amount, final)
         if amount:
-            self.captured_amount = amount
-            self.change_status(PaymentStatus.CONFIRMED)
+            self.captured_amount -= amount
+            if final:
+                self.change_status(PaymentStatus.CONFIRMED)
+        return amount
 
     def release(self):
+        ''' Annilates captured payment '''
         if self.status != PaymentStatus.PREAUTH:
             raise ValueError(
                 'Only pre-authorized payments can be released.')
@@ -152,6 +156,7 @@ def release(self):
         self.change_status(PaymentStatus.REFUNDED)
 
     def refund(self, amount=None):
+        ''' Refund payment, return amount which was refunded '''
         if self.status != PaymentStatus.CONFIRMED:
             raise ValueError(
                 'Only charged payments can be refunded.')
@@ -159,12 +164,14 @@ def refund(self, amount=None):
             if amount > self.captured_amount:
                 raise ValueError(
                     'Refund amount can not be greater then captured amount')
-            provider = provider_factory(self.variant)
-            amount = provider.refund(self, amount)
+        provider = provider_factory(self.variant)
+        amount = provider.refund(self, amount)
+        if amount:
             self.captured_amount -= amount
-        if self.captured_amount == 0 and self.status != PaymentStatus.REFUNDED:
-            self.change_status(PaymentStatus.REFUNDED)
-        self.save()
+            if self.captured_amount == 0 and self.status != PaymentStatus.REFUNDED:
+                self.change_status(PaymentStatus.REFUNDED)
+            self.save()
+        return amount
 
     @property
     def attrs(self):
diff --git a/payments/test_core.py b/payments/test_core.py
index f6f3001fe..47a4bf672 100644
--- a/payments/test_core.py
+++ b/payments/test_core.py
@@ -49,7 +49,9 @@ def test_capture_preauth_successfully(self, mocked_capture_method):
             mocked_save_method.return_value = None
             mocked_capture_method.return_value = amount
 
-            payment = BasePayment(variant='default', status=PaymentStatus.PREAUTH)
+            captured_amount = Decimal('100')
+            payment = BasePayment(variant='default', captured_amount=captured_amount,
+                                  status=PaymentStatus.PREAUTH)
             payment.capture(amount)
 
             self.assertEqual(payment.status, PaymentStatus.CONFIRMED)
@@ -110,7 +112,7 @@ def test_refund_without_amount(self, mocked_refund_method):
             payment.refund(refund_amount)
             self.assertEqual(payment.status, status)
             self.assertEqual(payment.captured_amount, captured_amount)
-        self.assertEqual(mocked_refund_method.call_count, 0)
+        self.assertEqual(mocked_refund_method.call_count, 1)
 
     @patch('payments.dummy.DummyProvider.refund')
     def test_refund_partial_success(self, mocked_refund_method):

From 7a0f2e40c7a80d432acc115f8f56387a78918af0 Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Fri, 6 Oct 2017 21:39:01 +0200
Subject: [PATCH 03/40] split logic and model, replace address data with
 function

---
 payments/models.py     | 133 +++++++++++++++++++++++------------------
 payments/testcommon.py |  59 ++++++++++++++++++
 payments/utils.py      |  54 +++++++++++++++++
 3 files changed, 187 insertions(+), 59 deletions(-)
 create mode 100644 payments/testcommon.py

diff --git a/payments/models.py b/payments/models.py
index 2cbd8206a..f4266176f 100644
--- a/payments/models.py
+++ b/payments/models.py
@@ -8,6 +8,7 @@
 from django.utils.translation import ugettext_lazy as _
 
 from .core import provider_factory
+from .utils import add_prefixed_address, getter_prefixed_address
 from . import FraudStatus, PaymentStatus
 
 
@@ -32,51 +33,8 @@ def __setattr__(self, key, value):
         self._payment.extra_data = json.dumps(data)
 
 
-class BasePayment(models.Model):
-    '''
-    Represents a single transaction. Each instance has one or more PaymentItem.
-    '''
-    variant = models.CharField(max_length=255)
-    #: Transaction status
-    status = models.CharField(
-        max_length=10, choices=PaymentStatus.CHOICES,
-        default=PaymentStatus.WAITING)
-    fraud_status = models.CharField(
-        _('fraud check'), max_length=10, choices=FraudStatus.CHOICES,
-        default=FraudStatus.UNKNOWN)
-    fraud_message = models.TextField(blank=True, default='')
-    #: Creation date and time
-    created = models.DateTimeField(auto_now_add=True)
-    #: Date and time of last modification
-    modified = models.DateTimeField(auto_now=True)
-    #: Transaction ID (if applicable)
-    transaction_id = models.CharField(max_length=255, blank=True)
-    #: Currency code (may be provider-specific)
-    currency = models.CharField(max_length=10)
-    #: Total amount (gross)
-    total = models.DecimalField(max_digits=9, decimal_places=2, default='0.0')
-    delivery = models.DecimalField(
-        max_digits=9, decimal_places=2, default='0.0')
-    tax = models.DecimalField(max_digits=9, decimal_places=2, default='0.0')
-    description = models.TextField(blank=True, default='')
-    billing_first_name = models.CharField(max_length=256, blank=True)
-    billing_last_name = models.CharField(max_length=256, blank=True)
-    billing_address_1 = models.CharField(max_length=256, blank=True)
-    billing_address_2 = models.CharField(max_length=256, blank=True)
-    billing_city = models.CharField(max_length=256, blank=True)
-    billing_postcode = models.CharField(max_length=256, blank=True)
-    billing_country_code = models.CharField(max_length=2, blank=True)
-    billing_country_area = models.CharField(max_length=256, blank=True)
-    billing_email = models.EmailField(blank=True)
-    customer_ip_address = models.GenericIPAddressField(blank=True, null=True)
-    extra_data = models.TextField(blank=True, default='')
-    message = models.TextField(blank=True, default='')
-    token = models.CharField(max_length=36, blank=True, default='')
-    captured_amount = models.DecimalField(
-        max_digits=9, decimal_places=2, default='0.0')
-
-    class Meta:
-        abstract = True
+class BasePaymentLogic(object):
+    """ Logic of a Payment object, e.g. for tests """
 
     def change_status(self, status, message=''):
         '''
@@ -99,20 +57,8 @@ def change_fraud_status(self, status, message='', commit=True):
         if commit:
             self.save()
 
-    def save(self, **kwargs):
-        if not self.token:
-            tries = {}  # Stores a set of tried values
-            while True:
-                token = str(uuid4())
-                if token in tries and len(tries) >= 100:  # After 100 tries we are impliying an infinite loop
-                    raise SystemExit('A possible infinite loop was detected')
-                else:
-                    if not self.__class__._default_manager.filter(token=token).exists():
-                        self.token = token
-                        break
-                tries.add(token)
-
-        return super(BasePayment, self).save(**kwargs)
+    def __str__(self):
+        return self.variant
 
     def __unicode__(self):
         return self.variant
@@ -130,6 +76,14 @@ def get_failure_url(self):
     def get_success_url(self):
         raise NotImplementedError()
 
+    # needs to be implemented, see BasePaymentWithAddress for an example
+    def get_shipping_address(self):
+        raise NotImplementedError()
+
+    # needs to be implemented, see BasePaymentWithAddress for an example
+    def get_billing_address(self):
+        raise NotImplementedError()
+
     def get_process_url(self):
         return reverse('process_payment', kwargs={'token': self.token})
 
@@ -166,6 +120,67 @@ def refund(self, amount=None):
             self.change_status(PaymentStatus.REFUNDED)
         self.save()
 
+    def create_token(self):
+        if not self.token:
+            tries = {}  # Stores a set of tried values
+            while True:
+                token = str(uuid4())
+                if token in tries and len(tries) >= 100:  # After 100 tries we are impliying an infinite loop
+                    raise SystemExit('A possible infinite loop was detected')
+                else:
+                    if not self.__class__._default_manager.filter(token=token).exists():
+                        self.token = token
+                        break
+                tries.add(token)
+
     @property
     def attrs(self):
         return PaymentAttributeProxy(self)
+
+class BasePayment(models.Model, BasePaymentLogic):
+    '''
+    Represents a single transaction. Each instance has one or more PaymentItem.
+    '''
+    variant = models.CharField(max_length=255)
+    #: Transaction status
+    status = models.CharField(
+        max_length=10, choices=PaymentStatus.CHOICES,
+        default=PaymentStatus.WAITING)
+    fraud_status = models.CharField(
+        _('fraud check'), max_length=10, choices=FraudStatus.CHOICES,
+        default=FraudStatus.UNKNOWN)
+    fraud_message = models.TextField(blank=True, default='')
+    #: Creation date and time
+    created = models.DateTimeField(auto_now_add=True)
+    #: Date and time of last modification
+    modified = models.DateTimeField(auto_now=True)
+    #: Transaction ID (if applicable)
+    transaction_id = models.CharField(max_length=255, blank=True)
+    #: Currency code (may be provider-specific)
+    currency = models.CharField(max_length=10)
+    #: Total amount (gross)
+    total = models.DecimalField(max_digits=9, decimal_places=2, default='0.0')
+    delivery = models.DecimalField(
+        max_digits=9, decimal_places=2, default='0.0')
+    tax = models.DecimalField(max_digits=9, decimal_places=2, default='0.0')
+    description = models.TextField(blank=True, default='')
+    billing_email = models.EmailField(blank=True)
+    customer_ip_address = models.GenericIPAddressField(blank=True, null=True)
+    extra_data = models.TextField(blank=True, default='')
+    message = models.TextField(blank=True, default='')
+    token = models.CharField(max_length=36, blank=True, default='')
+    captured_amount = models.DecimalField(
+        max_digits=9, decimal_places=2, default='0.0')
+
+    class Meta:
+        abstract = True
+
+    def save(self, **kwargs):
+        self.create_token()
+        return super(BasePayment, self).save(**kwargs)
+
+@add_prefixed_address("billing")
+class BasePaymentWithAddress(BasePayment):
+    """ Has real billing address + shippingaddress alias on billing address (alias for backward compatibility) """
+    get_billing_address = getter_prefixed_address("billing")
+    get_shipping_address = get_billing_address
diff --git a/payments/testcommon.py b/payments/testcommon.py
new file mode 100644
index 000000000..37c9e75d4
--- /dev/null
+++ b/payments/testcommon.py
@@ -0,0 +1,59 @@
+
+from decimal import Decimal
+from .models import BasePaymentLogic
+from . import PaymentStatus, PurchasedItem
+from .utils import getter_prefixed_address
+from datetime import datetime
+
+def create_test_payment(**_kwargs):
+    class TestPayment(BasePaymentLogic):
+        def __init__(self, **kwargs):
+            self.__dict__.update(kwargs)
+        id = 523
+        pk = id
+        description = 'payment'
+        currency = 'USD'
+        delivery = Decimal(10.8)
+        status = PaymentStatus.WAITING
+        message = ""
+        tax = Decimal(10)
+        token = "undefined"
+        total = Decimal(100)
+        captured_amount = Decimal("0.0")
+        extra_data = ""
+        variant = "342"
+        transaction_id = ""
+        created = datetime.now()
+        modified = datetime.now()
+
+        billing_first_name = 'John'
+        billing_last_name = 'Smith'
+        billing_address_1 = 'JohnStreet 23'
+        billing_address_2 = ''
+        billing_city = 'Neches'
+        billing_postcode = "75779"
+        billing_country_code = "US"
+        billing_country_area = "Tennessee"
+        billing_email = "example@example.com"
+
+        customer_ip_address = "192.78.6.6"
+
+        def get_purchased_items(self):
+            return [
+                PurchasedItem(
+                    name='foo', quantity=Decimal('10'), price=Decimal('20'),
+                    currency='USD', sku='bar')]
+
+        def get_failure_url(self):
+            return 'http://cancel.com'
+
+        def get_process_url(self):
+            return 'http://example.com'
+
+        def get_success_url(self):
+            return 'http://success.com'
+
+        def save(self):
+            return self
+    TestPayment.__dict__.update(_kwargs)
+    return TestPayment
diff --git a/payments/utils.py b/payments/utils.py
index 2d78659e0..d14de2dcd 100644
--- a/payments/utils.py
+++ b/payments/utils.py
@@ -1,6 +1,8 @@
 from datetime import date
+import re
 
 from django.utils.translation import ugettext_lazy as _
+from django.db import models
 
 
 def get_month_choices():
@@ -12,3 +14,55 @@ def get_year_choices():
     year_choices = [(str(x), str(x)) for x in range(
         date.today().year, date.today().year + 15)]
     return [('', _('Year'))] + year_choices
+
+_extract_streetnr = re.compile(r"([0-9]+)\s*$")
+def extract_streetnr(address, fallback=None):
+    ret = _extract_streetnr.findall(address)
+    if ret:
+        return ret[0]
+    else:
+        return fallback
+
+def getter_prefixed_address(prefix):
+    """ create getter for prefixed address format """
+    first_name = "{}_first_name".format(prefix)
+    last_name = "{}_last_name".format(prefix)
+    address_1 = "{}_address_1".format(prefix)
+    address_2 = "{}_address_2".format(prefix)
+    city = "{}_city".format(prefix)
+    postcode = "{}_postcode".format(prefix)
+    country_code = "{}_country_code".format(prefix)
+    country_area = "{}_country_area".format(prefix)
+    def _get_address(self):
+        return {
+            "first_name": getattr(self, first_name, None),
+            "last_name": getattr(self, last_name, None),
+            "address_1": getattr(self, address_1, None),
+            "address_2": getattr(self, address_2, None),
+            "city": getattr(self, city, None),
+            "postcode": getattr(self, postcode, None),
+            "country_code": getattr(self, country_code, None),
+            "country_area": getattr(self, country_area, None)}
+    return _get_address
+
+def add_prefixed_address(prefix):
+    """ add address with prefix to class """
+    first_name = "{}_first_name".format(prefix)
+    last_name = "{}_last_name".format(prefix)
+    address_1 = "{}_address_1".format(prefix)
+    address_2 = "{}_address_2".format(prefix)
+    city = "{}_city".format(prefix)
+    postcode = "{}_postcode".format(prefix)
+    country_code = "{}_country_code".format(prefix)
+    country_area = "{}_country_area".format(prefix)
+    def class_to_customize(dclass):
+        setattr(dclass, first_name, models.CharField(max_length=256, blank=True))
+        setattr(dclass, last_name, models.CharField(max_length=256, blank=True))
+        setattr(dclass, address_1, models.CharField(max_length=256, blank=True))
+        setattr(dclass, address_2, models.CharField(max_length=256, blank=True))
+        setattr(dclass, city, models.CharField(max_length=256, blank=True))
+        setattr(dclass, postcode, models.CharField(max_length=256, blank=True))
+        setattr(dclass, country_code, models.CharField(max_length=2, blank=True))
+        setattr(dclass, country_area, models.CharField(max_length=256, blank=True))
+        return dclass
+    return class_to_customize

From 0dc734e2e5bc629d9c07e37af666bc508ee0f2e1 Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Fri, 6 Oct 2017 23:05:23 +0200
Subject: [PATCH 04/40] split address into get_billing_address,
 get_shipping_address functions (allows much more freedom) and adjust all
 providers for the change use create_test_payment for tests, TestPayment is
 now a Mock use unittest mocks when python3

---
 payments/authorizenet/__init__.py          | 18 ++++-----
 payments/authorizenet/test_authorizenet.py | 27 +++----------
 payments/braintree/forms.py                | 22 ++++++-----
 payments/braintree/test_braintree.py       | 27 +++----------
 payments/coinbase/test_coinbase.py         | 35 +++-------------
 payments/cybersource/__init__.py           | 17 ++++----
 payments/cybersource/test_cybersource.py   | 42 +++++---------------
 payments/dotpay/test_dotpay.py             | 25 +++---------
 payments/dummy/test_dummy.py               | 24 +----------
 payments/models.py                         |  3 ++
 payments/paypal/test_paypal.py             | 46 +++++-----------------
 payments/sagepay/__init__.py               | 37 +++++++++--------
 payments/sagepay/test_sagepay.py           | 28 +++----------
 payments/sofort/__init__.py                | 19 ++++-----
 payments/sofort/test_sofort.py             | 30 ++++----------
 payments/stripe/forms.py                   | 18 +++++----
 payments/stripe/test_stripe.py             | 45 +++------------------
 payments/test_core.py                      |  5 ++-
 payments/testcommon.py                     | 34 ++++++++++++----
 payments/wallet/test_wallet.py             | 38 ++++--------------
 setup.py                                   | 13 +++---
 21 files changed, 182 insertions(+), 371 deletions(-)

diff --git a/payments/authorizenet/__init__.py b/payments/authorizenet/__init__.py
index 6a94cfde7..50dcdb7ef 100644
--- a/payments/authorizenet/__init__.py
+++ b/payments/authorizenet/__init__.py
@@ -23,19 +23,19 @@ def __init__(self, login_id, transaction_key,
                 'Authorize.Net does not support pre-authorization.')
 
     def get_transactions_data(self, payment):
-        data = {
+        billing = payment.get_billing_address()
+        return {
             'x_amount': payment.total,
             'x_currency_code': payment.currency,
             'x_description': payment.description,
-            'x_first_name': payment.billing_first_name,
-            'x_last_name': payment.billing_last_name,
-            'x_address': "%s, %s" % (payment.billing_address_1,
-                                     payment.billing_address_2),
-            'x_city': payment.billing_city,
-            'x_zip': payment.billing_postcode,
-            'x_country': payment.billing_country_area
+            'x_first_name': billing["first_name"],
+            'x_last_name': billing["last_name"],
+            'x_address': "%s, %s" % (billing["address_1"],
+                                     billing["address_2"]),
+            'x_city': billing["city"],
+            'x_zip': billing["postcode"],
+            'x_country': billing["country_area"]
         }
-        return data
 
     def get_product_data(self, payment, extra_data=None):
         data = self.get_transactions_data(payment)
diff --git a/payments/authorizenet/test_authorizenet.py b/payments/authorizenet/test_authorizenet.py
index d70acd1dd..deb711a63 100644
--- a/payments/authorizenet/test_authorizenet.py
+++ b/payments/authorizenet/test_authorizenet.py
@@ -1,9 +1,13 @@
 from __future__ import unicode_literals
 from unittest import TestCase
-from mock import patch, MagicMock, Mock
+try:
+    from unittest.mock import patch, MagicMock
+except ImportError:
+    from mock import patch, MagicMock
 
 from . import AuthorizeNetProvider
 from .. import PaymentStatus, RedirectNeeded
+from ..testcommon import create_test_payment
 
 
 LOGIN_ID = 'abcd1234'
@@ -18,26 +22,7 @@
 STATUS_CONFIRMED = '1'
 
 
-class Payment(Mock):
-    id = 1
-    variant = 'authorizenet'
-    currency = 'USD'
-    total = 100
-    status = PaymentStatus.WAITING
-    transaction_id = None
-    captured_amount = 0
-
-    def get_process_url(self):
-        return 'http://example.com'
-
-    def get_failure_url(self):
-        return 'http://cancel.com'
-
-    def get_success_url(self):
-        return 'http://success.com'
-
-    def change_status(self, status):
-        self.status = status
+Payment = create_test_payment()
 
 
 
diff --git a/payments/braintree/forms.py b/payments/braintree/forms.py
index e52bef2d0..9d64f6219 100644
--- a/payments/braintree/forms.py
+++ b/payments/braintree/forms.py
@@ -41,20 +41,22 @@ def get_credit_card_clean_data(self):
                 'expiration_year': self.cleaned_data.get('expiration').year}
 
     def get_billing_data(self):
+        billing = self.payment.get_billing_address()
         return {
-            'first_name': self.payment.billing_first_name,
-            'last_name': self.payment.billing_last_name,
-            'street_address': self.payment.billing_address_1,
-            'extended_address': self.payment.billing_address_2,
-            'locality': self.payment.billing_city,
-            'region': self.payment.billing_country_area,
-            'postal_code': self.payment.billing_postcode,
-            'country_code_alpha2': self.payment.billing_country_code}
+            'first_name': billing["first_name"],
+            'last_name': billing["last_name"],
+            'street_address': billing["address_1"],
+            'extended_address': billing["address_2"],
+            'locality': billing["city"],
+            'region': billing["country_area"],
+            'postal_code': billing["postcode"],
+            'country_code_alpha2': billing["country_code"]}
 
     def get_customer_data(self):
+        billing = self.payment.get_billing_address()
         return {
-            'first_name': self.payment.billing_first_name,
-            'last_name': self.payment.billing_last_name}
+            'first_name': billing["first_name"],
+            'last_name': billing["last_name"]}
 
     def save(self):
         braintree.Transaction.submit_for_settlement(self.transaction_id)
diff --git a/payments/braintree/test_braintree.py b/payments/braintree/test_braintree.py
index 00ad7d00d..71e4b368e 100644
--- a/payments/braintree/test_braintree.py
+++ b/payments/braintree/test_braintree.py
@@ -1,9 +1,13 @@
 from __future__ import unicode_literals
 from unittest import TestCase
-from mock import patch, MagicMock, Mock
+try:
+    from unittest.mock import patch, MagicMock
+except ImportError:
+    from mock import patch, MagicMock
 
 from . import BraintreeProvider
 from .. import PaymentStatus, RedirectNeeded
+from ..testcommon import create_test_payment
 
 
 MERCHANT_ID = 'test11'
@@ -18,26 +22,7 @@
     'cvv2': '1234'}
 
 
-class Payment(Mock):
-    id = 1
-    variant = 'braintree'
-    currency = 'USD'
-    total = 100
-    status = PaymentStatus.WAITING
-    transaction_id = None
-    captured_amount = 0
-
-    def get_process_url(self):
-        return 'http://example.com'
-
-    def get_failure_url(self):
-        return 'http://cancel.com'
-
-    def get_success_url(self):
-        return 'http://success.com'
-
-    def change_status(self, status):
-        self.status = status
+Payment = create_test_payment()
 
 
 class TestBraintreeProvider(TestCase):
diff --git a/payments/coinbase/test_coinbase.py b/payments/coinbase/test_coinbase.py
index e694a525b..5a8d11240 100644
--- a/payments/coinbase/test_coinbase.py
+++ b/payments/coinbase/test_coinbase.py
@@ -4,12 +4,16 @@
 import json
 from decimal import Decimal
 from unittest import TestCase
+try:
+    from unittest.mock import patch, MagicMock
+except ImportError:
+    from mock import patch, MagicMock
 
 from django.http import HttpResponse, HttpResponseForbidden
-from mock import MagicMock, patch
 
 from .. import PaymentStatus
 from . import CoinbaseProvider
+from ..testcommon import create_test_payment
 
 PAYMENT_TOKEN = '5a4dae68-2715-4b1e-8bb2-2c2dbe9255f6'
 KEY = 'abc123'
@@ -23,34 +27,7 @@
             PAYMENT_TOKEN, KEY)).encode('utf-8')).hexdigest()}}
 
 
-class Payment(object):
-
-    id = 1
-    description = 'payment'
-    currency = 'BTC'
-    total = Decimal(100)
-    status = PaymentStatus.WAITING
-    token = PAYMENT_TOKEN
-    variant = VARIANT
-
-    def change_status(self, status):
-        self.status = status
-
-    def get_failure_url(self):
-        return 'http://cancel.com'
-
-    def get_process_url(self):
-        return 'http://example.com'
-
-    def get_purchased_items(self):
-        return []
-
-    def save(self):
-        return self
-
-    def get_success_url(self):
-        return 'http://success.com'
-
+Payment = create_test_payment(variant=VARIANT, token=PAYMENT_TOKEN, description='payment', currency='BTC', total=Decimal(100))
 
 class TestCoinbaseProvider(TestCase):
 
diff --git a/payments/cybersource/__init__.py b/payments/cybersource/__init__.py
index 6b119ebf2..ae2da808c 100644
--- a/payments/cybersource/__init__.py
+++ b/payments/cybersource/__init__.py
@@ -366,15 +366,16 @@ def _prepare_card_data(self, data):
         return card
 
     def _prepare_billing_data(self, payment):
+        _billing_address = payment.get_billing_address()
         billing = self.client.factory.create('data:BillTo')
-        billing.firstName = payment.billing_first_name
-        billing.lastName = payment.billing_last_name
-        billing.street1 = payment.billing_address_1
-        billing.street2 = payment.billing_address_2
-        billing.city = payment.billing_city
-        billing.postalCode = payment.billing_postcode
-        billing.country = payment.billing_country_code
-        billing.state = payment.billing_country_area
+        billing.firstName = _billing_address["first_name"]
+        billing.lastName = _billing_address["last_name"]
+        billing.street1 = _billing_address["address_1"]
+        billing.street2 = _billing_address["address_2"]
+        billing.city = _billing_address["city"]
+        billing.postalCode = _billing_address["postcode"]
+        billing.country = _billing_address["country_code"]
+        billing.state = _billing_address["country_area"]
         billing.email = payment.billing_email
         billing.ipAddress = payment.customer_ip_address
         return billing
diff --git a/payments/cybersource/test_cybersource.py b/payments/cybersource/test_cybersource.py
index 9ff21d532..3264b9ad4 100644
--- a/payments/cybersource/test_cybersource.py
+++ b/payments/cybersource/test_cybersource.py
@@ -2,12 +2,17 @@
 from decimal import Decimal
 from unittest import TestCase
 from django.core import signing
-from mock import patch, MagicMock, Mock
+try:
+    from unittest.mock import patch, MagicMock
+except ImportError:
+    from mock import patch, MagicMock
 
 from . import CyberSourceProvider, AUTHENTICATE_REQUIRED, ACCEPTED, \
     TRANSACTION_SETTLED
 from .. import PaymentStatus, PurchasedItem, RedirectNeeded
 
+from ..testcommon import create_test_payment
+
 MERCHANT_ID = 'abcd1234'
 PASSWORD = '1234abdd1234abcd'
 ORG_ID = 'abc'
@@ -20,40 +25,13 @@
     'cvv2': '1234',
     'fingerprint': 'abcd1234'}
 
-
-class Payment(Mock):
-    id = 1
-    variant = 'cybersource'
-    currency = 'USD'
-    total = 100
-    status = PaymentStatus.WAITING
-    transaction_id = None
-    captured_amount = 0
-    message = ''
-
+_Payment = create_test_payment()
+class Payment(_Payment):
+    # MagicMock is not serializable so overwrite attrs Proxy
     class attrs(object):
         fingerprint_session_id = 'fake'
         merchant_defined_data = {}
-
-    def get_process_url(self):
-        return 'http://example.com'
-
-    def get_failure_url(self):
-        return 'http://cancel.com'
-
-    def get_success_url(self):
-        return 'http://success.com'
-
-    def change_status(self, status, message=''):
-        self.status = status
-        self.message = message
-
-    def get_purchased_items(self):
-        return [
-            PurchasedItem(
-                name='foo', quantity=Decimal('10'), price=Decimal('20'),
-                currency='USD', sku='bar')]
-
+        capture = {}
 
 class TestCybersourceProvider(TestCase):
 
diff --git a/payments/dotpay/test_dotpay.py b/payments/dotpay/test_dotpay.py
index 73d4b44e2..b70dc3323 100644
--- a/payments/dotpay/test_dotpay.py
+++ b/payments/dotpay/test_dotpay.py
@@ -3,11 +3,15 @@
 from unittest import TestCase
 
 from django.http import HttpResponse, HttpResponseForbidden
-from mock import MagicMock, Mock
+try:
+    from unittest.mock import MagicMock
+except ImportError:
+    from mock import MagicMock
 
 from .. import PaymentStatus
 from .forms import ACCEPTED, REJECTED
 from . import DotpayProvider
+from ..testcommon import create_test_payment
 
 VARIANT = 'dotpay'
 PIN = '123'
@@ -45,24 +49,7 @@ def get_post_with_md5(post):
     return post
 
 
-class Payment(Mock):
-    id = 1
-    variant = VARIANT
-    currency = 'USD'
-    total = 100
-    status = PaymentStatus.WAITING
-
-    def get_process_url(self):
-        return 'http://example.com'
-
-    def get_failure_url(self):
-        return 'http://cancel.com'
-
-    def get_success_url(self):
-        return 'http://success.com'
-
-    def change_status(self, status):
-        self.status = status
+Payment = create_test_payment(variant=VARIANT, id=1, currency='USD')
 
 
 class TestDotpayProvider(TestCase):
diff --git a/payments/dummy/test_dummy.py b/payments/dummy/test_dummy.py
index e61e6262a..3f6e8649e 100644
--- a/payments/dummy/test_dummy.py
+++ b/payments/dummy/test_dummy.py
@@ -12,32 +12,12 @@
 
 from . import DummyProvider
 from .. import FraudStatus, PaymentError, PaymentStatus, RedirectNeeded
+from ..testcommon import create_test_payment
 
 VARIANT = 'dummy-3ds'
 
 
-class Payment(object):
-    id = 1
-    variant = VARIANT
-    currency = 'USD'
-    total = 100
-    status = PaymentStatus.WAITING
-    fraud_status = ''
-
-    def get_process_url(self):
-        return 'http://example.com'
-
-    def get_failure_url(self):
-        return 'http://cancel.com'
-
-    def get_success_url(self):
-        return 'http://success.com'
-
-    def change_status(self, new_status):
-        self.status = new_status
-
-    def change_fraud_status(self, fraud_status):
-        self.fraud_status = fraud_status
+Payment = create_test_payment(variant=VARIANT)
 
 
 class TestDummy3DSProvider(TestCase):
diff --git a/payments/models.py b/payments/models.py
index f4266176f..77ebdb211 100644
--- a/payments/models.py
+++ b/payments/models.py
@@ -184,3 +184,6 @@ class BasePaymentWithAddress(BasePayment):
     """ Has real billing address + shippingaddress alias on billing address (alias for backward compatibility) """
     get_billing_address = getter_prefixed_address("billing")
     get_shipping_address = get_billing_address
+
+    class Meta:
+        abstract = True
diff --git a/payments/paypal/test_paypal.py b/payments/paypal/test_paypal.py
index d2ac278f8..441c51dcc 100644
--- a/payments/paypal/test_paypal.py
+++ b/payments/paypal/test_paypal.py
@@ -2,18 +2,22 @@
 import json
 from decimal import Decimal
 from unittest import TestCase
-from mock import patch, MagicMock, Mock
+try:
+    from unittest.mock import patch, MagicMock
+except ImportError:
+    from mock import patch, MagicMock
 
 from django.utils import timezone
 from requests import HTTPError
 
 from . import PaypalProvider, PaypalCardProvider
-from .. import PurchasedItem, RedirectNeeded, PaymentError, PaymentStatus
+from .. import RedirectNeeded, PaymentError, PaymentStatus
+from ..testcommon import create_test_payment
 
 CLIENT_ID = 'abc123'
 PAYMENT_TOKEN = '5a4dae68-2715-4b1e-8bb2-2c2dbe9255f6'
 SECRET = '123abc'
-VARIANT = 'wallet'
+VARIANT = 'paypal'
 
 PROCESS_DATA = {
     'name': 'John Doe',
@@ -22,46 +26,14 @@
     'expiration_1': '2020',
     'cvv2': '1234'}
 
-
-class Payment(Mock):
-    id = 1
-    description = 'payment'
-    currency = 'USD'
-    delivery = Decimal(10)
-    status = PaymentStatus.WAITING
-    tax = Decimal(10)
-    token = PAYMENT_TOKEN
-    total = Decimal(100)
-    captured_amount = Decimal(0)
-    variant = VARIANT
-    transaction_id = None
-    message = ''
-    extra_data = json.dumps({'links': {
+Payment = create_test_payment(variant=VARIANT, token=PAYMENT_TOKEN)
+Payment.extra_data = json.dumps({'links': {
         'approval_url': None,
         'capture': {'href': 'http://capture.com'},
         'refund': {'href': 'http://refund.com'},
         'execute': {'href': 'http://execute.com'}
     }})
 
-    def change_status(self, status, message=''):
-        self.status = status
-        self.message = message
-
-    def get_failure_url(self):
-        return 'http://cancel.com'
-
-    def get_process_url(self):
-        return 'http://example.com'
-
-    def get_purchased_items(self):
-        return [
-            PurchasedItem(
-                name='foo', quantity=Decimal('10'), price=Decimal('20'),
-                currency='USD', sku='bar')]
-
-    def get_success_url(self):
-        return 'http://success.com'
-
 
 class TestPaypalProvider(TestCase):
 
diff --git a/payments/sagepay/__init__.py b/payments/sagepay/__init__.py
index d2dcc1ca3..d8f9a90ea 100644
--- a/payments/sagepay/__init__.py
+++ b/payments/sagepay/__init__.py
@@ -60,6 +60,8 @@ def aes_dec(self, data):
     def get_hidden_fields(self, payment):
         payment.save()
         return_url = self.get_return_url(payment)
+        _billing_address = payment.get_billing_address()
+        _shipping_address = payment.get_billing_address()
         data = {
             'VendorTxCode': payment.pk,
             'Amount': "%.2f" % (payment.total,),
@@ -67,23 +69,24 @@ def get_hidden_fields(self, payment):
             'Description': "Payment #%s" % (payment.pk,),
             'SuccessURL': return_url,
             'FailureURL': return_url,
-            'BillingSurname': payment.billing_last_name,
-            'BillingFirstnames': payment.billing_first_name,
-            'BillingAddress1': payment.billing_address_1,
-            'BillingAddress2': payment.billing_address_2,
-            'BillingCity': payment.billing_city,
-            'BillingPostCode': payment.billing_postcode,
-            'BillingCountry': payment.billing_country_code,
-            'DeliverySurname': payment.billing_last_name,
-            'DeliveryFirstnames': payment.billing_first_name,
-            'DeliveryAddress1': payment.billing_address_1,
-            'DeliveryAddress2': payment.billing_address_2,
-            'DeliveryCity': payment.billing_city,
-            'DeliveryPostCode': payment.billing_postcode,
-            'DeliveryCountry': payment.billing_country_code}
-        if payment.billing_country_code == 'US':
-            data['BillingState'] = payment.billing_country_area
-            data['DeliveryState'] = payment.billing_country_area
+            'BillingSurname': _billing_address["last_name"],
+            'BillingFirstnames': _billing_address["first_name"],
+            'BillingAddress1': _billing_address["address_1"],
+            'BillingAddress2': _billing_address["address_2"],
+            'BillingCity': _billing_address["city"],
+            'BillingPostCode': _billing_address["postcode"],
+            'BillingCountry': _billing_address["country_code"],
+            'DeliverySurname': _shipping_address["last_name"],
+            'DeliveryFirstnames': _shipping_address["first_name"],
+            'DeliveryAddress1': _shipping_address["address_1"],
+            'DeliveryAddress2': _shipping_address["address_2"],
+            'DeliveryCity': _shipping_address["city"],
+            'DeliveryPostCode': _shipping_address["postcode"],
+            'DeliveryCountry': _shipping_address["country_code"]}
+        if _billing_address["country_code"] == 'US':
+            data['BillingState'] = _billing_address["country_area"]
+        if _shipping_address["country_code"] == 'US':
+            data['DeliveryState'] = _shipping_address["country_area"]
         udata = "&".join("%s=%s" % kv for kv in data.items())
         crypt = self.aes_enc(udata)
         return {'VPSProtocol': self._version, 'TxType': 'PAYMENT',
diff --git a/payments/sagepay/test_sagepay.py b/payments/sagepay/test_sagepay.py
index 5686dfa70..6cc96722e 100644
--- a/payments/sagepay/test_sagepay.py
+++ b/payments/sagepay/test_sagepay.py
@@ -1,36 +1,20 @@
 from __future__ import unicode_literals
 from unittest import TestCase
-from mock import patch, MagicMock, Mock
+try:
+    from unittest.mock import patch, MagicMock
+except ImportError:
+    from mock import patch, MagicMock
 
 from . import SagepayProvider
 from .. import PaymentStatus
+from ..testcommon import create_test_payment
 
 
 VENDOR = 'abcd1234'
 ENCRYPTION_KEY = '1234abdd1234abcd'
 
 
-class Payment(Mock):
-    id = 1
-    variant = 'sagepay'
-    currency = 'USD'
-    total = 100
-    status = PaymentStatus.WAITING
-    transaction_id = None
-    captured_amount = 0
-    billing_first_name = 'John'
-
-    def get_process_url(self):
-        return 'http://example.com'
-
-    def get_failure_url(self):
-        return 'http://cancel.com'
-
-    def get_success_url(self):
-        return 'http://success.com'
-
-    def change_status(self, status):
-        self.status = status
+Payment = create_test_payment()
 
 
 class TestSagepayProvider(TestCase):
diff --git a/payments/sofort/__init__.py b/payments/sofort/__init__.py
index 4035225c0..9d0dacee3 100644
--- a/payments/sofort/__init__.py
+++ b/payments/sofort/__init__.py
@@ -12,7 +12,7 @@
 
 
 class SofortProvider(BasicProvider):
-    
+
     def __init__(self, *args, **kwargs):
         self.secret = kwargs.pop('key')
         self.client_id = kwargs.pop('id')
@@ -20,7 +20,7 @@ def __init__(self, *args, **kwargs):
         self.endpoint = kwargs.pop(
              'endpoint', 'https://api.sofort.com/api/xml')
         super(SofortProvider, self).__init__(*args, **kwargs)
-    
+
     def post_request(self, xml_request):
         response = requests.post(
             self.endpoint,
@@ -29,7 +29,7 @@ def post_request(self, xml_request):
             auth=(self.client_id, self.secret))
         doc = xmltodict.parse(response.content)
         return doc, response
-        
+
     def get_form(self, payment, data=None):
         if not payment.id:
             payment.save()
@@ -75,12 +75,13 @@ def process_data(self, payment, request):
             payment.captured_amount = payment.total
             payment.change_status(PaymentStatus.CONFIRMED)
             payment.extra_data = json.dumps(doc)
-            sender_data = doc['transactions']['transaction_details']['sender']
-            holder_data = sender_data['holder']
-            first_name, last_name = holder_data.rsplit(' ', 1)
-            payment.billing_first_name = first_name
-            payment.billing_last_name = last_name
-            payment.billing_country_code = sender_data['country_code']
+            # overwriting names should not be possible
+            #sender_data = doc['transactions']['transaction_details']['sender']
+            #holder_data = sender_data['holder']
+            #first_name, last_name = holder_data.rsplit(' ', 1)
+            #payment.billing_first_name = first_name
+            #payment.billing_last_name = last_name
+            #payment.billing_country_code = sender_data['country_code']
             payment.save()
             return redirect(payment.get_success_url())
 
diff --git a/payments/sofort/test_sofort.py b/payments/sofort/test_sofort.py
index 4096cf577..1fdda67d8 100644
--- a/payments/sofort/test_sofort.py
+++ b/payments/sofort/test_sofort.py
@@ -1,37 +1,21 @@
 from __future__ import unicode_literals
-from unittest import TestCase
-from mock import patch, MagicMock, Mock
 import json
+from unittest import TestCase
+try:
+    from unittest.mock import patch, MagicMock
+except ImportError:
+    from mock import patch, MagicMock
 
 from . import SofortProvider
 from .. import PaymentStatus, RedirectNeeded
+from ..testcommon import create_test_payment
 
 SECRET = 'abcd1234'
 CLIENT_ID = '1234'
 PROJECT_ID = 'abcd'
 
 
-class Payment(Mock):
-    id = 1
-    variant = 'sagepay'
-    currency = 'USD'
-    total = 100
-    status = PaymentStatus.WAITING
-    transaction_id = None
-    captured_amount = 0
-    billing_first_name = 'John'
-
-    def get_process_url(self):
-        return 'http://example.com'
-
-    def get_failure_url(self):
-        return 'http://cancel.com'
-
-    def get_success_url(self):
-        return 'http://success.com'
-
-    def change_status(self, status):
-        self.status = status
+Payment = create_test_payment()
 
 
 class TestSofortProvider(TestCase):
diff --git a/payments/stripe/forms.py b/payments/stripe/forms.py
index 0c83657be..32d622759 100644
--- a/payments/stripe/forms.py
+++ b/payments/stripe/forms.py
@@ -29,6 +29,7 @@ def clean(self):
         if not self.errors:
             if not self.payment.transaction_id:
                 stripe.api_key = self.provider.secret_key
+                _billing_address = self.payment.get_billing_address()
                 try:
                     self.charge = stripe.Charge.create(
                         capture=False,
@@ -36,8 +37,8 @@ def clean(self):
                         currency=self.payment.currency,
                         card=data['stripeToken'],
                         description='%s %s' % (
-                            self.payment.billing_last_name,
-                            self.payment.billing_first_name))
+                            _billing_address["last_name"],
+                            _billing_address["first_name"]))
                 except stripe.CardError as e:
                     # Making sure we retrieve the charge
                     charge_id = e.json_body['error']['charge']
@@ -81,14 +82,15 @@ class PaymentForm(StripeFormMixin, CreditCardPaymentFormWithName):
 
     def __init__(self, *args, **kwargs):
         super(PaymentForm, self).__init__(*args, **kwargs)
+        _billing_address = self.payment.get_billing_address()
         stripe_attrs = self.fields['stripeToken'].widget.attrs
         stripe_attrs['data-publishable-key'] = self.provider.public_key
-        stripe_attrs['data-address-line1'] = self.payment.billing_address_1
-        stripe_attrs['data-address-line2'] = self.payment.billing_address_2
-        stripe_attrs['data-address-city'] = self.payment.billing_city
-        stripe_attrs['data-address-state'] = self.payment.billing_country_area
-        stripe_attrs['data-address-zip'] = self.payment.billing_postcode
-        stripe_attrs['data-address-country'] = self.payment.billing_country_code
+        stripe_attrs['data-address-line1'] = _billing_address["address_1"]
+        stripe_attrs['data-address-line2'] = _billing_address["address_2"]
+        stripe_attrs['data-address-city'] = _billing_address["city"]
+        stripe_attrs['data-address-state'] = _billing_address["country_area"]
+        stripe_attrs['data-address-zip'] = _billing_address["postcode"]
+        stripe_attrs['data-address-country'] = _billing_address["country_code"]
         widget_map = {
             'name': SensitiveTextInput(
                 attrs={'autocomplete': 'cc-name', 'required': 'required'}),
diff --git a/payments/stripe/test_stripe.py b/payments/stripe/test_stripe.py
index dd08ccfde..eca8f2058 100644
--- a/payments/stripe/test_stripe.py
+++ b/payments/stripe/test_stripe.py
@@ -2,56 +2,23 @@
 from __future__ import unicode_literals
 from contextlib import contextmanager
 
-from mock import patch, Mock
 from unittest import TestCase
+try:
+    from unittest.mock import patch
+except ImportError:
+    from mock import patch
 import stripe
 
 from . import StripeProvider, StripeCardProvider
 from .. import FraudStatus, PaymentStatus, RedirectNeeded
+from ..testcommon import create_test_payment
 
 
 SECRET_KEY = '1234abcd'
 PUBLIC_KEY = 'abcd1234'
 
 
-class Payment(Mock):
-
-    id = 1
-    description = 'payment'
-    currency = 'USD'
-    delivery = 10
-    status = PaymentStatus.WAITING
-    message = None
-    tax = 10
-    total = 100
-    captured_amount = 0
-    transaction_id = None
-
-    def change_status(self, status, message=''):
-        self.status = status
-        self.message = message
-
-    def change_fraud_status(self, status, message='', commit=True):
-        self.fraud_status = status
-        self.fraud_message = message
-
-    def capture(self, amount=None):
-        amount = amount or self.total
-        self.captured_amount = amount
-        self.change_status(PaymentStatus.CONFIRMED)
-
-    def get_failure_url(self):
-        return 'http://cancel.com'
-
-    def get_process_url(self):
-        return 'http://example.com'
-
-    def get_purchased_items(self):
-        return []
-
-    def get_success_url(self):
-        return 'http://success.com'
-
+Payment = create_test_payment()
 
 @contextmanager
 def mock_stripe_Charge_create(error_msg=None):
diff --git a/payments/test_core.py b/payments/test_core.py
index f6f3001fe..a4d918ca7 100644
--- a/payments/test_core.py
+++ b/payments/test_core.py
@@ -1,7 +1,10 @@
 from __future__ import unicode_literals
 from decimal import Decimal
 from unittest import TestCase
-from mock import patch, NonCallableMock
+try:
+    from unittest.mock import patch, NonCallableMock
+except ImportError:
+    from mock import  patch, NonCallableMock
 
 from payments import core
 from .forms import CreditCardPaymentFormWithName, PaymentForm
diff --git a/payments/testcommon.py b/payments/testcommon.py
index 37c9e75d4..de568ceaa 100644
--- a/payments/testcommon.py
+++ b/payments/testcommon.py
@@ -4,11 +4,13 @@
 from . import PaymentStatus, PurchasedItem
 from .utils import getter_prefixed_address
 from datetime import datetime
+try:
+    from unittest.mock import Mock
+except ImportError:
+    from mock import Mock
 
 def create_test_payment(**_kwargs):
-    class TestPayment(BasePaymentLogic):
-        def __init__(self, **kwargs):
-            self.__dict__.update(kwargs)
+    class TestPayment(Mock, BasePaymentLogic):
         id = 523
         pk = id
         description = 'payment'
@@ -17,12 +19,12 @@ def __init__(self, **kwargs):
         status = PaymentStatus.WAITING
         message = ""
         tax = Decimal(10)
-        token = "undefined"
+        token = "354338723"
         total = Decimal(100)
         captured_amount = Decimal("0.0")
         extra_data = ""
-        variant = "342"
-        transaction_id = ""
+        variant = "undefined"
+        transaction_id = None
         created = datetime.now()
         modified = datetime.now()
 
@@ -35,9 +37,23 @@ def __init__(self, **kwargs):
         billing_country_code = "US"
         billing_country_area = "Tennessee"
         billing_email = "example@example.com"
-
         customer_ip_address = "192.78.6.6"
 
+        get_billing_address = getter_prefixed_address("billing")
+        get_shipping_address = get_billing_address
+
+        def capture(self, amount=None):
+            amount = amount or self.total
+            self.captured_amount = amount
+            self.change_status(PaymentStatus.CONFIRMED)
+
+        def change_status(self, status, message=''):
+            '''
+            Updates the Payment status and sends the status_changed signal.
+            '''
+            self.status = status
+            self.message = message
+
         def get_purchased_items(self):
             return [
                 PurchasedItem(
@@ -55,5 +71,7 @@ def get_success_url(self):
 
         def save(self):
             return self
-    TestPayment.__dict__.update(_kwargs)
+    # workaround limitation in python
+    for key, val in _kwargs.items():
+        setattr(TestPayment, key, val)
     return TestPayment
diff --git a/payments/wallet/test_wallet.py b/payments/wallet/test_wallet.py
index 5a984a634..d5f9c394b 100644
--- a/payments/wallet/test_wallet.py
+++ b/payments/wallet/test_wallet.py
@@ -1,14 +1,18 @@
 from __future__ import unicode_literals
 import time
 from decimal import Decimal
-from unittest import TestCase
 
 from django.http import HttpResponse, HttpResponseForbidden
 import jwt
-from mock import MagicMock
+from unittest import TestCase
+try:
+    from unittest.mock import MagicMock
+except ImportError:
+    from mock import MagicMock
 
 from .. import PaymentStatus
 from . import GoogleWalletProvider
+from ..testcommon import create_test_payment
 
 PAYMENT_TOKEN = '5a4dae68-2715-4b1e-8bb2-2c2dbe9255f6'
 SELLER_ID = 'abc123'
@@ -31,35 +35,7 @@
         'orderId': '1234567890'}}
 
 
-class Payment(object):
-
-    id = 1
-    description = 'payment'
-    currency = 'USD'
-    delivery = Decimal(10)
-    status = PaymentStatus.WAITING
-    tax = Decimal(10)
-    token = PAYMENT_TOKEN
-    total = Decimal(100)
-    variant = VARIANT
-
-    def change_status(self, status):
-        self.status = status
-
-    def get_failure_url(self):
-        return 'http://cancel.com'
-
-    def get_process_url(self):
-        return 'http://example.com'
-
-    def get_purchased_items(self):
-        return []
-
-    def save(self):
-        return self
-
-    def get_success_url(self):
-        return 'http://success.com'
+Payment = create_test_payment(variant=VARIANT, token=PAYMENT_TOKEN)
 
 
 class TestGoogleWalletProvider(TestCase):
diff --git a/setup.py b/setup.py
index 520b268d5..6f811f6c2 100755
--- a/setup.py
+++ b/setup.py
@@ -30,11 +30,17 @@
     'suds-jurko>=0.6',
     'xmltodict>=0.9.2']
 
+TEST_REQUIREMENTS = [
+    'pytest',
+    'pytest-django'
+]
+if sys.version_info.major < 3:
+    TEST_REQUIREMENTS.append('mock')
 
 class PyTest(TestCommand):
     user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")]
     test_args = []
-    
+
     def initialize_options(self):
         TestCommand.initialize_options(self)
         self.pytest_args = []
@@ -76,8 +82,5 @@ def run_tests(self):
     install_requires=REQUIREMENTS,
     cmdclass={
         'test': PyTest},
-    tests_require=[
-        'mock',
-        'pytest',
-        'pytest-django'],
+    tests_require=TEST_REQUIREMENTS,
     zip_safe=False)

From 380ac73cf134bbe4fee4216042d86ef3b70325cd Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Fri, 6 Oct 2017 23:37:15 +0200
Subject: [PATCH 05/40] don't crash if a signal client crashes but log error

---
 payments/models.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/payments/models.py b/payments/models.py
index 2cbd8206a..38dc8f7bb 100644
--- a/payments/models.py
+++ b/payments/models.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 import json
 from uuid import uuid4
+import logging
 
 from django.conf import settings
 from django.core.urlresolvers import reverse
@@ -10,6 +11,8 @@
 from .core import provider_factory
 from . import FraudStatus, PaymentStatus
 
+# Get an instance of a logger
+logger = logging.getLogger(__name__)
 
 class PaymentAttributeProxy(object):
 
@@ -86,7 +89,9 @@ def change_status(self, status, message=''):
         self.status = status
         self.message = message
         self.save()
-        status_changed.send(sender=type(self), instance=self)
+        for receiver, result in status_changed.send_robust(sender=type(self), instance=self):
+            if isinstance(result, Exception):
+                logger.critical(result)
 
     def change_fraud_status(self, status, message='', commit=True):
         available_statuses = [choice[0] for choice in FraudStatus.CHOICES]

From d014dbf84c8bb29e8899caa774856b03f798510a Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Fri, 6 Oct 2017 23:41:26 +0200
Subject: [PATCH 06/40] fix wrong initial value in test

---
 payments/test_core.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/payments/test_core.py b/payments/test_core.py
index 47a4bf672..860c2cb95 100644
--- a/payments/test_core.py
+++ b/payments/test_core.py
@@ -49,7 +49,7 @@ def test_capture_preauth_successfully(self, mocked_capture_method):
             mocked_save_method.return_value = None
             mocked_capture_method.return_value = amount
 
-            captured_amount = Decimal('100')
+            captured_amount = Decimal('0')
             payment = BasePayment(variant='default', captured_amount=captured_amount,
                                   status=PaymentStatus.PREAUTH)
             payment.capture(amount)
@@ -65,7 +65,7 @@ def test_capture_preauth_without_amount(self, mocked_capture_method):
             mocked_save_method.return_value = None
             mocked_capture_method.return_value = amount
 
-            captured_amount = Decimal('100')
+            captured_amount = Decimal('0')
             status = PaymentStatus.PREAUTH
             payment = BasePayment(variant='default', status=status,
                                   captured_amount=captured_amount)

From dfe71b676bdaf2f6ae5204aca0bdc3a59c1bc242 Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Fri, 6 Oct 2017 23:55:09 +0200
Subject: [PATCH 07/40] fix inverted calculation

---
 payments/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/payments/models.py b/payments/models.py
index 559e50c36..fb0cef9f7 100644
--- a/payments/models.py
+++ b/payments/models.py
@@ -141,7 +141,7 @@ def capture(self, amount=None, final=True):
         provider = provider_factory(self.variant)
         amount = provider.capture(self, amount, final)
         if amount:
-            self.captured_amount -= amount
+            self.captured_amount += amount
             if final:
                 self.change_status(PaymentStatus.CONFIRMED)
         return amount

From 2f712e8329ae9cd06890d66b740fa9e9e636308c Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Sat, 7 Oct 2017 07:25:48 +0200
Subject: [PATCH 08/40] add test for robust signal handling

---
 payments/test_core.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/payments/test_core.py b/payments/test_core.py
index f6f3001fe..3effa3c1a 100644
--- a/payments/test_core.py
+++ b/payments/test_core.py
@@ -3,6 +3,8 @@
 from unittest import TestCase
 from mock import patch, NonCallableMock
 
+from django.dispatch import Signal
+
 from payments import core
 from .forms import CreditCardPaymentFormWithName, PaymentForm
 from .models import BasePayment
@@ -42,6 +44,29 @@ def test_capture_with_wrong_status(self):
         payment = BasePayment(variant='default', status=PaymentStatus.WAITING)
         self.assertRaises(ValueError, payment.capture)
 
+    @patch('payments.signals.status_changed', new_callable=Signal)
+    def test_robust_signals(self, mocked_signal):
+        with patch.object(BasePayment, 'save') as mocked_save_method:
+            mocked_save_method.return_value = None
+            def rogue_handler(sender, instance, **kwargs):
+                raise Exception("Here be dragons")
+            def benign_handler(sender, instance, **kwargs):
+                pass
+            class UnrelatedClass(object):
+                pass
+            def unrelated_handler(sender, instance, **kwargs):
+                raise Exception("Should not be called")
+            mocked_signal.connect(rogue_handler, sender=BasePayment)
+            mocked_signal.connect(benign_handler, sender=BasePayment)
+            mocked_signal.connect(unrelated_handler, sender=UnrelatedClass)
+            payment = BasePayment(variant='default', status=PaymentStatus.PREAUTH)
+            with self.assertLogs("payments.models", "CRITICAL") as logs:
+                payment.change_status(PaymentStatus.WAITING, "fooo")
+            self.assertEqual(logs.output, ['CRITICAL:payments.models:Here be dragons'])
+
+
+
+
     @patch('payments.dummy.DummyProvider.capture')
     def test_capture_preauth_successfully(self, mocked_capture_method):
         amount = Decimal('20')

From b618a42eaadcf5b58ab67a61046335716c115a5e Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Sat, 7 Oct 2017 08:04:46 +0200
Subject: [PATCH 09/40] remove superfluous newlines

---
 payments/test_core.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/payments/test_core.py b/payments/test_core.py
index 3effa3c1a..7e7edd3fa 100644
--- a/payments/test_core.py
+++ b/payments/test_core.py
@@ -64,9 +64,6 @@ def unrelated_handler(sender, instance, **kwargs):
                 payment.change_status(PaymentStatus.WAITING, "fooo")
             self.assertEqual(logs.output, ['CRITICAL:payments.models:Here be dragons'])
 
-
-
-
     @patch('payments.dummy.DummyProvider.capture')
     def test_capture_preauth_successfully(self, mocked_capture_method):
         amount = Decimal('20')

From 03eb55cd394055f0115d6fe158b9ec0e16436be4 Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Sat, 7 Oct 2017 10:11:31 +0200
Subject: [PATCH 10/40] don't test with assertLogs if not available

---
 payments/test_core.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/payments/test_core.py b/payments/test_core.py
index 7e7edd3fa..c66d3cdc9 100644
--- a/payments/test_core.py
+++ b/payments/test_core.py
@@ -60,9 +60,11 @@ def unrelated_handler(sender, instance, **kwargs):
             mocked_signal.connect(benign_handler, sender=BasePayment)
             mocked_signal.connect(unrelated_handler, sender=UnrelatedClass)
             payment = BasePayment(variant='default', status=PaymentStatus.PREAUTH)
-            with self.assertLogs("payments.models", "CRITICAL") as logs:
-                payment.change_status(PaymentStatus.WAITING, "fooo")
-            self.assertEqual(logs.output, ['CRITICAL:payments.models:Here be dragons'])
+            # python < 3.4 has no asserLogs
+            if hasattr(self, "assertLogs"):
+                with self.assertLogs("payments.models", "CRITICAL") as logs:
+                    payment.change_status(PaymentStatus.WAITING, "fooo")
+                self.assertEqual(logs.output, ['CRITICAL:payments.models:Here be dragons'])
 
     @patch('payments.dummy.DummyProvider.capture')
     def test_capture_preauth_successfully(self, mocked_capture_method):

From 932467a51b9efc8810386690cf9f5b6da6ac2cd9 Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Sat, 7 Oct 2017 10:48:39 +0200
Subject: [PATCH 11/40] add not well tested but not failing paydirekt provider

---
 payments/paydirekt/__init__.py       | 244 +++++++++++++++++++++++++++
 payments/paydirekt/test_paydirekt.py |  38 +++++
 setup.py                             |   1 +
 3 files changed, 283 insertions(+)
 create mode 100644 payments/paydirekt/__init__.py
 create mode 100644 payments/paydirekt/test_paydirekt.py

diff --git a/payments/paydirekt/__init__.py b/payments/paydirekt/__init__.py
new file mode 100644
index 000000000..857bf91f6
--- /dev/null
+++ b/payments/paydirekt/__init__.py
@@ -0,0 +1,244 @@
+""" paydirekt payment provider """
+
+
+from __future__ import unicode_literals
+try:
+    # For Python 3.0 and later
+    from urllib.error import URLError
+    from urllib.parse import urlencode
+except ImportError:
+    # Fall back to Python 2's urllib2
+    from urllib2 import URLError
+    from urllib import urlencode
+
+import uuid
+from datetime import datetime, tzinfo, timedelta
+from base64 import urlsafe_b64encode, urlsafe_b64decode
+import os
+import email.utils
+import hmac
+import simplejson as json
+import time
+import logging
+
+import requests
+from django.http import HttpResponseRedirect, HttpResponseForbidden, HttpResponse
+from django.conf import settings
+
+from .. import PaymentError, PaymentStatus, RedirectNeeded
+from ..core import BasicProvider
+from ..utils import extract_streetnr
+
+logger = logging.getLogger(__name__)
+
+class utctimezone(tzinfo):
+    def __init__(self):
+        pass
+
+    def utcoffset(self, dt):
+        return timedelta(0)
+
+    def dst(self, dt):
+        return timedelta(0)
+
+    def tzname(self, dt):
+        return "UTC"
+
+def check_response(response, response_json=None):
+    if response.status_code not in [200, 201]:
+        if response_json:
+            try:
+                errorcode = response_json["messages"][0]["code"] if "messages" in response_json and len(response_json["messages"]) > 0 else None
+                raise PaymentError("{}\n--------------------\n{}".format(response.status_code, response_json), code=errorcode)
+            except KeyError:
+                raise PaymentError(str(response.status_code))
+        else:
+            raise PaymentError(str(response.status_code))
+
+# Capture: if False ORDER is used
+class PaydirektProvider(BasicProvider):
+    '''
+    paydirekt payment provider
+
+    api_key:
+        seller key, assigned by paydirekt
+    secret:
+        seller secret key (=encoded in base64)
+    endpoint:
+        which endpoint to use
+    '''
+    access_token = None
+    expires_in = None
+
+    path_token = "{}/api/merchantintegration/v1/token/obtain"
+    path_checkout = "{}/api/checkout/v1/checkouts"
+    path_capture = "{}/api/checkout/v1/checkouts/{}/captures"
+    path_close = "{}/api/checkout/v1/checkouts/{}/close"
+    path_refund = "{}/api/checkout/v1/checkouts/{}/refunds"
+
+
+    translate_status = {
+        "APPROVED": PaymentStatus.CONFIRMED,
+        "OPEN": PaymentStatus.PREAUTH,
+        "PENDING": PaymentStatus.WAITING,
+        "REJECTED": PaymentStatus.REJECTED,
+        "CANCELED": PaymentStatus.ERROR,
+        "CLOSED": PaymentStatus.CONFIRMED,
+        "EXPIRED": PaymentStatus.ERROR,
+    }
+    header_default = {
+        "Content-Type": "application/hal+json;charset=utf-8",
+    }
+
+
+    def __init__(self, api_key, secret, endpoint="https://api.sandbox.paydirekt.de", \
+                 overcapture=False, **kwargs):
+        if not isinstance(secret, bytes):
+            self.secret_b64 = secret.encode('utf-8')
+        self.api_key = api_key
+        self.endpoint = endpoint
+        self.overcapture = overcapture
+        super(PaydirektProvider, self).__init__(**kwargs)
+
+    def retrieve_oauth_token(self):
+        """ Retrieves oauth Token and save it as instance variable """
+        token_uuid = str(uuid.uuid4()).encode("utf-8")
+        nonce = urlsafe_b64encode(os.urandom(48))
+        date_now = datetime.now(utctimezone)
+        bytessign = token_uuid+b":"+date_now.strftime("%Y%m%d%H%M%S").encode('utf-8')+b":"+self.api_key.encode('utf-8')+b":"+nonce
+        h_temp = hmac.new(urlsafe_b64decode(self.secret_b64), msg=bytessign, digestmod='sha256')
+
+        header = PaydirektProvider.header_default.copy()
+        header["X-Auth-Key"] = self.api_key
+        header["X-Request-ID"] = token_uuid
+
+        header["X-Auth-Code"] = str(urlsafe_b64encode(h_temp.digest()), 'ascii')
+        header["Date"] = email.utils.format_datetime(date_now, usegmt=True)
+        body = {
+            "grantType" : "api_key",
+            "randomNonce" : str(nonce, "ascii")
+        }
+        response = requests.post(self.path_token.format(self.endpoint), data=json.dumps(body, use_decimal=True), headers=header)
+        token_raw = json.loads(response.text, use_decimal=True)
+        check_response(response, token_raw)
+
+        self.access_token = token_raw["access_token"]
+        self.expires_in = date_now+timedelta(seconds=token_raw["expires_in"])
+
+    def check_and_update_token(self):
+        """ Check if token exists or has expired, renew it in this case """
+        if not self.expires_in or self.expires_in >= datetime.now(timezone.utc):
+            self.retrieve_oauth_token()
+
+    def get_form(self, payment, data=None):
+        if not payment.id:
+            payment.save()
+        self.check_and_update_token()
+        headers = PaydirektProvider.header_default.copy()
+        headers["Authorization"] = "Bearer %s" % self.access_token
+        # email_hash = sha256(payment.billing_email.encode("utf-8")).digest())
+        body = {
+            "type": "ORDER" if not self._capture else "DIRECT_SALE",
+            "totalAmount": payment.total,
+            "shippingAmount": payment.delivery,
+            "orderAmount": payment.total - payment.delivery,
+            "currency": payment.currency,
+            #"items": getattr(payment, "items", None),
+            #"shoppingCartType": getattr(payment, "carttype", None),
+            #"deliveryType": getattr(payment, "deliverytype", None),
+            # payment id can repeat if different shop systems are used
+            "merchantOrderReferenceNumber": "%s:%s" % (hex(int(time.time())), payment.id),
+            "redirectUrlAfterSuccess": payment.get_success_url(),
+            "redirectUrlAfterCancellation": payment.get_failure_url(),
+            "redirectUrlAfterRejection": payment.get_failure_url(),
+            "callbackUrlStatusUpdates": self.get_return_url(payment),
+            #"sha256hashedEmailAddress": str(urlsafe_b64encode(email_hash), 'ascii'),
+            "minimumAge": getattr(payment, "minimumage", None),
+            "redirectUrlAfterAgeVerificationFailure": payment.get_failure_url(),
+            #"note": payment.message[0:37]
+
+        }
+        if self.overcapture and body["type"] == "ORDER":
+            body["overcapture"] = True
+
+        shipping = payment.get_shipping_address()
+
+        shipping = {
+            "addresseeGivenName": shipping["first_name"],
+            "addresseeLastName": shipping["last_name"],
+            "company": shipping.get("company", None),
+            #"additionalAddressInformation": shipping["address_2"],
+            "street": shipping["address_1"],
+            "streetNr": extract_streetnr(shipping["address_1"], "0"),
+            "zip": shipping["postcode"],
+            "city": shipping["city"],
+            "countryCode": shipping["country_code"],
+            "state": shipping["country_area"],
+            "emailAddress": payment.billing_email
+        }
+        #strip zeroes
+        shipping = {k: v for k, v in shipping.items() if v}
+        body = {k: v for k, v in body.items() if v}
+
+        body["shippingAddress"] = shipping
+
+        response = requests.post(self.path_checkout.format(self.endpoint), data=json.dumps(body, use_decimal=True), headers=headers)
+        json_response = json.loads(response.text, use_decimal=True)
+        check_response(response, json_response)
+        raise RedirectNeeded(json_response["_links"]["approve"]["href"])
+
+    def process_data(self, payment, request):
+        try:
+            results = json.loads(request.body, use_decimal=True)
+        except (ValueError, TypeError):
+            logger.error("paydirekt returned unparseable object")
+            return HttpResponseForbidden('FAILED')
+        if not payment.transaction_id:
+            payment.transaction_id = results["checkoutId"]
+        if results["checkoutStatus"] == "APPROVED":
+            if self._capture:
+                payment.change_status(PaymentStatus.CONFIRMED)
+            else:
+                payment.change_status(PaymentStatus.PREAUTH)
+        else:
+            payment.change_status(self.translate_status[results["checkoutStatus"]])
+        payment.save()
+        return HttpResponse('OK')
+
+    def capture(self, payment, amount=None, final=True):
+        if not amount:
+            amount = payment.total
+        self.check_and_update_token()
+        header = PaydirektProvider.header_default.copy()
+        header["Authorization"] = "Bearer %s" % self.access_token
+        body = {
+            "amount": amount,
+            "finalCapture": final,
+            "callbackUrlStatusUpdates": self.get_return_url(payment)
+        }
+        response = requests.post(self.path_capture.format(self.endpoint, payment.transaction_id), \
+                                 data=json.dumps(body, use_decimal=True), headers=header)
+        json_response = json.loads(response.text, use_decimal=True)
+        check_response(response, json_response)
+        if final:
+            response = requests.post(self.path_close.format(self.endpoint, payment.transaction_id), \
+                                     headers=header)
+            json_response = json.loads(response.text, use_decimal=True)
+            check_response(response, json_response)
+        return amount
+
+    def refund(self, payment, amount=None):
+        if not amount:
+            amount = payment.total
+        self.check_and_update_token()
+        header = PaydirektProvider.header_default.copy()
+        header["Authorization"] = "Bearer %s" % self.access_token
+        body = {
+            "amount": amount,
+            "callbackUrlStatusUpdates": self.get_return_url(payment)
+        }
+        response = requests.post(self.path_refund.format(self.endpoint, payment.transaction_id), \
+                                 data=json.dumps(body, use_decimal=True), headers=header)
+        json_response = json.loads(response.text, use_decimal=True)
+        check_response(response, json_response)
+        return amount
diff --git a/payments/paydirekt/test_paydirekt.py b/payments/paydirekt/test_paydirekt.py
new file mode 100644
index 000000000..209fb66e5
--- /dev/null
+++ b/payments/paydirekt/test_paydirekt.py
@@ -0,0 +1,38 @@
+
+import simplejson as json
+
+from unittest import TestCase
+try:
+    from unittest.mock import MagicMock, patch
+except ImportError:
+    from mock import MagicMock, patch
+
+from . import PaydirektProvider
+from .. import FraudStatus, PaymentError, PaymentStatus, RedirectNeeded
+from ..testcommon import create_test_payment
+
+VARIANT = 'paydirekt'
+API_KEY = '5a4dae68-2715-4b1e-8bb2-2c2dbe9255f6'
+SECRET = '123abc'
+
+PROCESS_DATA = {
+  "checkoutId" : "64e0bd1f-c3a3-47e1-aaff-75e690c062f8",
+  "merchantOrderReferenceNumber" : "order-A12223412",
+  "checkoutStatus" : "APPROVED"
+}
+
+Payment = create_test_payment(variant=VARIANT, currency='EUR')
+
+
+class TestPaydirektProvider(TestCase):
+
+    def setUp(self):
+        self.payment = Payment()
+        self.provider = PaydirektProvider(API_KEY, SECRET)
+
+    def test_process_data_works(self):
+        request = MagicMock()
+        request.body = json.dumps(PROCESS_DATA)
+        response = self.provider.process_data(self.payment, request)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(self.payment.status, PaymentStatus.CONFIRMED)
diff --git a/setup.py b/setup.py
index 6f811f6c2..22f7f0b0d 100755
--- a/setup.py
+++ b/setup.py
@@ -15,6 +15,7 @@
     'payments.dummy',
     'payments.dotpay',
     'payments.paypal',
+    'payments.paydirekt',
     'payments.sagepay',
     'payments.sofort',
     'payments.stripe',

From 5b6f807813811288b81ff8a4905daad28254f14c Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Sun, 8 Oct 2017 13:30:59 +0200
Subject: [PATCH 12/40] paydirekt fixes, add item support

---
 payments/paydirekt/__init__.py | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/payments/paydirekt/__init__.py b/payments/paydirekt/__init__.py
index 857bf91f6..1981bdb39 100644
--- a/payments/paydirekt/__init__.py
+++ b/payments/paydirekt/__init__.py
@@ -130,6 +130,17 @@ def check_and_update_token(self):
         if not self.expires_in or self.expires_in >= datetime.now(timezone.utc):
             self.retrieve_oauth_token()
 
+    def _prepare_items(self, payment):
+        items = []
+        for newitem in payment.get_purchased_items():
+            items.append({
+                "name": newitem.name,
+                # limit to 2 decimal_places even 4 decimal_places should be possible
+                "price": newitem.price.quantize(Decimal('0.01')),
+                "quantity": int(newitem.quantity)
+            })
+        return items
+
     def get_form(self, payment, data=None):
         if not payment.id:
             payment.save()
@@ -143,6 +154,7 @@ def get_form(self, payment, data=None):
             "shippingAmount": payment.delivery,
             "orderAmount": payment.total - payment.delivery,
             "currency": payment.currency,
+            "refundLimit": 100,
             #"items": getattr(payment, "items", None),
             #"shoppingCartType": getattr(payment, "carttype", None),
             #"deliveryType": getattr(payment, "deliverytype", None),
@@ -151,10 +163,10 @@ def get_form(self, payment, data=None):
             "redirectUrlAfterSuccess": payment.get_success_url(),
             "redirectUrlAfterCancellation": payment.get_failure_url(),
             "redirectUrlAfterRejection": payment.get_failure_url(),
+            "redirectUrlAfterAgeVerificationFailure": payment.get_failure_url(),
             "callbackUrlStatusUpdates": self.get_return_url(payment),
             #"sha256hashedEmailAddress": str(urlsafe_b64encode(email_hash), 'ascii'),
             "minimumAge": getattr(payment, "minimumage", None),
-            "redirectUrlAfterAgeVerificationFailure": payment.get_failure_url(),
             #"note": payment.message[0:37]
 
         }
@@ -176,12 +188,16 @@ def get_form(self, payment, data=None):
             "state": shipping["country_area"],
             "emailAddress": payment.billing_email
         }
-        #strip zeroes
+        #strip Nones
         shipping = {k: v for k, v in shipping.items() if v}
         body = {k: v for k, v in body.items() if v}
 
         body["shippingAddress"] = shipping
 
+        items = self._prepare_items(payment)
+        if len(items) > 0:
+            body["items"] = items
+
         response = requests.post(self.path_checkout.format(self.endpoint), data=json.dumps(body, use_decimal=True), headers=headers)
         json_response = json.loads(response.text, use_decimal=True)
         check_response(response, json_response)
@@ -220,11 +236,6 @@ def capture(self, payment, amount=None, final=True):
                                  data=json.dumps(body, use_decimal=True), headers=header)
         json_response = json.loads(response.text, use_decimal=True)
         check_response(response, json_response)
-        if final:
-            response = requests.post(self.path_close.format(self.endpoint, payment.transaction_id), \
-                                     headers=header)
-            json_response = json.loads(response.text, use_decimal=True)
-            check_response(response, json_response)
         return amount
 
     def refund(self, payment, amount=None):

From 52fb42390e3520a40f812da3bde490d0c0525bce Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Sun, 8 Oct 2017 13:33:38 +0200
Subject: [PATCH 13/40] fix tests and missing simplejson dependency

---
 payments/testcommon.py | 2 +-
 setup.py               | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/payments/testcommon.py b/payments/testcommon.py
index de568ceaa..cf0ada0d9 100644
--- a/payments/testcommon.py
+++ b/payments/testcommon.py
@@ -57,7 +57,7 @@ def change_status(self, status, message=''):
         def get_purchased_items(self):
             return [
                 PurchasedItem(
-                    name='foo', quantity=Decimal('10'), price=Decimal('20'),
+                    name='foo', quantity=10, price=Decimal('20'),
                     currency='USD', sku='bar')]
 
         def get_failure_url(self):
diff --git a/setup.py b/setup.py
index 6f811f6c2..61e5345d0 100755
--- a/setup.py
+++ b/setup.py
@@ -28,7 +28,8 @@
     'requests>=1.2.0',
     'stripe>=1.9.8',
     'suds-jurko>=0.6',
-    'xmltodict>=0.9.2']
+    'xmltodict>=0.9.2'
+    'simplejson>=3.11']
 
 TEST_REQUIREMENTS = [
     'pytest',

From 49cc6adca11b60f5708e26fb4035bd28cb40f4cd Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Sun, 8 Oct 2017 17:49:04 +0200
Subject: [PATCH 14/40] improve paydirekt api, add  tests

---
 payments/paydirekt/__init__.py       |  59 +++++--
 payments/paydirekt/test_paydirekt.py | 223 ++++++++++++++++++++++++++-
 setup.py                             |   2 +-
 3 files changed, 269 insertions(+), 15 deletions(-)

diff --git a/payments/paydirekt/__init__.py b/payments/paydirekt/__init__.py
index 1981bdb39..f43384779 100644
--- a/payments/paydirekt/__init__.py
+++ b/payments/paydirekt/__init__.py
@@ -22,7 +22,7 @@
 import logging
 
 import requests
-from django.http import HttpResponseRedirect, HttpResponseForbidden, HttpResponse
+from django.http import HttpResponseRedirect, HttpResponseForbidden, HttpResponseServerError, HttpResponse
 from django.conf import settings
 
 from .. import PaymentError, PaymentStatus, RedirectNeeded
@@ -104,7 +104,7 @@ def retrieve_oauth_token(self):
         """ Retrieves oauth Token and save it as instance variable """
         token_uuid = str(uuid.uuid4()).encode("utf-8")
         nonce = urlsafe_b64encode(os.urandom(48))
-        date_now = datetime.now(utctimezone)
+        date_now = datetime.now(utctimezone())
         bytessign = token_uuid+b":"+date_now.strftime("%Y%m%d%H%M%S").encode('utf-8')+b":"+self.api_key.encode('utf-8')+b":"+nonce
         h_temp = hmac.new(urlsafe_b64decode(self.secret_b64), msg=bytessign, digestmod='sha256')
 
@@ -141,6 +141,15 @@ def _prepare_items(self, payment):
             })
         return items
 
+    def _retrieve_amount(self, url):
+        ret = requests.get(url)
+        try:
+            results = json.loads(request.text, use_decimal=True)
+        except (ValueError, TypeError):
+            logger.error("paydirekt returned unparseable object")
+            return None
+        return results.get("amount", None)
+
     def get_form(self, payment, data=None):
         if not payment.id:
             payment.save()
@@ -154,7 +163,7 @@ def get_form(self, payment, data=None):
             "shippingAmount": payment.delivery,
             "orderAmount": payment.total - payment.delivery,
             "currency": payment.currency,
-            "refundLimit": 100,
+            "refundLimit": 110,
             #"items": getattr(payment, "items", None),
             #"shoppingCartType": getattr(payment, "carttype", None),
             #"deliveryType": getattr(payment, "deliverytype", None),
@@ -211,19 +220,49 @@ def process_data(self, payment, request):
             return HttpResponseForbidden('FAILED')
         if not payment.transaction_id:
             payment.transaction_id = results["checkoutId"]
-        if results["checkoutStatus"] == "APPROVED":
-            if self._capture:
+            payment.save()
+        if "checkoutStatus" in results:
+            if results["checkoutStatus"] == "APPROVED":
+                if self._capture:
+                    payment.change_status(PaymentStatus.CONFIRMED)
+                else:
+                    payment.change_status(PaymentStatus.PREAUTH)
+            elif results["checkoutStatus"] == "CLOSED":
                 payment.change_status(PaymentStatus.CONFIRMED)
-            else:
-                payment.change_status(PaymentStatus.PREAUTH)
-        else:
-            payment.change_status(self.translate_status[results["checkoutStatus"]])
-        payment.save()
+            elif not results["checkoutStatus"] in ["OPEN", "PENDING"]:
+                payment.change_status(PaymentStatus.ERROR)
+        elif "refundStatus" in results:
+            if results["refundStatus"] == "FAILED":
+                logger.error("refund failed, try to recover")
+                amount = self._retrieve_amount("/".join(self.path_refund.format(self.endpoint, payment.transaction_id), results["transactionId"]))
+                if not amount:
+                    logger.error("refund recovery failed")
+                    payment.change_status(PaymentStatus.ERROR)
+                    return HttpResponseForbidden('FAILED')
+                logger.error("refund recovery successfull")
+                payment.captured_amount += amount
+                payment.change_status(PaymentStatus.ERROR)
+        elif "captureStatus" in results:
+            # e.g. if not enough money or capture limit reached
+            if results["captureStatus"] == "FAILED":
+                logger.error("capture failed, try to recover")
+                amount = self._retrieve_amount("/".join(self.path_capture.format(self.endpoint, payment.transaction_id), results["transactionId"]))
+                if not amount:
+                    logger.error("capture recovery failed")
+                    payment.change_status(PaymentStatus.ERROR)
+                    return HttpResponseForbidden('FAILED')
+                logger.error("capture recovery successfull")
+                payment.captured_amount -= amount
+                #payment.change_status(PaymentStatus.ERROR)
         return HttpResponse('OK')
 
     def capture(self, payment, amount=None, final=True):
         if not amount:
             amount = payment.total
+        if self.overcapture and amount > payment.total*Decimal("1.1"):
+            return None
+        elif not self.overcapture and amount > payment.total:
+            return None
         self.check_and_update_token()
         header = PaydirektProvider.header_default.copy()
         header["Authorization"] = "Bearer %s" % self.access_token
diff --git a/payments/paydirekt/test_paydirekt.py b/payments/paydirekt/test_paydirekt.py
index 209fb66e5..ab4ec5bd7 100644
--- a/payments/paydirekt/test_paydirekt.py
+++ b/payments/paydirekt/test_paydirekt.py
@@ -15,12 +15,175 @@
 API_KEY = '5a4dae68-2715-4b1e-8bb2-2c2dbe9255f6'
 SECRET = '123abc'
 
-PROCESS_DATA = {
-  "checkoutId" : "64e0bd1f-c3a3-47e1-aaff-75e690c062f8",
+directsale_data = {
+  "checkoutId" : "6be6a80d-ef67-47c8-a5bd-2461d11da24c",
   "merchantOrderReferenceNumber" : "order-A12223412",
   "checkoutStatus" : "APPROVED"
 }
 
+order_data = {
+  "checkoutId" : "dcc6cebc-5d92-4212-bca9-a442a32448e1",
+  "merchantOrderReferenceNumber" : "order-A12223412",
+  "checkoutStatus" : "APPROVED"
+}
+
+token_retrieve = {
+  "access_token" : "EeNDpcqKeTJmbHYYvLvAFB7lnEaS0n8m6WNxL4IvHcLDa3iJ6XngQncrvXHKfJ4fxXhd1WCuRFFl4q617gkQrbSTIl_OtFeg39USAQMQTjWfP-ylUCnXZuORN6Zmscn0NLVY3OMrsbAqm5lECST07DjQLtLz8JO7E5urwjxxMMDPeRpOxg8yvoV42tQ5-AVahFu3i19HWphh2IPpZoE73t9k0pFtL6zGqo6CgjIHWcouDgNAQtFbleJkQtxPRNkUM-g-qgVMI7NiUhLVtFkohBa8-wUd5MU49YmaIgJlCWuGH_c6WhJMf4ryGOzi7bsCwS2NPR9HEzVWvAOA8-aUMiX0ud81pOlaJTiEyV3AWj1DyqOS_bcWiCwov-Xj2uU26aleSQPdDxJmwTphifRVqFBbC2LVb74VkRdJneaLXH-gw_Ge4Vzq_a1-v-CRZVkC90x7iy5IjaSh12HWs73UV0WpFEWQlWTmdahX9VLMBr-DNvDr8vbxlEL_h5uI28t4A1kRHF_lIO7z3lE7JMWQbEh-REEOH68yCK3nPx5_yVnsnFBNwXMSPBVP5ShWSwHCj-DPubbT_EmlbUsfagbBHCNQrOPUJCilkdOKcJNum9My4cXj8_aqtDGwM_pyNnxnpv_4qztBDPF5EbZsHzfhqNdaN09HHxoDW4DdclyYb__NpVNEQ8VOojYB-xmIhV2296BhrQlHGBKWXqf0hsDxjsTDrH2DoAVW3PvxLMrN_GMZXATVQFWHUgrd3oPGZYxgua5bs0mcPVFJujgbYR8SlHER6X5jb_3TnJbDWYowa0gzpQzr2dzW4RQxzjxGoD2dXgwZVZNIjj-X9y3NlCyxCxZmkaAa3jSiKRq6pYuRQNbfuMVU7nJZG5J_1BNGmvRWXhe9VJ6FH5lvPNfV1kyXj8EpSvgtYExSoXp5utKIiytVzXmZ6FwmoWYlI4WlofXnmRvDuC9dUeMpY9LuHI7zY-u3FSPvw2XuXaCPowy28u0RHIWhE9PE66pOoRWjwKpGblG7emXvDcvRNVw6YUCsJKiV2skEZbBw9P78DKyDWgBcbUNlqGngkxuPdPbIro0G_CjIO15iuw98TiQw7upmvzA1fiyc21prZbQ0y4AxaSYZDgMjzfuIA6vbw1F6O3pwOp1SrzU_Z9BK4caboU78mhcYO6bte926BUyTF0nA-9iIZld-BFfQXR-2GHsts2ltbuMkUBLf-1OTqKNocAL7vyHISKxqBL4BhVnxl2RjyoFP_luJuRx_MM2uRlLgtcQghc_K9gi80vwFgPi2Mfx2dpRTO2MT_io8QJmcIWjiDxo",
+  "token_type" : "bearer",
+  "expires_in" : 3599,
+  "scope" : "checkout reporting thirdparty",
+  "jti" : "3e0d485a-8433-47b4-b8c8-7e3f7571614b"
+}
+
+checkout_direct_sale = {
+  "checkoutId" : "6be6a80d-ef67-47c8-a5bd-2461d11da24c",
+  "type" : "DIRECT_SALE",
+  "status" : "OPEN",
+  "creationTimestamp" : "2017-10-02T08:41:09.728Z",
+  "totalAmount" : 100.0,
+  "shippingAmount" : 3.5,
+  "orderAmount" : 96.5,
+  "refundLimit" : 200,
+  "currency" : "EUR",
+  "items" : [ {
+    "quantity" : 3,
+    "name" : "Bobbycar",
+    "ean" : "800001303",
+    "price" : 25.99
+  }, {
+    "quantity" : 1,
+    "name" : "Helm",
+    "price" : 18.53
+  } ],
+  "deliveryType" : "STANDARD",
+  "shippingAddress" : {
+    "addresseeGivenName" : "Marie",
+    "addresseeLastName" : "Mustermann",
+    "company" : "Musterbau GmbH & Co KG",
+    "street" : "Kastanienallee",
+    "streetNr" : "999",
+    "additionalAddressInformation" : "Im Rückgebäude",
+    "zip" : "90402",
+    "city" : "Schwaig",
+    "countryCode" : "DE",
+    "state" : "Bayern"
+  },
+  "merchantOrderReferenceNumber" : "order-A12223412",
+  "merchantCustomerNumber" : "cust-732477",
+  "merchantInvoiceReferenceNumber" : "20150112334345",
+  "merchantReconciliationReferenceNumber" : "recon-A12223412",
+  "note" : "Ihr Einkauf bei Spielauto-Versand.",
+  "minimumAge" : 18,
+  "redirectUrlAfterSuccess" : "https://spielauto-versand.de/order/123/success",
+  "redirectUrlAfterCancellation" : "https://spielauto-versand.de/order/123/cancellation",
+  "redirectUrlAfterAgeVerificationFailure" : "https://spielauto-versand.de/order/123/ageverificationfailed",
+  "redirectUrlAfterRejection" : "https://spielauto-versand.de/order/123/rejection",
+  "callbackUrlStatusUpdates" : "https://spielauto-versand.de/callback/status",
+  "deliveryInformation" : {
+    "expectedShippingDate" : "2016-10-19T12:00:00.000Z",
+    "logisticsProvider" : "DHL",
+    "trackingNumber" : "1234567890"
+  },
+  "_links" : {
+    "approve" : {
+      "href" : "https://paydirekt.de/checkout/#/checkout/6be6a80d-ef67-47c8-a5bd-2461d11da24c"
+    },
+    "self" : {
+      "href" : "https://api.paydirekt.de/api/checkout/v1/checkouts/6be6a80d-ef67-47c8-a5bd-2461d11da24c"
+    }
+  }
+}
+
+checkout_order = {
+  "checkoutId" : "dcc6cebc-5d92-4212-bca9-a442a32448e1",
+  "type" : "ORDER",
+  "status" : "OPEN",
+  "creationTimestamp" : "2017-10-02T08:39:26.460Z",
+  "totalAmount" : 100.0,
+  "shippingAmount" : 3.5,
+  "orderAmount" : 96.5,
+  "refundLimit" : 200,
+  "currency" : "EUR",
+  "items" : [ {
+    "quantity" : 3,
+    "name" : "Bobbycar",
+    "ean" : "800001303",
+    "price" : 25.99
+  }, {
+    "quantity" : 1,
+    "name" : "Helm",
+    "price" : 18.53
+  } ],
+  "shoppingCartType" : "PHYSICAL",
+  "deliveryType" : "STANDARD",
+  "shippingAddress" : {
+    "addresseeGivenName" : "Marie",
+    "addresseeLastName" : "Mustermann",
+    "company" : "Musterbau GmbH & Co KG",
+    "street" : "Kastanienallee",
+    "streetNr" : "999",
+    "additionalAddressInformation" : "Im Rückgebäude",
+    "zip" : "90402",
+    "city" : "Schwaig",
+    "countryCode" : "DE",
+    "state" : "Bayern"
+  },
+  "merchantOrderReferenceNumber" : "order-A12223412",
+  "merchantCustomerNumber" : "cust-732477",
+  "merchantInvoiceReferenceNumber" : "20150112334345",
+  "redirectUrlAfterSuccess" : "https://spielauto-versand.de/order/123/success",
+  "redirectUrlAfterCancellation" : "https://spielauto-versand.de/order/123/cancellation",
+  "redirectUrlAfterRejection" : "https://spielauto-versand.de/order/123/rejection",
+  "callbackUrlStatusUpdates" : "https://spielauto-versand.de/callback/status",
+  "_links" : {
+    "approve" : {
+      "href" : "https://paydirekt.de/checkout/#/checkout/dcc6cebc-5d92-4212-bca9-a442a32448e1"
+    },
+    "self" : {
+      "href" : "https://api.paydirekt.de/api/checkout/v1/checkouts/dcc6cebc-5d92-4212-bca9-a442a32448e1"
+    }
+  }
+}
+capture_response = {
+  "type" : "CAPTURE_ORDER",
+  "transactionId" : "79cc2cdc-75ea-4496-9fd9-866b8b82dc39",
+  "amount" : 10,
+  "merchantReconciliationReferenceNumber" : "recon-1234",
+  "finalCapture" : False,
+  "merchantCaptureReferenceNumber" : "capture-21323",
+  "captureInvoiceReferenceNumber" : "invoice-1234",
+  "callbackUrlStatusUpdates" : "https://spielauto-versand.de/callback/status",
+  "deliveryInformation" : {
+    "expectedShippingDate" : "2016-10-19T12:00:00.000Z",
+    "logisticsProvider" : "DHL",
+    "trackingNumber" : "1234567890"
+  },
+  "status" : "SUCCESSFUL",
+  "_links" : {
+    "self" : {
+      "href" : "https://api.paydirekt.de/api/checkout/v1/checkouts/f3fa56c8-5633-435b-96c2-60c343b315b7/captures/79cc2cdc-75ea-4496-9fd9-866b8b82dc39"
+    }
+  }
+}
+
+refund_response = {
+  "type" : "REFUND",
+  "transactionId" : "690bee3c-cbd2-4826-b883-0d85d25b1081",
+  "amount" : 10,
+  "merchantReconciliationReferenceNumber" : "recon-1234",
+  "note" : "Ihre Bestellung vom 31.03.2015",
+  "merchantRefundReferenceNumber" : "refund-99989",
+  "status" : "PENDING",
+  "reason" : "MERCHANT_CAN_NOT_DELIVER_GOODS",
+  "callbackUrlStatusUpdates" : "https://spielauto-versand.de/callback/status",
+  "_links" : {
+    "self" : {
+      "href" : "https://api.paydirekt.de/api/checkout/v1/checkouts/dcc6cebc-5d92-4212-bca9-a442a32448e1/refunds/690bee3c-cbd2-4826-b883-0d85d25b1081"
+    }
+  }
+}
+
+
 Payment = create_test_payment(variant=VARIANT, currency='EUR')
 
 
@@ -30,9 +193,61 @@ def setUp(self):
         self.payment = Payment()
         self.provider = PaydirektProvider(API_KEY, SECRET)
 
-    def test_process_data_works(self):
+    def test_process_data(self):
         request = MagicMock()
-        request.body = json.dumps(PROCESS_DATA)
+        request.body = json.dumps(directsale_data)
         response = self.provider.process_data(self.payment, request)
         self.assertEqual(response.status_code, 200)
         self.assertEqual(self.payment.status, PaymentStatus.CONFIRMED)
+
+    @patch("requests.post")
+    def test_checkout_direct(self, mocked_post):
+        def return_url_data(url, *args, **kwargs):
+            if url == self.provider.path_token.format(self.provider.endpoint):
+                return token_retrieve
+            elif url == self.provider.path_checkout.format(self.provider.endpoint):
+                return checkout_direct_sale
+        mocked_post.side_effect = return_url_data
+        with self.assertRaises(RedirectNeeded) as cm:
+            self.provider.get_form(self.payment)
+        self.assertEqual(cm.exception.args[0], "https://paydirekt.de/checkout/#/checkout/6be6a80d-ef67-47c8-a5bd-2461d11da24c")
+
+    @patch("requests.post")
+    def test_checkout_order(self, mocked_post):
+        def return_url_data(url, *args, **kwargs):
+            response = MagicMock()
+            if url == self.provider.path_token.format(self.provider.endpoint):
+                response.text = token_retrieve
+            elif url == self.provider.path_checkout.format(self.provider.endpoint):
+                response.text = json.dumps(checkout_order
+            return response
+        mocked_post.side_effect = return_url_data
+
+        with self.assertRaises(RedirectNeeded) as cm:
+            self.provider.get_form(self.payment)
+        self.assertEqual(cm.exception.args[0], "https://paydirekt.de/checkout/#/checkout/dcc6cebc-5d92-4212-bca9-a442a32448e1")
+
+    @patch("requests.post")
+    def test_capture_refund(self, mocked_post):
+        request = MagicMock()
+        request.body = json.dumps(order_data)
+        self.provider.process_data(self.payment, request)
+
+        def return_url_data(url, *args, **kwargs):
+            response = MagicMock()
+            if url == self.provider.path_token.format(self.provider.endpoint):
+                response.text = token_retrieve
+            elif url == self.provider.path_capture.format(self.provider.endpoint, self.payment.transaction_id):
+                response.text = capture_response
+            return response
+        mocked_post.side_effect = return_url_data
+
+        ret = self.provider.capture(self.payment)
+        self.assertEqual(ret, Decimal(100))
+        self.assertEqual(self.payment.status, PaymentStatus.CONFIRMED)
+        self.assertEqual(self.payment.captured_amount, Decimal("0.0"))
+
+        ret = self.provider.refund(self.payment)
+        self.assertEqual(ret, Decimal(100))
+        self.assertEqual(self.payment.status, PaymentStatus.CONFIRMED)
+        self.assertEqual(self.payment.captured_amount, Decimal("0.0"))
diff --git a/setup.py b/setup.py
index 972a10a9a..aa1663b05 100755
--- a/setup.py
+++ b/setup.py
@@ -29,7 +29,7 @@
     'requests>=1.2.0',
     'stripe>=1.9.8',
     'suds-jurko>=0.6',
-    'xmltodict>=0.9.2'
+    'xmltodict>=0.9.2',
     'simplejson>=3.11']
 
 TEST_REQUIREMENTS = [

From 736e16a47d346178c720c0a9b6c7b5d163f45cea Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Sun, 8 Oct 2017 20:33:21 +0200
Subject: [PATCH 15/40] simplejson only required from paydirekt, and a comma is
 missing

---
 setup.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 61e5345d0..6f811f6c2 100755
--- a/setup.py
+++ b/setup.py
@@ -28,8 +28,7 @@
     'requests>=1.2.0',
     'stripe>=1.9.8',
     'suds-jurko>=0.6',
-    'xmltodict>=0.9.2'
-    'simplejson>=3.11']
+    'xmltodict>=0.9.2']
 
 TEST_REQUIREMENTS = [
     'pytest',

From cd6ab1cf7f05833e71bb5d8553e036ddd0c5238a Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Sun, 8 Oct 2017 23:07:20 +0200
Subject: [PATCH 16/40] python2 support, fix tests, enable additional address
 informations

---
 payments/paydirekt/__init__.py       | 59 ++++++++++++++++------------
 payments/paydirekt/test_paydirekt.py | 34 +++++++++++-----
 2 files changed, 57 insertions(+), 36 deletions(-)

diff --git a/payments/paydirekt/__init__.py b/payments/paydirekt/__init__.py
index f43384779..f7a0531ec 100644
--- a/payments/paydirekt/__init__.py
+++ b/payments/paydirekt/__init__.py
@@ -1,21 +1,25 @@
 """ paydirekt payment provider """
 
-
 from __future__ import unicode_literals
-try:
+import six
+if six.PY3:
     # For Python 3.0 and later
     from urllib.error import URLError
     from urllib.parse import urlencode
-except ImportError:
+else:
+    # for hmac
+    import hashlib
     # Fall back to Python 2's urllib2
     from urllib2 import URLError
     from urllib import urlencode
 
 import uuid
-from datetime import datetime, tzinfo, timedelta
+from datetime import timedelta
+from datetime import datetime as dt
+
+from decimal import Decimal
 from base64 import urlsafe_b64encode, urlsafe_b64decode
 import os
-import email.utils
 import hmac
 import simplejson as json
 import time
@@ -31,18 +35,15 @@
 
 logger = logging.getLogger(__name__)
 
-class utctimezone(tzinfo):
-    def __init__(self):
-        pass
-
-    def utcoffset(self, dt):
-        return timedelta(0)
-
-    def dst(self, dt):
-        return timedelta(0)
-
-    def tzname(self, dt):
-        return "UTC"
+# from email utils, for python 2+3 support
+def format_timetuple_and_zone(timetuple, zone):
+    return '%s, %02d %s %04d %02d:%02d:%02d %s' % (
+        ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'][timetuple[6]],
+        timetuple[2],
+        ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
+         'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'][timetuple[1] - 1],
+        timetuple[0], timetuple[3], timetuple[4], timetuple[5],
+zone)
 
 def check_response(response, response_json=None):
     if response.status_code not in [200, 201]:
@@ -93,8 +94,7 @@ class PaydirektProvider(BasicProvider):
 
     def __init__(self, api_key, secret, endpoint="https://api.sandbox.paydirekt.de", \
                  overcapture=False, **kwargs):
-        if not isinstance(secret, bytes):
-            self.secret_b64 = secret.encode('utf-8')
+        self.secret_b64 = secret.encode('utf-8')
         self.api_key = api_key
         self.endpoint = endpoint
         self.overcapture = overcapture
@@ -104,19 +104,25 @@ def retrieve_oauth_token(self):
         """ Retrieves oauth Token and save it as instance variable """
         token_uuid = str(uuid.uuid4()).encode("utf-8")
         nonce = urlsafe_b64encode(os.urandom(48))
-        date_now = datetime.now(utctimezone())
+        date_now = dt.utcnow()
         bytessign = token_uuid+b":"+date_now.strftime("%Y%m%d%H%M%S").encode('utf-8')+b":"+self.api_key.encode('utf-8')+b":"+nonce
-        h_temp = hmac.new(urlsafe_b64decode(self.secret_b64), msg=bytessign, digestmod='sha256')
+        if six.PY3:
+            h_temp = hmac.new(urlsafe_b64decode(self.secret_b64), msg=bytessign, digestmod="sha256")
+        else:
+            h_temp = hmac.new(urlsafe_b64decode(self.secret_b64), msg=bytessign, digestmod=hashlib.sha256)
 
         header = PaydirektProvider.header_default.copy()
         header["X-Auth-Key"] = self.api_key
         header["X-Request-ID"] = token_uuid
 
-        header["X-Auth-Code"] = str(urlsafe_b64encode(h_temp.digest()), 'ascii')
-        header["Date"] = email.utils.format_datetime(date_now, usegmt=True)
+        if six.PY3:
+            header["X-Auth-Code"] = str(urlsafe_b64encode(h_temp.digest()), 'ascii')
+        else:
+            header["X-Auth-Code"] = urlsafe_b64encode(h_temp.digest())
+        header["Date"] = format_timetuple_and_zone(date_now.utctimetuple(), "GMT")
         body = {
             "grantType" : "api_key",
-            "randomNonce" : str(nonce, "ascii")
+            "randomNonce" : str(nonce, "ascii") if six.PY3 else nonce
         }
         response = requests.post(self.path_token.format(self.endpoint), data=json.dumps(body, use_decimal=True), headers=header)
         token_raw = json.loads(response.text, use_decimal=True)
@@ -127,7 +133,7 @@ def retrieve_oauth_token(self):
 
     def check_and_update_token(self):
         """ Check if token exists or has expired, renew it in this case """
-        if not self.expires_in or self.expires_in >= datetime.now(timezone.utc):
+        if not self.expires_in or self.expires_in >= dt.utcnow():
             self.retrieve_oauth_token()
 
     def _prepare_items(self, payment):
@@ -188,7 +194,7 @@ def get_form(self, payment, data=None):
             "addresseeGivenName": shipping["first_name"],
             "addresseeLastName": shipping["last_name"],
             "company": shipping.get("company", None),
-            #"additionalAddressInformation": shipping["address_2"],
+            "additionalAddressInformation": shipping["address_2"],
             "street": shipping["address_1"],
             "streetNr": extract_streetnr(shipping["address_1"], "0"),
             "zip": shipping["postcode"],
@@ -209,6 +215,7 @@ def get_form(self, payment, data=None):
 
         response = requests.post(self.path_checkout.format(self.endpoint), data=json.dumps(body, use_decimal=True), headers=headers)
         json_response = json.loads(response.text, use_decimal=True)
+
         check_response(response, json_response)
         raise RedirectNeeded(json_response["_links"]["approve"]["href"])
 
diff --git a/payments/paydirekt/test_paydirekt.py b/payments/paydirekt/test_paydirekt.py
index ab4ec5bd7..172198c68 100644
--- a/payments/paydirekt/test_paydirekt.py
+++ b/payments/paydirekt/test_paydirekt.py
@@ -1,5 +1,6 @@
-
+# coding=utf-8
 import simplejson as json
+from decimal import Decimal
 
 from unittest import TestCase
 try:
@@ -12,8 +13,8 @@
 from ..testcommon import create_test_payment
 
 VARIANT = 'paydirekt'
-API_KEY = '5a4dae68-2715-4b1e-8bb2-2c2dbe9255f6'
-SECRET = '123abc'
+API_KEY = '87dbc6cd-91d2-4574-bcb5-2aaaf924386d'
+SECRET = '9Tth0qty_9zplTyY0d_QbHYvKM4iSngjoipWO6VxAao='
 
 directsale_data = {
   "checkoutId" : "6be6a80d-ef67-47c8-a5bd-2461d11da24c",
@@ -190,7 +191,7 @@
 class TestPaydirektProvider(TestCase):
 
     def setUp(self):
-        self.payment = Payment()
+        self.payment = Payment(minimumage=0)
         self.provider = PaydirektProvider(API_KEY, SECRET)
 
     def test_process_data(self):
@@ -203,10 +204,15 @@ def test_process_data(self):
     @patch("requests.post")
     def test_checkout_direct(self, mocked_post):
         def return_url_data(url, *args, **kwargs):
+            response = MagicMock()
+            response.status_code = 200
             if url == self.provider.path_token.format(self.provider.endpoint):
-                return token_retrieve
+                response.text = json.dumps(token_retrieve)
             elif url == self.provider.path_checkout.format(self.provider.endpoint):
-                return checkout_direct_sale
+                response.text = json.dumps(checkout_direct_sale)
+            else:
+                raise
+            return response
         mocked_post.side_effect = return_url_data
         with self.assertRaises(RedirectNeeded) as cm:
             self.provider.get_form(self.payment)
@@ -216,10 +222,13 @@ def return_url_data(url, *args, **kwargs):
     def test_checkout_order(self, mocked_post):
         def return_url_data(url, *args, **kwargs):
             response = MagicMock()
+            response.status_code = 200
             if url == self.provider.path_token.format(self.provider.endpoint):
-                response.text = token_retrieve
+                response.text = json.dumps(token_retrieve)
             elif url == self.provider.path_checkout.format(self.provider.endpoint):
-                response.text = json.dumps(checkout_order
+                response.text = json.dumps(checkout_order)
+            else:
+                raise
             return response
         mocked_post.side_effect = return_url_data
 
@@ -235,10 +244,15 @@ def test_capture_refund(self, mocked_post):
 
         def return_url_data(url, *args, **kwargs):
             response = MagicMock()
+            response.status_code = 200
             if url == self.provider.path_token.format(self.provider.endpoint):
-                response.text = token_retrieve
+                response.text = json.dumps(token_retrieve)
             elif url == self.provider.path_capture.format(self.provider.endpoint, self.payment.transaction_id):
-                response.text = capture_response
+                response.text = json.dumps(capture_response)
+            elif url == self.provider.path_refund.format(self.provider.endpoint, self.payment.transaction_id):
+                response.text = json.dumps(refund_response)
+            else:
+                raise
             return response
         mocked_post.side_effect = return_url_data
 

From cee3967527337920d74c9fefa37425841919efab Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Sun, 8 Oct 2017 23:22:58 +0200
Subject: [PATCH 17/40] speed up extract_streetnr, fix possible regex attack

---
 payments/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/payments/utils.py b/payments/utils.py
index d14de2dcd..210d550b3 100644
--- a/payments/utils.py
+++ b/payments/utils.py
@@ -17,7 +17,7 @@ def get_year_choices():
 
 _extract_streetnr = re.compile(r"([0-9]+)\s*$")
 def extract_streetnr(address, fallback=None):
-    ret = _extract_streetnr.findall(address)
+    ret = _extract_streetnr.findall(address[-15:])
     if ret:
         return ret[0]
     else:

From 9ded5b66da13c40be3f5db90b7cd891d91b11dcc Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Sun, 8 Oct 2017 23:53:28 +0200
Subject: [PATCH 18/40] fix order test

---
 payments/paydirekt/test_paydirekt.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/payments/paydirekt/test_paydirekt.py b/payments/paydirekt/test_paydirekt.py
index 172198c68..d55663a4c 100644
--- a/payments/paydirekt/test_paydirekt.py
+++ b/payments/paydirekt/test_paydirekt.py
@@ -231,6 +231,7 @@ def return_url_data(url, *args, **kwargs):
                 raise
             return response
         mocked_post.side_effect = return_url_data
+        self.provider._capture = False
 
         with self.assertRaises(RedirectNeeded) as cm:
             self.provider.get_form(self.payment)

From 815361f9483eb8941ad0100879f5b60a84a3612f Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Mon, 9 Oct 2017 02:26:02 +0200
Subject: [PATCH 19/40] tests and bugfixes for paydirekt

---
 payments/paydirekt/__init__.py       |  11 +-
 payments/paydirekt/test_paydirekt.py | 227 +++++++++++++++++++++------
 2 files changed, 187 insertions(+), 51 deletions(-)

diff --git a/payments/paydirekt/__init__.py b/payments/paydirekt/__init__.py
index f7a0531ec..19a317491 100644
--- a/payments/paydirekt/__init__.py
+++ b/payments/paydirekt/__init__.py
@@ -150,7 +150,7 @@ def _prepare_items(self, payment):
     def _retrieve_amount(self, url):
         ret = requests.get(url)
         try:
-            results = json.loads(request.text, use_decimal=True)
+            results = json.loads(ret.text, use_decimal=True)
         except (ValueError, TypeError):
             logger.error("paydirekt returned unparseable object")
             return None
@@ -241,26 +241,29 @@ def process_data(self, payment, request):
         elif "refundStatus" in results:
             if results["refundStatus"] == "FAILED":
                 logger.error("refund failed, try to recover")
-                amount = self._retrieve_amount("/".join(self.path_refund.format(self.endpoint, payment.transaction_id), results["transactionId"]))
+                amount = self._retrieve_amount("/".join([self.path_refund.format(self.endpoint, payment.transaction_id), results["transactionId"]]))
                 if not amount:
                     logger.error("refund recovery failed")
                     payment.change_status(PaymentStatus.ERROR)
                     return HttpResponseForbidden('FAILED')
                 logger.error("refund recovery successfull")
                 payment.captured_amount += amount
+                payment.save()
                 payment.change_status(PaymentStatus.ERROR)
         elif "captureStatus" in results:
             # e.g. if not enough money or capture limit reached
             if results["captureStatus"] == "FAILED":
                 logger.error("capture failed, try to recover")
-                amount = self._retrieve_amount("/".join(self.path_capture.format(self.endpoint, payment.transaction_id), results["transactionId"]))
+                amount = self._retrieve_amount("/".join([self.path_capture.format(self.endpoint, payment.transaction_id), results["transactionId"]]))
                 if not amount:
                     logger.error("capture recovery failed")
                     payment.change_status(PaymentStatus.ERROR)
                     return HttpResponseForbidden('FAILED')
                 logger.error("capture recovery successfull")
                 payment.captured_amount -= amount
-                #payment.change_status(PaymentStatus.ERROR)
+                payment.save()
+                payment.change_status(PaymentStatus.ERROR)
+        payment.save()
         return HttpResponse('OK')
 
     def capture(self, payment, amount=None, final=True):
diff --git a/payments/paydirekt/test_paydirekt.py b/payments/paydirekt/test_paydirekt.py
index d55663a4c..e4c0ea73b 100644
--- a/payments/paydirekt/test_paydirekt.py
+++ b/payments/paydirekt/test_paydirekt.py
@@ -16,17 +16,49 @@
 API_KEY = '87dbc6cd-91d2-4574-bcb5-2aaaf924386d'
 SECRET = '9Tth0qty_9zplTyY0d_QbHYvKM4iSngjoipWO6VxAao='
 
-directsale_data = {
+directsale_open_data = {
+  "checkoutId" : "6be6a80d-ef67-47c8-a5bd-2461d11da24c",
+  "merchantOrderReferenceNumber" : "order-A12223412",
+  "checkoutStatus" : "OPEN"
+}
+
+directsale_approve_data = {
   "checkoutId" : "6be6a80d-ef67-47c8-a5bd-2461d11da24c",
   "merchantOrderReferenceNumber" : "order-A12223412",
   "checkoutStatus" : "APPROVED"
 }
 
-order_data = {
+order_open_data = {
+  "checkoutId" : "dcc6cebc-5d92-4212-bca9-a442a32448e1",
+  "merchantOrderReferenceNumber" : "order-A12223412",
+  "checkoutStatus" : "OPEN"
+}
+order_approve_data = {
   "checkoutId" : "dcc6cebc-5d92-4212-bca9-a442a32448e1",
   "merchantOrderReferenceNumber" : "order-A12223412",
   "checkoutStatus" : "APPROVED"
 }
+order_close_data = {
+  "checkoutId" : "dcc6cebc-5d92-4212-bca9-a442a32448e1",
+  "merchantOrderReferenceNumber" : "order-A12223413",
+  "checkoutStatus" : "CLOSED"
+}
+
+capture_process_data = {
+  "checkoutId" : "e8118aa3-5bcd-450c-9f10-3785cf94053e",
+  "merchantOrderReferenceNumber" : "order-A12223412",
+  "merchantCaptureReferenceNumber" : "capture-21323",
+  "captureStatus" : "SUCCESSFUL",
+  "transactionId" : "ae68fd9f-6e9d-4a14-8507-507ab72d4986"
+}
+
+refund_process_data = {
+  "checkoutId" : "7be9023d-39c5-4f9e-ba22-2500e0d3aeb3",
+  "transactionId" : "4faa3e79-93fb-47af-a65c-96b89d80700a",
+  "merchantRefundReferenceNumber" : "refund-12345",
+  "merchantReconciliationReferenceNumber" : "reconciliation-12345",
+  "refundStatus" : "SUCCESSFUL"
+}
 
 token_retrieve = {
   "access_token" : "EeNDpcqKeTJmbHYYvLvAFB7lnEaS0n8m6WNxL4IvHcLDa3iJ6XngQncrvXHKfJ4fxXhd1WCuRFFl4q617gkQrbSTIl_OtFeg39USAQMQTjWfP-ylUCnXZuORN6Zmscn0NLVY3OMrsbAqm5lECST07DjQLtLz8JO7E5urwjxxMMDPeRpOxg8yvoV42tQ5-AVahFu3i19HWphh2IPpZoE73t9k0pFtL6zGqo6CgjIHWcouDgNAQtFbleJkQtxPRNkUM-g-qgVMI7NiUhLVtFkohBa8-wUd5MU49YmaIgJlCWuGH_c6WhJMf4ryGOzi7bsCwS2NPR9HEzVWvAOA8-aUMiX0ud81pOlaJTiEyV3AWj1DyqOS_bcWiCwov-Xj2uU26aleSQPdDxJmwTphifRVqFBbC2LVb74VkRdJneaLXH-gw_Ge4Vzq_a1-v-CRZVkC90x7iy5IjaSh12HWs73UV0WpFEWQlWTmdahX9VLMBr-DNvDr8vbxlEL_h5uI28t4A1kRHF_lIO7z3lE7JMWQbEh-REEOH68yCK3nPx5_yVnsnFBNwXMSPBVP5ShWSwHCj-DPubbT_EmlbUsfagbBHCNQrOPUJCilkdOKcJNum9My4cXj8_aqtDGwM_pyNnxnpv_4qztBDPF5EbZsHzfhqNdaN09HHxoDW4DdclyYb__NpVNEQ8VOojYB-xmIhV2296BhrQlHGBKWXqf0hsDxjsTDrH2DoAVW3PvxLMrN_GMZXATVQFWHUgrd3oPGZYxgua5bs0mcPVFJujgbYR8SlHER6X5jb_3TnJbDWYowa0gzpQzr2dzW4RQxzjxGoD2dXgwZVZNIjj-X9y3NlCyxCxZmkaAa3jSiKRq6pYuRQNbfuMVU7nJZG5J_1BNGmvRWXhe9VJ6FH5lvPNfV1kyXj8EpSvgtYExSoXp5utKIiytVzXmZ6FwmoWYlI4WlofXnmRvDuC9dUeMpY9LuHI7zY-u3FSPvw2XuXaCPowy28u0RHIWhE9PE66pOoRWjwKpGblG7emXvDcvRNVw6YUCsJKiV2skEZbBw9P78DKyDWgBcbUNlqGngkxuPdPbIro0G_CjIO15iuw98TiQw7upmvzA1fiyc21prZbQ0y4AxaSYZDgMjzfuIA6vbw1F6O3pwOp1SrzU_Z9BK4caboU78mhcYO6bte926BUyTF0nA-9iIZld-BFfQXR-2GHsts2ltbuMkUBLf-1OTqKNocAL7vyHISKxqBL4BhVnxl2RjyoFP_luJuRx_MM2uRlLgtcQghc_K9gi80vwFgPi2Mfx2dpRTO2MT_io8QJmcIWjiDxo",
@@ -74,7 +106,7 @@
   "merchantInvoiceReferenceNumber" : "20150112334345",
   "merchantReconciliationReferenceNumber" : "recon-A12223412",
   "note" : "Ihr Einkauf bei Spielauto-Versand.",
-  "minimumAge" : 18,
+  "minimumAge" : 0,
   "redirectUrlAfterSuccess" : "https://spielauto-versand.de/order/123/success",
   "redirectUrlAfterCancellation" : "https://spielauto-versand.de/order/123/cancellation",
   "redirectUrlAfterAgeVerificationFailure" : "https://spielauto-versand.de/order/123/ageverificationfailed",
@@ -101,9 +133,9 @@
   "status" : "OPEN",
   "creationTimestamp" : "2017-10-02T08:39:26.460Z",
   "totalAmount" : 100.0,
-  "shippingAmount" : 3.5,
-  "orderAmount" : 96.5,
-  "refundLimit" : 200,
+  "shippingAmount" : 2,
+  "orderAmount" : 98,
+  "refundLimit" : 110,
   "currency" : "EUR",
   "items" : [ {
     "quantity" : 3,
@@ -173,7 +205,7 @@
   "amount" : 10,
   "merchantReconciliationReferenceNumber" : "recon-1234",
   "note" : "Ihre Bestellung vom 31.03.2015",
-  "merchantRefundReferenceNumber" : "refund-99989",
+  "merchantRefundReferenceNumber" : "refund-21219",
   "status" : "PENDING",
   "reason" : "MERCHANT_CAN_NOT_DELIVER_GOODS",
   "callbackUrlStatusUpdates" : "https://spielauto-versand.de/callback/status",
@@ -184,85 +216,186 @@
   }
 }
 
+get_100_capture = {
+  "type" : "CAPTURE_ORDER",
+  "transactionId" : "79cc2cdc-75ea-4496-9fd9-866b8b82dc39",
+  "amount" : 100,
+  "merchantReconciliationReferenceNumber" : "recon-34",
+  "finalCapture" : False,
+  "merchantCaptureReferenceNumber" : "capture-2123",
+  "captureInvoiceReferenceNumber" : "invoice-1234",
+  "callbackUrlStatusUpdates" : "https://spielauto-versand.de/callback/status",
+  "deliveryInformation" : {
+    "expectedShippingDate" : "2016-10-19T12:00:00.000Z",
+    "logisticsProvider" : "DHL",
+    "trackingNumber" : "1234567890"
+  },
+  "status" : "SUCCESSFUL",
+  "paymentInformationId" : "0000000-1111-2222-3333-949499202",
+  "_links" : {
+    "self" : {
+      "href" : "https://api.paydirekt.de/api/checkout/v1/checkouts/f3fa56c8-5633-435b-96c2-60c343b315b7/captures/79cc2cdc-75ea-4496-9fd9-866b8b82dc39"
+    }
+  }
+}
+
+get_100_refund = {
+  "type" : "REFUND",
+  "transactionId" : "690bee3c-cbd2-4826-b883-0d85d25b1081",
+  "amount" : 100,
+  "merchantReconciliationReferenceNumber" : "recon-1234",
+  "note" : "Ihre Bestellung vom 31.03.2015",
+  "merchantRefundReferenceNumber" : "refund-99989",
+  "status" : "PENDING",
+  "reason" : "MERCHANT_CAN_NOT_DELIVER_GOODS",
+  "callbackUrlStatusUpdates" : "https://spielauto-versand.de/callback/status",
+  "paymentInformationId" : "0000000-1111-2222-3333-444444444444",
+  "_links" : {
+    "self" : {
+      "href" : "https://api.paydirekt.de/api/checkout/v1/checkouts/dcc6cebc-5d92-4212-bca9-a442a32448e1/refunds/690bee3c-cbd2-4826-b883-0d85d25b1081"
+    }
+  }
+}
 
 Payment = create_test_payment(variant=VARIANT, currency='EUR')
 
 
 class TestPaydirektProvider(TestCase):
 
-    def setUp(self):
-        self.payment = Payment(minimumage=0)
-        self.provider = PaydirektProvider(API_KEY, SECRET)
 
-    def test_process_data(self):
+    def test_direct_sale_response(self):
+        payment = Payment(minimumage=0)
+        provider = PaydirektProvider(API_KEY, SECRET)
+
         request = MagicMock()
-        request.body = json.dumps(directsale_data)
-        response = self.provider.process_data(self.payment, request)
+        request.body = json.dumps(directsale_open_data)
+        response = provider.process_data(payment, request)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(payment.status, PaymentStatus.WAITING)
+        request.body = json.dumps(directsale_approve_data)
+        response = provider.process_data(payment, request)
         self.assertEqual(response.status_code, 200)
-        self.assertEqual(self.payment.status, PaymentStatus.CONFIRMED)
+        self.assertEqual(payment.status, PaymentStatus.CONFIRMED)
+
+    def test_order_response(self):
+        payment = Payment(minimumage=0)
+        provider = PaydirektProvider(API_KEY, SECRET, capture=False)
+
+        request = MagicMock()
+        request.body = json.dumps(order_open_data)
+        response = provider.process_data(payment, request)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(payment.status, PaymentStatus.WAITING)
+        request.body = json.dumps(order_approve_data)
+        response = provider.process_data(payment, request)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(payment.status, PaymentStatus.PREAUTH)
+        request.body = json.dumps(capture_process_data)
+        response = provider.process_data(payment, request)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(payment.status, PaymentStatus.PREAUTH)
+        request.body = json.dumps(order_close_data)
+        response = provider.process_data(payment, request)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(payment.status, PaymentStatus.CONFIRMED)
+
+
+
 
     @patch("requests.post")
     def test_checkout_direct(self, mocked_post):
+        payment = Payment(minimumage=0)
+        provider = PaydirektProvider(API_KEY, SECRET, capture=False)
         def return_url_data(url, *args, **kwargs):
             response = MagicMock()
             response.status_code = 200
-            if url == self.provider.path_token.format(self.provider.endpoint):
+            if url == provider.path_token.format(provider.endpoint):
                 response.text = json.dumps(token_retrieve)
-            elif url == self.provider.path_checkout.format(self.provider.endpoint):
+            elif url == provider.path_checkout.format(provider.endpoint):
                 response.text = json.dumps(checkout_direct_sale)
             else:
                 raise
             return response
         mocked_post.side_effect = return_url_data
         with self.assertRaises(RedirectNeeded) as cm:
-            self.provider.get_form(self.payment)
+            provider.get_form(payment)
         self.assertEqual(cm.exception.args[0], "https://paydirekt.de/checkout/#/checkout/6be6a80d-ef67-47c8-a5bd-2461d11da24c")
 
-    @patch("requests.post")
-    def test_checkout_order(self, mocked_post):
-        def return_url_data(url, *args, **kwargs):
-            response = MagicMock()
-            response.status_code = 200
-            if url == self.provider.path_token.format(self.provider.endpoint):
-                response.text = json.dumps(token_retrieve)
-            elif url == self.provider.path_checkout.format(self.provider.endpoint):
-                response.text = json.dumps(checkout_order)
-            else:
-                raise
-            return response
-        mocked_post.side_effect = return_url_data
-        self.provider._capture = False
-
-        with self.assertRaises(RedirectNeeded) as cm:
-            self.provider.get_form(self.payment)
-        self.assertEqual(cm.exception.args[0], "https://paydirekt.de/checkout/#/checkout/dcc6cebc-5d92-4212-bca9-a442a32448e1")
-
     @patch("requests.post")
     def test_capture_refund(self, mocked_post):
+        payment = Payment(minimumage=0)
+        provider = PaydirektProvider(API_KEY, SECRET, capture=False)
         request = MagicMock()
-        request.body = json.dumps(order_data)
-        self.provider.process_data(self.payment, request)
+        request.body = json.dumps(order_approve_data)
+        provider.process_data(payment, request)
 
         def return_url_data(url, *args, **kwargs):
             response = MagicMock()
             response.status_code = 200
-            if url == self.provider.path_token.format(self.provider.endpoint):
+            if url == provider.path_token.format(provider.endpoint):
                 response.text = json.dumps(token_retrieve)
-            elif url == self.provider.path_capture.format(self.provider.endpoint, self.payment.transaction_id):
+            elif url == provider.path_capture.format(provider.endpoint, payment.transaction_id):
                 response.text = json.dumps(capture_response)
-            elif url == self.provider.path_refund.format(self.provider.endpoint, self.payment.transaction_id):
+            elif url == provider.path_refund.format(provider.endpoint, payment.transaction_id):
                 response.text = json.dumps(refund_response)
             else:
                 raise
             return response
         mocked_post.side_effect = return_url_data
 
-        ret = self.provider.capture(self.payment)
+        ret = provider.capture(payment)
         self.assertEqual(ret, Decimal(100))
-        self.assertEqual(self.payment.status, PaymentStatus.CONFIRMED)
-        self.assertEqual(self.payment.captured_amount, Decimal("0.0"))
+        self.assertEqual(payment.status, PaymentStatus.PREAUTH)
+        self.assertEqual(payment.captured_amount, Decimal("0.0"))
 
-        ret = self.provider.refund(self.payment)
+        ret = provider.refund(payment)
         self.assertEqual(ret, Decimal(100))
-        self.assertEqual(self.payment.status, PaymentStatus.CONFIRMED)
-        self.assertEqual(self.payment.captured_amount, Decimal("0.0"))
+        self.assertEqual(payment.status, PaymentStatus.PREAUTH)
+        self.assertEqual(payment.captured_amount, Decimal("0.0"))
+
+
+    @patch("requests.get")
+    def test_refund_fail(self, mocked_get):
+        payment = Payment(minimumage=0)
+        provider = PaydirektProvider(API_KEY, SECRET, capture=False)
+        def return_url_data(url, *args, **kwargs):
+            response = MagicMock()
+            response.status_code = 200
+            response.text = json.dumps(get_100_refund)
+            return response
+        mocked_get.side_effect = return_url_data
+        request = MagicMock()
+        request.body = json.dumps(order_approve_data)
+        provider.process_data(payment, request)
+
+        self.assertEqual(payment.status, PaymentStatus.PREAUTH)
+        self.assertEqual(payment.captured_amount, Decimal(0))
+        d = refund_process_data.copy()
+        d["refundStatus"] = "FAILED"
+        request.body = json.dumps(d)
+        response = provider.process_data(payment, request)
+        self.assertEqual(payment.captured_amount, Decimal(100))
+        self.assertEqual(payment.status, PaymentStatus.ERROR)
+
+    @patch("requests.get")
+    def test_capture_fail(self, mocked_get):
+        payment = Payment(minimumage=0, captured_amount=Decimal(100))
+        provider = PaydirektProvider(API_KEY, SECRET, capture=False)
+        def return_url_data(url, *args, **kwargs):
+            response = MagicMock()
+            response.status_code = 200
+            response.text = json.dumps(get_100_capture)
+            return response
+        mocked_get.side_effect = return_url_data
+        request = MagicMock()
+        request.body = json.dumps(order_approve_data)
+        provider.process_data(payment, request)
+
+        self.assertEqual(payment.status, PaymentStatus.PREAUTH)
+        self.assertEqual(payment.captured_amount, Decimal(100))
+        d = capture_process_data.copy()
+        d["captureStatus"] = "FAILED"
+        request.body = json.dumps(d)
+        response = provider.process_data(payment, request)
+        self.assertEqual(payment.captured_amount, Decimal(0))
+        self.assertEqual(payment.status, PaymentStatus.ERROR)

From 579d28738620ba9cff76957a780472f4069b18e9 Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Mon, 9 Oct 2017 13:32:27 +0200
Subject: [PATCH 20/40] fix retrieving transaction, make it thread safe, better
 handling of order close

---
 payments/paydirekt/__init__.py | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/payments/paydirekt/__init__.py b/payments/paydirekt/__init__.py
index 19a317491..0403f482d 100644
--- a/payments/paydirekt/__init__.py
+++ b/payments/paydirekt/__init__.py
@@ -24,6 +24,7 @@
 import simplejson as json
 import time
 import logging
+import threading
 
 import requests
 from django.http import HttpResponseRedirect, HttpResponseForbidden, HttpResponseServerError, HttpResponse
@@ -98,6 +99,7 @@ def __init__(self, api_key, secret, endpoint="https://api.sandbox.paydirekt.de",
         self.api_key = api_key
         self.endpoint = endpoint
         self.overcapture = overcapture
+        self.updating_token_lock = threading.Lock()
         super(PaydirektProvider, self).__init__(**kwargs)
 
     def retrieve_oauth_token(self):
@@ -133,8 +135,15 @@ def retrieve_oauth_token(self):
 
     def check_and_update_token(self):
         """ Check if token exists or has expired, renew it in this case """
-        if not self.expires_in or self.expires_in >= dt.utcnow():
-            self.retrieve_oauth_token()
+        self.updating_token_lock.acquire()
+        try:
+            if not self.expires_in or self.expires_in >= dt.utcnow()-timedelta(seconds=3):
+                self.retrieve_oauth_token()
+        except Exception as exc:
+            self.updating_token_lock.release()
+            raise exc
+        self.updating_token_lock.release()
+
 
     def _prepare_items(self, payment):
         items = []
@@ -148,7 +157,10 @@ def _prepare_items(self, payment):
         return items
 
     def _retrieve_amount(self, url):
-        ret = requests.get(url)
+        headers = {}
+        headers["Authorization"] = "Bearer %s" % self.access_token
+        self.check_and_update_token()
+        ret = requests.get(url, headers=headers)
         try:
             results = json.loads(ret.text, use_decimal=True)
         except (ValueError, TypeError):
@@ -235,7 +247,10 @@ def process_data(self, payment, request):
                 else:
                     payment.change_status(PaymentStatus.PREAUTH)
             elif results["checkoutStatus"] == "CLOSED":
-                payment.change_status(PaymentStatus.CONFIRMED)
+                if self.status != PaymentStatus.REFUNDED:
+                    payment.change_status(PaymentStatus.CONFIRMED)
+                elif self.status == PaymentStatus.PREAUTH and payment.captured_amount == 0:
+                    payment.change_status(PaymentStatus.REFUNDED)
             elif not results["checkoutStatus"] in ["OPEN", "PENDING"]:
                 payment.change_status(PaymentStatus.ERROR)
         elif "refundStatus" in results:
@@ -301,4 +316,10 @@ def refund(self, payment, amount=None):
                                  data=json.dumps(body, use_decimal=True), headers=header)
         json_response = json.loads(response.text, use_decimal=True)
         check_response(response, json_response)
+        if self.status == PaymentStatus.PREAUTH and amount == payment.captured_amount:
+            self.check_and_update_token()
+            response = requests.post(self.path_close.format(self.endpoint, payment.transaction_id), \
+                                 headers=header)
+            json_response = json.loads(response.text, use_decimal=True)
+            check_response(response, json_response)
         return amount

From dacb622b0dece731813f300df41fb56350892b70 Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Mon, 9 Oct 2017 13:38:34 +0200
Subject: [PATCH 21/40] fixes for paydirekt, update paydirekt tests

---
 payments/paydirekt/__init__.py       |  6 ++---
 payments/paydirekt/test_paydirekt.py | 34 ++++++++++++++++++++++------
 2 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/payments/paydirekt/__init__.py b/payments/paydirekt/__init__.py
index 0403f482d..9816eead8 100644
--- a/payments/paydirekt/__init__.py
+++ b/payments/paydirekt/__init__.py
@@ -247,9 +247,9 @@ def process_data(self, payment, request):
                 else:
                     payment.change_status(PaymentStatus.PREAUTH)
             elif results["checkoutStatus"] == "CLOSED":
-                if self.status != PaymentStatus.REFUNDED:
+                if payment.status != PaymentStatus.REFUNDED:
                     payment.change_status(PaymentStatus.CONFIRMED)
-                elif self.status == PaymentStatus.PREAUTH and payment.captured_amount == 0:
+                elif payment.status == PaymentStatus.PREAUTH and payment.captured_amount == 0:
                     payment.change_status(PaymentStatus.REFUNDED)
             elif not results["checkoutStatus"] in ["OPEN", "PENDING"]:
                 payment.change_status(PaymentStatus.ERROR)
@@ -316,7 +316,7 @@ def refund(self, payment, amount=None):
                                  data=json.dumps(body, use_decimal=True), headers=header)
         json_response = json.loads(response.text, use_decimal=True)
         check_response(response, json_response)
-        if self.status == PaymentStatus.PREAUTH and amount == payment.captured_amount:
+        if payment.status == PaymentStatus.PREAUTH and amount == payment.captured_amount:
             self.check_and_update_token()
             response = requests.post(self.path_close.format(self.endpoint, payment.transaction_id), \
                                  headers=header)
diff --git a/payments/paydirekt/test_paydirekt.py b/payments/paydirekt/test_paydirekt.py
index e4c0ea73b..9044ddd02 100644
--- a/payments/paydirekt/test_paydirekt.py
+++ b/payments/paydirekt/test_paydirekt.py
@@ -353,17 +353,26 @@ def return_url_data(url, *args, **kwargs):
         self.assertEqual(payment.status, PaymentStatus.PREAUTH)
         self.assertEqual(payment.captured_amount, Decimal("0.0"))
 
-
+    @patch("requests.post")
     @patch("requests.get")
-    def test_refund_fail(self, mocked_get):
+    def test_refund_fail(self, mocked_get, mocked_post):
         payment = Payment(minimumage=0)
         provider = PaydirektProvider(API_KEY, SECRET, capture=False)
-        def return_url_data(url, *args, **kwargs):
+        def return_get_data(url, *args, **kwargs):
             response = MagicMock()
             response.status_code = 200
             response.text = json.dumps(get_100_refund)
             return response
-        mocked_get.side_effect = return_url_data
+        def return_post_data(url, *args, **kwargs):
+            response = MagicMock()
+            response.status_code = 200
+            if url == provider.path_token.format(provider.endpoint):
+                response.text = json.dumps(token_retrieve)
+            else:
+                raise
+            return response
+        mocked_get.side_effect = return_get_data
+        mocked_post.side_effect = return_post_data
         request = MagicMock()
         request.body = json.dumps(order_approve_data)
         provider.process_data(payment, request)
@@ -377,16 +386,27 @@ def return_url_data(url, *args, **kwargs):
         self.assertEqual(payment.captured_amount, Decimal(100))
         self.assertEqual(payment.status, PaymentStatus.ERROR)
 
+    @patch("requests.post")
     @patch("requests.get")
-    def test_capture_fail(self, mocked_get):
+    def test_capture_fail(self, mocked_get, mocked_post):
         payment = Payment(minimumage=0, captured_amount=Decimal(100))
         provider = PaydirektProvider(API_KEY, SECRET, capture=False)
-        def return_url_data(url, *args, **kwargs):
+        def return_get_data(url, *args, **kwargs):
             response = MagicMock()
             response.status_code = 200
             response.text = json.dumps(get_100_capture)
             return response
-        mocked_get.side_effect = return_url_data
+
+        def return_post_data(url, *args, **kwargs):
+            response = MagicMock()
+            response.status_code = 200
+            if url == provider.path_token.format(provider.endpoint):
+                response.text = json.dumps(token_retrieve)
+            else:
+                raise
+            return response
+        mocked_get.side_effect = return_get_data
+        mocked_post.side_effect = return_post_data
         request = MagicMock()
         request.body = json.dumps(order_approve_data)
         provider.process_data(payment, request)

From fdcef46258b6cd0b905f91ab304d9377dcbeb1f3 Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Mon, 9 Oct 2017 18:08:28 +0200
Subject: [PATCH 22/40] fix change_status logic: send only signal if status
 changes

---
 payments/models.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/payments/models.py b/payments/models.py
index fb0cef9f7..df59a9b0a 100644
--- a/payments/models.py
+++ b/payments/models.py
@@ -82,11 +82,14 @@ def change_status(self, status, message=''):
         '''
         Updates the Payment status and sends the status_changed signal.
         '''
-        from .signals import status_changed
-        self.status = status
-        self.message = message
-        self.save()
-        status_changed.send(sender=type(self), instance=self)
+        if  self.status != status:
+            from .signals import status_changed
+            self.status = status
+            self.message = message
+            self.save()
+            status_changed.send(sender=type(self), instance=self)
+        else:
+            self.save()
 
     def change_fraud_status(self, status, message='', commit=True):
         available_statuses = [choice[0] for choice in FraudStatus.CHOICES]

From 6ce254a85fe4be221a0e716f4a513e5e6849e509 Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Mon, 9 Oct 2017 18:15:28 +0200
Subject: [PATCH 23/40] fix refund logic

---
 payments/paydirekt/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/payments/paydirekt/__init__.py b/payments/paydirekt/__init__.py
index 9816eead8..d6043ee90 100644
--- a/payments/paydirekt/__init__.py
+++ b/payments/paydirekt/__init__.py
@@ -317,6 +317,8 @@ def refund(self, payment, amount=None):
         json_response = json.loads(response.text, use_decimal=True)
         check_response(response, json_response)
         if payment.status == PaymentStatus.PREAUTH and amount == payment.captured_amount:
+            # logic, elsewise multiple signals are emitted CONFIRMED -> REFUNDED
+            payment.change_status(PaymentStatus.REFUNDED)
             self.check_and_update_token()
             response = requests.post(self.path_close.format(self.endpoint, payment.transaction_id), \
                                  headers=header)

From fb0a31ef5f44e762ea666e86154a7f5fce29a3ed Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Mon, 9 Oct 2017 23:09:42 +0200
Subject: [PATCH 24/40] fix string instead Decimal bug for empty values

---
 payments/models.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/payments/models.py b/payments/models.py
index 77ebdb211..dc4d1e7bb 100644
--- a/payments/models.py
+++ b/payments/models.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 import json
 from uuid import uuid4
+from decimal import Decimal
 
 from django.conf import settings
 from django.core.urlresolvers import reverse
@@ -159,10 +160,10 @@ class BasePayment(models.Model, BasePaymentLogic):
     #: Currency code (may be provider-specific)
     currency = models.CharField(max_length=10)
     #: Total amount (gross)
-    total = models.DecimalField(max_digits=9, decimal_places=2, default='0.0')
+    total = models.DecimalField(max_digits=9, decimal_places=2, default=Decimal('0.0'))
     delivery = models.DecimalField(
-        max_digits=9, decimal_places=2, default='0.0')
-    tax = models.DecimalField(max_digits=9, decimal_places=2, default='0.0')
+        max_digits=9, decimal_places=2, default=Decimal('0.0'))
+    tax = models.DecimalField(max_digits=9, decimal_places=2, default=Decimal('0.0'))
     description = models.TextField(blank=True, default='')
     billing_email = models.EmailField(blank=True)
     customer_ip_address = models.GenericIPAddressField(blank=True, null=True)
@@ -170,7 +171,7 @@ class BasePayment(models.Model, BasePaymentLogic):
     message = models.TextField(blank=True, default='')
     token = models.CharField(max_length=36, blank=True, default='')
     captured_amount = models.DecimalField(
-        max_digits=9, decimal_places=2, default='0.0')
+        max_digits=9, decimal_places=2, default=Decimal('0.0'))
 
     class Meta:
         abstract = True

From 6f2912074342e800293defd48201e55cdf284fdb Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Tue, 10 Oct 2017 01:02:46 +0200
Subject: [PATCH 25/40] remove unneeded authentification

---
 payments/paydirekt/__init__.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/payments/paydirekt/__init__.py b/payments/paydirekt/__init__.py
index d6043ee90..8576a09fa 100644
--- a/payments/paydirekt/__init__.py
+++ b/payments/paydirekt/__init__.py
@@ -157,10 +157,7 @@ def _prepare_items(self, payment):
         return items
 
     def _retrieve_amount(self, url):
-        headers = {}
-        headers["Authorization"] = "Bearer %s" % self.access_token
-        self.check_and_update_token()
-        ret = requests.get(url, headers=headers)
+        ret = requests.get(url)
         try:
             results = json.loads(ret.text, use_decimal=True)
         except (ValueError, TypeError):

From bea26ace74d2f01e9b34975177297691615bd0ba Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Wed, 11 Oct 2017 22:01:12 +0200
Subject: [PATCH 26/40] ignore invalid callbacks, improve and fix paydirekt
 provider, test against invalid callback (which seems to appear)

---
 payments/paydirekt/__init__.py       | 36 +++++++++++++++-------------
 payments/paydirekt/test_paydirekt.py | 18 +++++++++++++-
 2 files changed, 36 insertions(+), 18 deletions(-)

diff --git a/payments/paydirekt/__init__.py b/payments/paydirekt/__init__.py
index 8576a09fa..daf0be382 100644
--- a/payments/paydirekt/__init__.py
+++ b/payments/paydirekt/__init__.py
@@ -7,8 +7,6 @@
     from urllib.error import URLError
     from urllib.parse import urlencode
 else:
-    # for hmac
-    import hashlib
     # Fall back to Python 2's urllib2
     from urllib2 import URLError
     from urllib import urlencode
@@ -21,6 +19,8 @@
 from base64 import urlsafe_b64encode, urlsafe_b64decode
 import os
 import hmac
+# for hmac and hashed email
+import hashlib
 import simplejson as json
 import time
 import logging
@@ -94,11 +94,12 @@ class PaydirektProvider(BasicProvider):
 
 
     def __init__(self, api_key, secret, endpoint="https://api.sandbox.paydirekt.de", \
-                 overcapture=False, **kwargs):
+                 overcapture=False, default_carttype="PHYSICAL", **kwargs):
         self.secret_b64 = secret.encode('utf-8')
         self.api_key = api_key
         self.endpoint = endpoint
         self.overcapture = overcapture
+        self.default_carttype = default_carttype
         self.updating_token_lock = threading.Lock()
         super(PaydirektProvider, self).__init__(**kwargs)
 
@@ -108,10 +109,7 @@ def retrieve_oauth_token(self):
         nonce = urlsafe_b64encode(os.urandom(48))
         date_now = dt.utcnow()
         bytessign = token_uuid+b":"+date_now.strftime("%Y%m%d%H%M%S").encode('utf-8')+b":"+self.api_key.encode('utf-8')+b":"+nonce
-        if six.PY3:
-            h_temp = hmac.new(urlsafe_b64decode(self.secret_b64), msg=bytessign, digestmod="sha256")
-        else:
-            h_temp = hmac.new(urlsafe_b64decode(self.secret_b64), msg=bytessign, digestmod=hashlib.sha256)
+        h_temp = hmac.new(urlsafe_b64decode(self.secret_b64), msg=bytessign, digestmod=hashlib.sha256)
 
         header = PaydirektProvider.header_default.copy()
         header["X-Auth-Key"] = self.api_key
@@ -171,7 +169,7 @@ def get_form(self, payment, data=None):
         self.check_and_update_token()
         headers = PaydirektProvider.header_default.copy()
         headers["Authorization"] = "Bearer %s" % self.access_token
-        # email_hash = sha256(payment.billing_email.encode("utf-8")).digest())
+        email_hash = hashlib.sha256(payment.billing_email.encode("utf-8")).digest()
         body = {
             "type": "ORDER" if not self._capture else "DIRECT_SALE",
             "totalAmount": payment.total,
@@ -179,22 +177,21 @@ def get_form(self, payment, data=None):
             "orderAmount": payment.total - payment.delivery,
             "currency": payment.currency,
             "refundLimit": 110,
-            #"items": getattr(payment, "items", None),
-            #"shoppingCartType": getattr(payment, "carttype", None),
-            #"deliveryType": getattr(payment, "deliverytype", None),
+            "shoppingCartType": getattr(payment, "carttype", self.default_carttype),
             # payment id can repeat if different shop systems are used
-            "merchantOrderReferenceNumber": "%s:%s" % (hex(int(time.time())), payment.id),
+            "merchantOrderReferenceNumber": "%s:%s" % (hex(int(time.time()))[2:], payment.id),
             "redirectUrlAfterSuccess": payment.get_success_url(),
             "redirectUrlAfterCancellation": payment.get_failure_url(),
             "redirectUrlAfterRejection": payment.get_failure_url(),
             "redirectUrlAfterAgeVerificationFailure": payment.get_failure_url(),
             "callbackUrlStatusUpdates": self.get_return_url(payment),
-            #"sha256hashedEmailAddress": str(urlsafe_b64encode(email_hash), 'ascii'),
-            "minimumAge": getattr(payment, "minimumage", None),
-            #"note": payment.message[0:37]
-
+            # email sent anyway (shipping)
+            "sha256hashedEmailAddress": str(urlsafe_b64encode(email_hash), 'ascii'),
+            "minimumAge": getattr(payment, "minimumage", None)
         }
-        if self.overcapture and body["type"] == "ORDER":
+        if body["type"] == "DIRECT_SALE":
+            body["note"] = payment.description[:37]
+        if self.overcapture and body["type"] in ["ORDER", "ORDER_SECURED"]:
             body["overcapture"] = True
 
         shipping = payment.get_shipping_address()
@@ -234,6 +231,11 @@ def process_data(self, payment, request):
         except (ValueError, TypeError):
             logger.error("paydirekt returned unparseable object")
             return HttpResponseForbidden('FAILED')
+        # ignore such requests
+        # they have no value and may break things
+        # they are maybe copies or existence checks
+        if not "checkoutId" in results:
+            return HttpResponse('OK')
         if not payment.transaction_id:
             payment.transaction_id = results["checkoutId"]
             payment.save()
diff --git a/payments/paydirekt/test_paydirekt.py b/payments/paydirekt/test_paydirekt.py
index 9044ddd02..7e5735573 100644
--- a/payments/paydirekt/test_paydirekt.py
+++ b/payments/paydirekt/test_paydirekt.py
@@ -16,6 +16,17 @@
 API_KEY = '87dbc6cd-91d2-4574-bcb5-2aaaf924386d'
 SECRET = '9Tth0qty_9zplTyY0d_QbHYvKM4iSngjoipWO6VxAao='
 
+sample_request_paydirekt = {'refundLimit': 110, 'orderAmount': Decimal('9.00'),
+        'shippingAddress': {'addresseeGivenName': 'fooo', 'emailAddress': 'test@test.de', 'addresseeLastName': 'noch ein test', 'city': 'M\xc3\xbcnchen', 'street': 'fooo 23', 'zip': '23233', 'streetNr': '23', 'countryCode': 'DE'},
+        'type': 'DIRECT_SALE',
+        'callbackUrlStatusUpdates': 'https://example.com/payments/process/13119ad6-1df2-49e1-a719-a26225b9bc44/',
+        'currency': 'EUR', 'totalAmount': Decimal('9.00'),
+        'merchantOrderReferenceNumber': '59dbfc86:35',
+        'redirectUrlAfterRejection': 'https://example.com/failure/',
+        'redirectUrlAfterAgeVerificationFailure': 'https://example.com/failure/',
+        'redirectUrlAfterSuccess': 'https://example.com/success/',
+        'redirectUrlAfterCancellation': 'https://example.com/failure/'}
+
 directsale_open_data = {
   "checkoutId" : "6be6a80d-ef67-47c8-a5bd-2461d11da24c",
   "merchantOrderReferenceNumber" : "order-A12223412",
@@ -257,7 +268,7 @@
   }
 }
 
-Payment = create_test_payment(variant=VARIANT, currency='EUR')
+Payment = create_test_payment(variant=VARIANT, currency='EUR', carttype=None)
 
 
 class TestPaydirektProvider(TestCase):
@@ -268,6 +279,11 @@ def test_direct_sale_response(self):
         provider = PaydirektProvider(API_KEY, SECRET)
 
         request = MagicMock()
+        # real request (private data replaced) encountered, should not error and still be in waiting state
+        request.body = json.dumps(sample_request_paydirekt)
+        response = provider.process_data(payment, request)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(payment.status, PaymentStatus.WAITING)
         request.body = json.dumps(directsale_open_data)
         response = provider.process_data(payment, request)
         self.assertEqual(response.status_code, 200)

From 281bae1ebed9ea5faea6d84f8d1cf1496d2cd630 Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Thu, 12 Oct 2017 03:00:08 +0200
Subject: [PATCH 27/40] fix refund all

---
 payments/paydirekt/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/payments/paydirekt/__init__.py b/payments/paydirekt/__init__.py
index daf0be382..fd2f0a49d 100644
--- a/payments/paydirekt/__init__.py
+++ b/payments/paydirekt/__init__.py
@@ -303,7 +303,7 @@ def capture(self, payment, amount=None, final=True):
 
     def refund(self, payment, amount=None):
         if not amount:
-            amount = payment.total
+            amount = payment.captured_amount
         self.check_and_update_token()
         header = PaydirektProvider.header_default.copy()
         header["Authorization"] = "Bearer %s" % self.access_token

From 386cc30b9707b8ab3bc32d394a847669d1f1a8ac Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Thu, 12 Oct 2017 03:03:51 +0200
Subject: [PATCH 28/40] set refund limit to 100%

---
 payments/paydirekt/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/payments/paydirekt/__init__.py b/payments/paydirekt/__init__.py
index fd2f0a49d..70aa96fc9 100644
--- a/payments/paydirekt/__init__.py
+++ b/payments/paydirekt/__init__.py
@@ -176,7 +176,7 @@ def get_form(self, payment, data=None):
             "shippingAmount": payment.delivery,
             "orderAmount": payment.total - payment.delivery,
             "currency": payment.currency,
-            "refundLimit": 110,
+            "refundLimit": 100,
             "shoppingCartType": getattr(payment, "carttype", self.default_carttype),
             # payment id can repeat if different shop systems are used
             "merchantOrderReferenceNumber": "%s:%s" % (hex(int(time.time()))[2:], payment.id),

From 5019c9e012ba9b56ee33aa088116e85c73421d75 Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Thu, 12 Oct 2017 05:12:08 +0200
Subject: [PATCH 29/40] allow responses without transaction_id and delay in
 case none is available, TODO: fix bug

---
 payments/paydirekt/__init__.py       | 10 +++++-----
 payments/paydirekt/test_paydirekt.py | 18 +++++++++++-------
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/payments/paydirekt/__init__.py b/payments/paydirekt/__init__.py
index 70aa96fc9..23c32ddce 100644
--- a/payments/paydirekt/__init__.py
+++ b/payments/paydirekt/__init__.py
@@ -223,6 +223,8 @@ def get_form(self, payment, data=None):
         json_response = json.loads(response.text, use_decimal=True)
 
         check_response(response, json_response)
+        payment.transaction_id = json_response["checkoutId"]
+        payment.save()
         raise RedirectNeeded(json_response["_links"]["approve"]["href"])
 
     def process_data(self, payment, request):
@@ -231,12 +233,10 @@ def process_data(self, payment, request):
         except (ValueError, TypeError):
             logger.error("paydirekt returned unparseable object")
             return HttpResponseForbidden('FAILED')
-        # ignore such requests
-        # they have no value and may break things
-        # they are maybe copies or existence checks
-        if not "checkoutId" in results:
-            return HttpResponse('OK')
         if not payment.transaction_id:
+            # delay
+            if not "checkoutId" in results:
+                return HttpResponseServerError('no transaction_id')
             payment.transaction_id = results["checkoutId"]
             payment.save()
         if "checkoutStatus" in results:
diff --git a/payments/paydirekt/test_paydirekt.py b/payments/paydirekt/test_paydirekt.py
index 7e5735573..f265bf665 100644
--- a/payments/paydirekt/test_paydirekt.py
+++ b/payments/paydirekt/test_paydirekt.py
@@ -282,7 +282,7 @@ def test_direct_sale_response(self):
         # real request (private data replaced) encountered, should not error and still be in waiting state
         request.body = json.dumps(sample_request_paydirekt)
         response = provider.process_data(payment, request)
-        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.status_code, 500)
         self.assertEqual(payment.status, PaymentStatus.WAITING)
         request.body = json.dumps(directsale_open_data)
         response = provider.process_data(payment, request)
@@ -338,7 +338,8 @@ def return_url_data(url, *args, **kwargs):
         self.assertEqual(cm.exception.args[0], "https://paydirekt.de/checkout/#/checkout/6be6a80d-ef67-47c8-a5bd-2461d11da24c")
 
     @patch("requests.post")
-    def test_capture_refund(self, mocked_post):
+    @patch("payments.core.provider_factory")
+    def test_capture_refund(self, mocked_post, mocked_factory):
         payment = Payment(minimumage=0)
         provider = PaydirektProvider(API_KEY, SECRET, capture=False)
         request = MagicMock()
@@ -354,18 +355,21 @@ def return_url_data(url, *args, **kwargs):
                 response.text = json.dumps(capture_response)
             elif url == provider.path_refund.format(provider.endpoint, payment.transaction_id):
                 response.text = json.dumps(refund_response)
+            elif url == provider.path_close.format(provider.endpoint, payment.transaction_id):
+                response.text = json.dumps(order_close_data)
             else:
-                raise
+                raise Exception(url)
             return response
         mocked_post.side_effect = return_url_data
+        mocked_factory.side_effect = lambda x: provider
 
-        ret = provider.capture(payment)
+        ret = payment.capture()
         self.assertEqual(ret, Decimal(100))
         self.assertEqual(payment.status, PaymentStatus.PREAUTH)
-        self.assertEqual(payment.captured_amount, Decimal("0.0"))
+        self.assertEqual(payment.captured_amount, Decimal("100.0"))
 
-        ret = provider.refund(payment)
-        self.assertEqual(ret, Decimal(100))
+        ret = payment.refund()
+        self.assertEqual(ret, Decimal(0))
         self.assertEqual(payment.status, PaymentStatus.PREAUTH)
         self.assertEqual(payment.captured_amount, Decimal("0.0"))
 

From 8e06926a000ac45ce334377710a43c6a47b9fa6e Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Thu, 12 Oct 2017 05:24:51 +0200
Subject: [PATCH 30/40] fix tests, bug only existed there

---
 payments/paydirekt/__init__.py       |  1 +
 payments/paydirekt/test_paydirekt.py | 17 ++++++++---------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/payments/paydirekt/__init__.py b/payments/paydirekt/__init__.py
index 23c32ddce..f971da073 100644
--- a/payments/paydirekt/__init__.py
+++ b/payments/paydirekt/__init__.py
@@ -283,6 +283,7 @@ def process_data(self, payment, request):
     def capture(self, payment, amount=None, final=True):
         if not amount:
             amount = payment.total
+        if not amount: raise Exception(self.total)
         if self.overcapture and amount > payment.total*Decimal("1.1"):
             return None
         elif not self.overcapture and amount > payment.total:
diff --git a/payments/paydirekt/test_paydirekt.py b/payments/paydirekt/test_paydirekt.py
index f265bf665..27c5cc981 100644
--- a/payments/paydirekt/test_paydirekt.py
+++ b/payments/paydirekt/test_paydirekt.py
@@ -338,8 +338,7 @@ def return_url_data(url, *args, **kwargs):
         self.assertEqual(cm.exception.args[0], "https://paydirekt.de/checkout/#/checkout/6be6a80d-ef67-47c8-a5bd-2461d11da24c")
 
     @patch("requests.post")
-    @patch("payments.core.provider_factory")
-    def test_capture_refund(self, mocked_post, mocked_factory):
+    def test_capture_refund(self, mocked_post):
         payment = Payment(minimumage=0)
         provider = PaydirektProvider(API_KEY, SECRET, capture=False)
         request = MagicMock()
@@ -361,18 +360,18 @@ def return_url_data(url, *args, **kwargs):
                 raise Exception(url)
             return response
         mocked_post.side_effect = return_url_data
-        mocked_factory.side_effect = lambda x: provider
 
-        ret = payment.capture()
+        ret = provider.capture(payment)
         self.assertEqual(ret, Decimal(100))
         self.assertEqual(payment.status, PaymentStatus.PREAUTH)
-        self.assertEqual(payment.captured_amount, Decimal("100.0"))
-
-        ret = payment.refund()
-        self.assertEqual(ret, Decimal(0))
-        self.assertEqual(payment.status, PaymentStatus.PREAUTH)
         self.assertEqual(payment.captured_amount, Decimal("0.0"))
 
+        payment.captured_amount = Decimal(100)
+        ret = provider.refund(payment)
+        self.assertEqual(ret, Decimal(100))
+        self.assertEqual(payment.status, PaymentStatus.REFUNDED)
+        self.assertEqual(payment.captured_amount, Decimal("100.0"))
+
     @patch("requests.post")
     @patch("requests.get")
     def test_refund_fail(self, mocked_get, mocked_post):

From 5ca8e1a85bb36b9d2428ec5b26ec9147612d560a Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Thu, 12 Oct 2017 13:04:24 +0200
Subject: [PATCH 31/40] ignore invalid requests again

---
 payments/paydirekt/__init__.py       | 7 ++++---
 payments/paydirekt/test_paydirekt.py | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/payments/paydirekt/__init__.py b/payments/paydirekt/__init__.py
index f971da073..3359c4378 100644
--- a/payments/paydirekt/__init__.py
+++ b/payments/paydirekt/__init__.py
@@ -224,6 +224,7 @@ def get_form(self, payment, data=None):
 
         check_response(response, json_response)
         payment.transaction_id = json_response["checkoutId"]
+        #payment.attrs = json_response["_links"]
         payment.save()
         raise RedirectNeeded(json_response["_links"]["approve"]["href"])
 
@@ -233,10 +234,10 @@ def process_data(self, payment, request):
         except (ValueError, TypeError):
             logger.error("paydirekt returned unparseable object")
             return HttpResponseForbidden('FAILED')
+        # ignore invalid requests
+        if not "checkoutId" in results:
+            return HttpResponse('OK')
         if not payment.transaction_id:
-            # delay
-            if not "checkoutId" in results:
-                return HttpResponseServerError('no transaction_id')
             payment.transaction_id = results["checkoutId"]
             payment.save()
         if "checkoutStatus" in results:
diff --git a/payments/paydirekt/test_paydirekt.py b/payments/paydirekt/test_paydirekt.py
index 27c5cc981..c98ccf37f 100644
--- a/payments/paydirekt/test_paydirekt.py
+++ b/payments/paydirekt/test_paydirekt.py
@@ -282,7 +282,7 @@ def test_direct_sale_response(self):
         # real request (private data replaced) encountered, should not error and still be in waiting state
         request.body = json.dumps(sample_request_paydirekt)
         response = provider.process_data(payment, request)
-        self.assertEqual(response.status_code, 500)
+        self.assertEqual(response.status_code, 200)
         self.assertEqual(payment.status, PaymentStatus.WAITING)
         request.body = json.dumps(directsale_open_data)
         response = provider.process_data(payment, request)

From 9316a12d316da0c9265e910ca0582b2d38c23652 Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Thu, 12 Oct 2017 17:02:10 +0200
Subject: [PATCH 32/40] use umlauts instead escape sequence

---
 payments/paydirekt/test_paydirekt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/payments/paydirekt/test_paydirekt.py b/payments/paydirekt/test_paydirekt.py
index c98ccf37f..2d8128761 100644
--- a/payments/paydirekt/test_paydirekt.py
+++ b/payments/paydirekt/test_paydirekt.py
@@ -17,7 +17,7 @@
 SECRET = '9Tth0qty_9zplTyY0d_QbHYvKM4iSngjoipWO6VxAao='
 
 sample_request_paydirekt = {'refundLimit': 110, 'orderAmount': Decimal('9.00'),
-        'shippingAddress': {'addresseeGivenName': 'fooo', 'emailAddress': 'test@test.de', 'addresseeLastName': 'noch ein test', 'city': 'M\xc3\xbcnchen', 'street': 'fooo 23', 'zip': '23233', 'streetNr': '23', 'countryCode': 'DE'},
+        'shippingAddress': {'addresseeGivenName': 'fooo', 'emailAddress': 'test@test.de', 'addresseeLastName': 'noch ein test', 'city': 'München', 'street': 'fooo 23', 'zip': '23233', 'streetNr': '23', 'countryCode': 'DE'},
         'type': 'DIRECT_SALE',
         'callbackUrlStatusUpdates': 'https://example.com/payments/process/13119ad6-1df2-49e1-a719-a26225b9bc44/',
         'currency': 'EUR', 'totalAmount': Decimal('9.00'),

From fc9551eee037c001ce27f8dbad698f0c4b866c59 Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Thu, 12 Oct 2017 17:51:32 +0200
Subject: [PATCH 33/40] update documentation

---
 payments/core.py   | 2 +-
 payments/models.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/payments/core.py b/payments/core.py
index 493a7c680..bc3a60be2 100644
--- a/payments/core.py
+++ b/payments/core.py
@@ -101,7 +101,7 @@ def release(self, payment):
         raise NotImplementedError()
 
     def refund(self, payment, amount=None):
-        ''' Refund payment, return amount which was refunded '''
+        ''' Refund payment, return amount which was refunded or None '''
         raise NotImplementedError()
 
 
diff --git a/payments/models.py b/payments/models.py
index 0903bd516..dc23a7a09 100644
--- a/payments/models.py
+++ b/payments/models.py
@@ -164,7 +164,7 @@ def release(self):
         self.change_status(PaymentStatus.REFUNDED)
 
     def refund(self, amount=None):
-        ''' Refund payment, return amount which was refunded '''
+        ''' Refund payment, return amount which was refunded or None '''
         if self.status != PaymentStatus.CONFIRMED:
             raise ValueError(
                 'Only charged payments can be refunded.')

From 0ec58160b3e61b46ba160464419e3fd9fbce05b2 Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Thu, 12 Oct 2017 18:59:18 +0200
Subject: [PATCH 34/40] replace extract_streetnr with split_streetnr

---
 payments/utils.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/payments/utils.py b/payments/utils.py
index 210d550b3..d2b7eed02 100644
--- a/payments/utils.py
+++ b/payments/utils.py
@@ -16,12 +16,12 @@ def get_year_choices():
     return [('', _('Year'))] + year_choices
 
 _extract_streetnr = re.compile(r"([0-9]+)\s*$")
-def extract_streetnr(address, fallback=None):
-    ret = _extract_streetnr.findall(address[-15:])
+def split_streetnr(address, fallback=None):
+    ret = _extract_streetnr.search(address[-15:])
     if ret:
-        return ret[0]
+        return address[:(ret.start()-15)].strip(), ret.group(0)
     else:
-        return fallback
+        return address.strip(), fallback
 
 def getter_prefixed_address(prefix):
     """ create getter for prefixed address format """

From 42c62e54b3425970aa615a3522b166a1aad838ea Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Thu, 12 Oct 2017 19:04:45 +0200
Subject: [PATCH 35/40] use split_streetnr instead extract_streetnr (cleaner
 results)

---
 payments/paydirekt/__init__.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/payments/paydirekt/__init__.py b/payments/paydirekt/__init__.py
index 3359c4378..4ced5eaea 100644
--- a/payments/paydirekt/__init__.py
+++ b/payments/paydirekt/__init__.py
@@ -32,7 +32,7 @@
 
 from .. import PaymentError, PaymentStatus, RedirectNeeded
 from ..core import BasicProvider
-from ..utils import extract_streetnr
+from ..utils import split_streetnr
 
 logger = logging.getLogger(__name__)
 
@@ -194,6 +194,7 @@ def get_form(self, payment, data=None):
         if self.overcapture and body["type"] in ["ORDER", "ORDER_SECURED"]:
             body["overcapture"] = True
 
+        street, streetnr = split_streetnr(shipping["address_1"], "0")
         shipping = payment.get_shipping_address()
 
         shipping = {
@@ -201,8 +202,8 @@ def get_form(self, payment, data=None):
             "addresseeLastName": shipping["last_name"],
             "company": shipping.get("company", None),
             "additionalAddressInformation": shipping["address_2"],
-            "street": shipping["address_1"],
-            "streetNr": extract_streetnr(shipping["address_1"], "0"),
+            "street": street,
+            "streetNr": streetnr,
             "zip": shipping["postcode"],
             "city": shipping["city"],
             "countryCode": shipping["country_code"],

From 91923e40d9b49bf56ef83562e94767abec91e327 Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Thu, 12 Oct 2017 19:10:23 +0200
Subject: [PATCH 36/40] fix use before create

---
 payments/paydirekt/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/payments/paydirekt/__init__.py b/payments/paydirekt/__init__.py
index 4ced5eaea..f2a801ddb 100644
--- a/payments/paydirekt/__init__.py
+++ b/payments/paydirekt/__init__.py
@@ -194,8 +194,8 @@ def get_form(self, payment, data=None):
         if self.overcapture and body["type"] in ["ORDER", "ORDER_SECURED"]:
             body["overcapture"] = True
 
-        street, streetnr = split_streetnr(shipping["address_1"], "0")
         shipping = payment.get_shipping_address()
+        street, streetnr = split_streetnr(shipping["address_1"], "0")
 
         shipping = {
             "addresseeGivenName": shipping["first_name"],

From 598ff9a27ec899a0b9e0278c85b4f25b3c193d67 Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Tue, 17 Oct 2017 12:52:32 +0200
Subject: [PATCH 37/40] protect against lingering connections

---
 payments/paydirekt/__init__.py | 35 +++++++++++++++++++++++++++-------
 1 file changed, 28 insertions(+), 7 deletions(-)

diff --git a/payments/paydirekt/__init__.py b/payments/paydirekt/__init__.py
index f2a801ddb..aebdfce08 100644
--- a/payments/paydirekt/__init__.py
+++ b/payments/paydirekt/__init__.py
@@ -27,6 +27,7 @@
 import threading
 
 import requests
+from requests.exceptions import Timeout
 from django.http import HttpResponseRedirect, HttpResponseForbidden, HttpResponseServerError, HttpResponse
 from django.conf import settings
 
@@ -124,19 +125,27 @@ def retrieve_oauth_token(self):
             "grantType" : "api_key",
             "randomNonce" : str(nonce, "ascii") if six.PY3 else nonce
         }
-        response = requests.post(self.path_token.format(self.endpoint), data=json.dumps(body, use_decimal=True), headers=header)
+        response = requests.post(self.path_token.format(self.endpoint), data=json.dumps(body, use_decimal=True), headers=header, timeout=20)
         token_raw = json.loads(response.text, use_decimal=True)
         check_response(response, token_raw)
 
         self.access_token = token_raw["access_token"]
         self.expires_in = date_now+timedelta(seconds=token_raw["expires_in"])
 
-    def check_and_update_token(self):
+    def check_and_update_token(self, times=0):
         """ Check if token exists or has expired, renew it in this case """
         self.updating_token_lock.acquire()
         try:
             if not self.expires_in or self.expires_in >= dt.utcnow()-timedelta(seconds=3):
                 self.retrieve_oauth_token()
+        except Timeout:
+            if times < 3:
+                self.updating_token_lock.release()
+                time.sleep(3)
+                return self.check_and_update_token(times+1)
+            else:
+                self.updating_token_lock.release()
+                raise PaymentError("Timeout")
         except Exception as exc:
             self.updating_token_lock.release()
             raise exc
@@ -158,6 +167,9 @@ def _retrieve_amount(self, url):
         ret = requests.get(url)
         try:
             results = json.loads(ret.text, use_decimal=True)
+        except Timeout:
+            logger.error("paydirekt had timeout")
+            return None
         except (ValueError, TypeError):
             logger.error("paydirekt returned unparseable object")
             return None
@@ -220,7 +232,10 @@ def get_form(self, payment, data=None):
         if len(items) > 0:
             body["items"] = items
 
-        response = requests.post(self.path_checkout.format(self.endpoint), data=json.dumps(body, use_decimal=True), headers=headers)
+        try:
+            response = requests.post(self.path_checkout.format(self.endpoint), data=json.dumps(body, use_decimal=True), headers=headers, timeout=20)
+        except Timeout:
+            raise PaymentError("Timeout")
         json_response = json.loads(response.text, use_decimal=True)
 
         check_response(response, json_response)
@@ -298,8 +313,11 @@ def capture(self, payment, amount=None, final=True):
             "finalCapture": final,
             "callbackUrlStatusUpdates": self.get_return_url(payment)
         }
-        response = requests.post(self.path_capture.format(self.endpoint, payment.transaction_id), \
-                                 data=json.dumps(body, use_decimal=True), headers=header)
+        try:
+            response = requests.post(self.path_capture.format(self.endpoint, payment.transaction_id), \
+                                     data=json.dumps(body, use_decimal=True), headers=header, timeout=20)
+        except Timeout:
+            raise PaymentError("Timeout")
         json_response = json.loads(response.text, use_decimal=True)
         check_response(response, json_response)
         return amount
@@ -314,8 +332,11 @@ def refund(self, payment, amount=None):
             "amount": amount,
             "callbackUrlStatusUpdates": self.get_return_url(payment)
         }
-        response = requests.post(self.path_refund.format(self.endpoint, payment.transaction_id), \
-                                 data=json.dumps(body, use_decimal=True), headers=header)
+        try:
+            response = requests.post(self.path_refund.format(self.endpoint, payment.transaction_id), \
+                                     data=json.dumps(body, use_decimal=True), headers=header, timeout=20)
+        except Timeout:
+            raise PaymentError("Timeout")
         json_response = json.loads(response.text, use_decimal=True)
         check_response(response, json_response)
         if payment.status == PaymentStatus.PREAUTH and amount == payment.captured_amount:

From fae58f2717c8712f0f9ebbfec6f8a160e3751efb Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Tue, 17 Oct 2017 12:55:24 +0200
Subject: [PATCH 38/40] fix generation of tokens

---
 payments/paydirekt/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/payments/paydirekt/__init__.py b/payments/paydirekt/__init__.py
index aebdfce08..379529ba2 100644
--- a/payments/paydirekt/__init__.py
+++ b/payments/paydirekt/__init__.py
@@ -136,7 +136,7 @@ def check_and_update_token(self, times=0):
         """ Check if token exists or has expired, renew it in this case """
         self.updating_token_lock.acquire()
         try:
-            if not self.expires_in or self.expires_in >= dt.utcnow()-timedelta(seconds=3):
+            if not self.expires_in or self.expires_in <= dt.utcnow()-timedelta(seconds=3):
                 self.retrieve_oauth_token()
         except Timeout:
             if times < 3:

From 2dfc99451b72b9ed0eef4f3722a658ad70828477 Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Tue, 17 Oct 2017 13:35:14 +0200
Subject: [PATCH 39/40] improve ordering, Timeouts, fix 3 second hole

---
 payments/paydirekt/__init__.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/payments/paydirekt/__init__.py b/payments/paydirekt/__init__.py
index 379529ba2..724d622fe 100644
--- a/payments/paydirekt/__init__.py
+++ b/payments/paydirekt/__init__.py
@@ -130,13 +130,14 @@ def retrieve_oauth_token(self):
         check_response(response, token_raw)
 
         self.access_token = token_raw["access_token"]
-        self.expires_in = date_now+timedelta(seconds=token_raw["expires_in"])
+        # expires_in with 5 seconds less, enough time for next command
+        self.expires_in = date_now+timedelta(seconds=token_raw["expires_in"]-5)
 
     def check_and_update_token(self, times=0):
         """ Check if token exists or has expired, renew it in this case """
         self.updating_token_lock.acquire()
         try:
-            if not self.expires_in or self.expires_in <= dt.utcnow()-timedelta(seconds=3):
+            if not self.expires_in or self.expires_in <= dt.utcnow():
                 self.retrieve_oauth_token()
         except Timeout:
             if times < 3:
@@ -164,12 +165,13 @@ def _prepare_items(self, payment):
         return items
 
     def _retrieve_amount(self, url):
-        ret = requests.get(url)
         try:
-            results = json.loads(ret.text, use_decimal=True)
+            ret = requests.get(url, timeout=20)
         except Timeout:
             logger.error("paydirekt had timeout")
             return None
+        try:
+            results = json.loads(ret.text, use_decimal=True)
         except (ValueError, TypeError):
             logger.error("paydirekt returned unparseable object")
             return None
@@ -178,7 +180,6 @@ def _retrieve_amount(self, url):
     def get_form(self, payment, data=None):
         if not payment.id:
             payment.save()
-        self.check_and_update_token()
         headers = PaydirektProvider.header_default.copy()
         headers["Authorization"] = "Bearer %s" % self.access_token
         email_hash = hashlib.sha256(payment.billing_email.encode("utf-8")).digest()
@@ -232,6 +233,7 @@ def get_form(self, payment, data=None):
         if len(items) > 0:
             body["items"] = items
 
+        self.check_and_update_token()
         try:
             response = requests.post(self.path_checkout.format(self.endpoint), data=json.dumps(body, use_decimal=True), headers=headers, timeout=20)
         except Timeout:
@@ -305,7 +307,6 @@ def capture(self, payment, amount=None, final=True):
             return None
         elif not self.overcapture and amount > payment.total:
             return None
-        self.check_and_update_token()
         header = PaydirektProvider.header_default.copy()
         header["Authorization"] = "Bearer %s" % self.access_token
         body = {
@@ -313,6 +314,7 @@ def capture(self, payment, amount=None, final=True):
             "finalCapture": final,
             "callbackUrlStatusUpdates": self.get_return_url(payment)
         }
+        self.check_and_update_token()
         try:
             response = requests.post(self.path_capture.format(self.endpoint, payment.transaction_id), \
                                      data=json.dumps(body, use_decimal=True), headers=header, timeout=20)
@@ -325,13 +327,13 @@ def capture(self, payment, amount=None, final=True):
     def refund(self, payment, amount=None):
         if not amount:
             amount = payment.captured_amount
-        self.check_and_update_token()
         header = PaydirektProvider.header_default.copy()
         header["Authorization"] = "Bearer %s" % self.access_token
         body = {
             "amount": amount,
             "callbackUrlStatusUpdates": self.get_return_url(payment)
         }
+        self.check_and_update_token()
         try:
             response = requests.post(self.path_refund.format(self.endpoint, payment.transaction_id), \
                                      data=json.dumps(body, use_decimal=True), headers=header, timeout=20)

From 823028ec90a20efff7d47eab328c6d72d99d6ebd Mon Sep 17 00:00:00 2001
From: alex <devkral@web.de>
Date: Tue, 17 Oct 2017 20:35:37 +0200
Subject: [PATCH 40/40] fix token generation

---
 payments/paydirekt/__init__.py | 51 ++++++++++------------------------
 1 file changed, 14 insertions(+), 37 deletions(-)

diff --git a/payments/paydirekt/__init__.py b/payments/paydirekt/__init__.py
index 724d622fe..c488b21d9 100644
--- a/payments/paydirekt/__init__.py
+++ b/payments/paydirekt/__init__.py
@@ -24,7 +24,6 @@
 import simplejson as json
 import time
 import logging
-import threading
 
 import requests
 from requests.exceptions import Timeout
@@ -101,7 +100,6 @@ def __init__(self, api_key, secret, endpoint="https://api.sandbox.paydirekt.de",
         self.endpoint = endpoint
         self.overcapture = overcapture
         self.default_carttype = default_carttype
-        self.updating_token_lock = threading.Lock()
         super(PaydirektProvider, self).__init__(**kwargs)
 
     def retrieve_oauth_token(self):
@@ -125,33 +123,15 @@ def retrieve_oauth_token(self):
             "grantType" : "api_key",
             "randomNonce" : str(nonce, "ascii") if six.PY3 else nonce
         }
-        response = requests.post(self.path_token.format(self.endpoint), data=json.dumps(body, use_decimal=True), headers=header, timeout=20)
-        token_raw = json.loads(response.text, use_decimal=True)
-        check_response(response, token_raw)
-
-        self.access_token = token_raw["access_token"]
-        # expires_in with 5 seconds less, enough time for next command
-        self.expires_in = date_now+timedelta(seconds=token_raw["expires_in"]-5)
-
-    def check_and_update_token(self, times=0):
-        """ Check if token exists or has expired, renew it in this case """
-        self.updating_token_lock.acquire()
         try:
-            if not self.expires_in or self.expires_in <= dt.utcnow():
-                self.retrieve_oauth_token()
+            response = requests.post(self.path_token.format(self.endpoint), data=json.dumps(body, use_decimal=True), headers=header, timeout=20)
         except Timeout:
-            if times < 3:
-                self.updating_token_lock.release()
-                time.sleep(3)
-                return self.check_and_update_token(times+1)
-            else:
-                self.updating_token_lock.release()
-                raise PaymentError("Timeout")
-        except Exception as exc:
-            self.updating_token_lock.release()
-            raise exc
-        self.updating_token_lock.release()
+            raise PaymentError("Timeout")
 
+        token_raw = json.loads(response.text, use_decimal=True)
+        check_response(response, token_raw)
+
+        return token_raw["access_token"]
 
     def _prepare_items(self, payment):
         items = []
@@ -181,7 +161,7 @@ def get_form(self, payment, data=None):
         if not payment.id:
             payment.save()
         headers = PaydirektProvider.header_default.copy()
-        headers["Authorization"] = "Bearer %s" % self.access_token
+        headers["Authorization"] = "Bearer %s" % self.retrieve_oauth_token()
         email_hash = hashlib.sha256(payment.billing_email.encode("utf-8")).digest()
         body = {
             "type": "ORDER" if not self._capture else "DIRECT_SALE",
@@ -233,7 +213,6 @@ def get_form(self, payment, data=None):
         if len(items) > 0:
             body["items"] = items
 
-        self.check_and_update_token()
         try:
             response = requests.post(self.path_checkout.format(self.endpoint), data=json.dumps(body, use_decimal=True), headers=headers, timeout=20)
         except Timeout:
@@ -308,13 +287,12 @@ def capture(self, payment, amount=None, final=True):
         elif not self.overcapture and amount > payment.total:
             return None
         header = PaydirektProvider.header_default.copy()
-        header["Authorization"] = "Bearer %s" % self.access_token
+        header["Authorization"] = "Bearer %s" % self.retrieve_oauth_token()
         body = {
             "amount": amount,
             "finalCapture": final,
             "callbackUrlStatusUpdates": self.get_return_url(payment)
         }
-        self.check_and_update_token()
         try:
             response = requests.post(self.path_capture.format(self.endpoint, payment.transaction_id), \
                                      data=json.dumps(body, use_decimal=True), headers=header, timeout=20)
@@ -328,12 +306,11 @@ def refund(self, payment, amount=None):
         if not amount:
             amount = payment.captured_amount
         header = PaydirektProvider.header_default.copy()
-        header["Authorization"] = "Bearer %s" % self.access_token
+        header["Authorization"] = "Bearer %s" % self.retrieve_oauth_token()
         body = {
             "amount": amount,
             "callbackUrlStatusUpdates": self.get_return_url(payment)
         }
-        self.check_and_update_token()
         try:
             response = requests.post(self.path_refund.format(self.endpoint, payment.transaction_id), \
                                      data=json.dumps(body, use_decimal=True), headers=header, timeout=20)
@@ -344,9 +321,9 @@ def refund(self, payment, amount=None):
         if payment.status == PaymentStatus.PREAUTH and amount == payment.captured_amount:
             # logic, elsewise multiple signals are emitted CONFIRMED -> REFUNDED
             payment.change_status(PaymentStatus.REFUNDED)
-            self.check_and_update_token()
-            response = requests.post(self.path_close.format(self.endpoint, payment.transaction_id), \
-                                 headers=header)
-            json_response = json.loads(response.text, use_decimal=True)
-            check_response(response, json_response)
+            try:
+                response = requests.post(self.path_close.format(self.endpoint, payment.transaction_id), \
+                                         headers=header)
+            except Timeout:
+                logger.error("Closing order failed")
         return amount