csrf debug false forbidden (#42)

* develop csrf debug false work #23

* refactor to start heartbeat process in test

* fix test cases

* don't raise exception on strange heartbeat message

* update coverage to 4.5.1

* save artifacts on build

* rename screenshot files

* format tox.ini

* create png artifact in test

* always copy pngs to artifacts

* set to always

* code cleanup

* fix code change

* temporary work

* deactivate cas_logfile

* add username to context, move is_authenticated flag

* cleanup and update to new python-social-auth

* ignore png in root

* fix dict attribute error

* add selenium IDE project

* use postgresql for tests because sqlite database gets locked

* add postgresql container

* fix test

.circleci/config.yml

@@ -24,10 +24,16 @@ jobs:
       #
       - run: bash bin/create_package.sh > /dev/null
       - run: pip install tox
+      - run: mkdir /tmp/artifacts
       - run: tox
       - run: pip install -r requirements.txt
       # TODO: coverage should be run in cotainer and data directory for output should be added as volume
-      - run: tox
+      - run: 
+          command: |
+            cp *.png /tmp/artifacts
+          when: always
+      - store_artifacts:
+          path: /tmp/artifacts
       - run: coverage xml
       - run: python-codacy-coverage -v -r coverage.xml
       #

.docker/config.json

@@ -0,0 +1,9 @@
+{
+	"auths": {
+		"https://index.docker.io/v1/": {}
+	},
+	"HttpHeaders": {
+		"User-Agent": "Docker-Client/18.03.0-ce (darwin)"
+	},
+	"credsStore": "osxkeychain"
+}

.gitignore

@@ -80,3 +80,7 @@ secrets/*
 *.retry
 
 .vscode
+k8s-files/ansible
+.docker/
+
+*.png

cli/tumbo-cli.py

@@ -324,15 +324,15 @@ def projects_list(self):
         print tabulate(table, headers=["Projectname", "State"])
 
     def project_create(self, name):
-        status_code, response = self._call_api(
+        status_code, _ = self._call_api(
             "/core/api/base/", method="POST", json={"name": name})
         if status_code == 201:
             print "Project %s created" % (name)
         else:
             print status_code
 
     def project_stop(self, name):
-        status_code, projects = self._call_api(
+        status_code, _ = self._call_api(
             "/core/api/base/%s/stop/" % name, method="POST")
         if status_code == 200:
             print "Project %s stopped" % name
@@ -921,7 +921,6 @@ def stop():
                 for (each_key, each_val) in conf.items(each_section):
                     # print each_val
                     # if each_val.startswith('b64:'):
-                    #    import pdb; pdb.set_trace()
                     #    each_val = standard_b64encode(each_val)
                     ini_dict[each_key.upper()] = each_val
 

diagrams/AuthenticationArchitecture.xml

@@ -0,0 +1 @@
+<mxfile userAgent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36" version="8.5.7" editor="www.draw.io" type="device"><diagram id="81b5b74f-e305-6880-c9e5-6831b1924a6f" name="Page-1">7Vzbcts2EP0aPdrDG0jq0ZbjtDPNjCdu0vQRIiEJMUWoIORLvr4ACVAkAEmURLHWuMlDhCvB3cXZs1gwI3+yfP1M4WrxhaQoG3lO+jry70ae54LQ5/+ImreqJlAVc4pT2WlT8Yh/IVnpyNo1TlHR6sgIyRhetSsTkucoYa06SCl5aXebkaz91BWcI6PiMYGZWfsXTtmiqo29aFP/G8LzhXqyG46rlilMnuaUrHP5vJHnz8o/VfMSqrnkixYLmJKXRpX/aeRPKCGs+rV8naBMyFaJrRp3v6W1XjdFOesyIAiqEc8wW8t3//3uQa6NvSl5lG+ExBhn5N++LDBDjyuYiNYXbgG8bsGWGS+5/GcKi0XZVxRmJGdSv67PyzDD85wXaCW92xnOsgnJCC0f5M+A+MvrC0bJE2q0hOUf8SRC8S8+K1TPe0aUYa67Gzk1I2JBpiSkcER39NqokpL5jMgSMfrGu8hWEEotSSsOvbAqv2xswh2Dqm7RsAcvkAOhtMN5PfdGF/yHVIddNaKrpho+JOFTes6EkCeM/ldT2eqDLmpyzqQmd2xRU5gxKdSWgsJ/1kQ1XBWluG94B9dbvW4a+a+5+PcrYmua82aKUkwFyslp+Yqqmat+hhWglGOZLBLKFmROcph92tTetu2kYRNcCfTth6i/Bqr4t+z2EzH2Jm0ErhnhVZvZ/yBCm1ZTOtx0ttpEQdY0kW8ZSLkzSOdIdZP6FBLoYjnOddiynCsFyxRlkOHntjewWUk59Q2l8K3RYUVwzorGkx9Exeax9TrlYyNfA+nD+vMf1Qq00Wo5ZDYruIR0m67F0cnM/eDC7CyFKJ4lVjtLYjSdlS2QshtBF3hDTnKk6u6xkEO5GpSnqgdZobyqaba/Yla9yDhwZbl8k2vH92X5AVHMRY2oHLPVvlu2DKy2bNsEpn0fZr+mwfk6ompAWa1BjtrY1aEbwVeESj3H2b0RAn1dzo6NcIyRAwPLv3PdzcQ7/YmTJ7mLmpuA+yjWtm2KOLLDadlBaFsKgfcGtyNw13CuCde/MArDNS5xmpYbKINTlN3WnLIJoxWrNLeBaVxq6xoetebPcrUtDmrHS8502ypzjwFMQ7FXrmsH4n4xzNDuLSyQcqtTuvGo9+sC0QxyFs8fJ9uxahZtDV+MG8OMvugVLlcZsnc3n3sKnxOGcA+XOBMy5GabwhzqNE8UHiDjVpeXNZx7nJOZuUF7vwLFwprMLLIws7gHYmbu5c+YLdbT/0rKPchT4V2NfwCY8vQs8gzC0+WptmjvRPcmSVAhopqvSPq2AYhu7bYj0PTabW4SgnGTnbjSpx/CUI6zlcM5c7nIzkSjySEUuWySD0WuuxNpP4yDNoQrAnOsa+gX+t3zWI9G+dyobUBei926J7BbG4/UrWQ28xIr403DaQg6R1YnGcR5IycXtI0sisKdhNGNwK7+pxNGk1JMYJYJMKuoBUdmDswXzR63eakj2KOnIcQ7AgjFVHoGiBY8lGjQV3gLUJwGts0ee1N/4xLqVdQhdhlxXzoQxG0iFIdgJw5o3cMYjLbCgAkinjZaPzas5GNEw8dYocmxvpUw0kaTLce/F4AmSnS9oMk4aKP7uwIUtXmariHDKDedwOXEH3o8FwNLPBdY4o+wh3jOzFTd/YT5nPC6b2VEfrFi1cO6yJbAsIV1fYjVOwszbvNisMcbHesTu3kxYPFi78KJgUjT+zje6cWM/qBfNuvFxhark1Bf6yTUHkrbf1C+K2FwSkh1rvDJYm+dE1OdDa7r9vYjQ6ePiF0pBsFfRiCMOJxpn3quFjjDqwIuMsxL9THmICcww1Hk8ytUHXCLwwHFnd/ac5wXZALtEgMAu3Ms7knd43j41KRrnvRObh4vmA/4Os4HHfmA6wSnI4ZreoF+znm/4wKz8vaK0I5yI4+IPmNx5uvc0PmAJ7/nYyi9HPl2SF93Ay9LDL8ltbwLvIJIy/v54THgZWaatdg6crRMxZZM8/6JgDbRliDdmCjSMpGuxrF7gSxLKtJCwzPhgdF+sGrf5NJYTZygktVsu+G1P/A3LHAagwDszC90xzpPxzpL7GODOj1BfBQ3Gg8DLIOHN13394mbF2hxa+zEx23eUDtXiMJum3fogCvsOX0QeAYMWEzyI8CAdoUzcodDgcCkjz3ltddswSXChckwyYe8vVkXKgyKLo7BnMhWfE8h0caLB73wlW0IcjDk+XoGQ8fOvXxlOyE5FRZjeT7WG8yFxg6j5blS0QwDNocS2o64hLSGwpCT0xrceoEz1q9SRKcZr+LsfntEP4lT84BpAosvpXhfuIgMdZ7wkUIDYOR13TmFqUiqaNV7vVwPLuvoKN3vIUhXR0kNqSewuMrIHOfvSuLqA5IeBG7kYZUCmiTBKvA+bhOaEZt/sVTNAlOGZrbbvc76A4sazsTVgMnVLN8lfAQtAP2bENtmOJcWTH8OPqYW9E/Pxs6AWjAdb/gxteD6+l6Ih9OC6Yi9j6kF/QQ29obbCyrH1dBC9DG14Ovpvii4Nj8SOJsezAOt+GPqwdPVMOBmMCHpO0YvhaGIdxSJnYb/2vFxaGGk1s+M+ggM1Lz9HOENffXB+Lp0z2Fc2MsHpwdf+9cjv3j3dd+aj/Zz31eOPviMT+cl4Vizt6MvDvPi5v8Iqbpv/iMW/9O/</diagram></mxfile>

docs/authentication.md

@@ -0,0 +1,86 @@
+## Steps
+
+### 1. Access Resource
+
+The resource is wrapped by the function `cas_login`.
+
+The client is passed to the inner function when:
+
+* The user has a valid session.
+* Base is marked as public.
+
+Otherwise comes to 2.
+
+See [code](aaa/cas/authentication.py)
+
+### 2. Redirect to Login
+
+Follow Redirect Response from 1. to CAS Login page with query parameter `service=...`
+
+See [code](aaa/cas/authentication.py)
+
+### 3. CAS Login
+
+The client follows the Redirect Response and authenticates on the CAS Login page.
+
+See [View code](aaa/cas/views.py#L26)
+
+### 4. CAS Authentication
+
+CAS authenticates the user against a configure backend.
+
+### 5. CAS Response
+
+On succesfull authentication CAS response with a Ticket for the user. The Ticket is transfered to the client as query parameter in a Redirect Response `Location` Header.
+
+See [View code](aaa/cas/views.py#L67)
+
+### 6. Ticket Transfer 
+
+The client follows the Redirect Response and calls the Service with a Ticket as query parameter.
+
+See [Wrapper function](aaa/cas/authentication.py#62)
+
+### 7. Ticket Verification
+
+CAS make a HTTP Call to the Ticket Service to verify if the ticket is valid for this service.
+
+See [Wrapper function](aaa/cas/authentication.py#60)
+
+#### 7.1 Verification
+
+For the verification the view [aaa/cas/views.py#80](aaa/cas/views.py#80) is responsible to verify the ticket and response in case of a valid ticket with a JWT Token. 
+
+#### 7.2 Authencation
+
+Now the [Wrapper function](aaa/cas/authentication.py) queries the `User` object from database and calls `django.contrib.auth.login` to create a session.
+
+#### 7.3 Response
+
+> Now comes the tricky part because the Session must be restricted to `/userland....`. But Django supports only one configuration. See [SESSION_COOKIE_PATH](https://docs.djangoproject.com/en/2.0/ref/settings/#std:setting-SESSION_COOKIE_PATH)
+
+With the following code we are able to create a new Session Key. The custom Cookie-Path is set the the session, so we can later process the response and overwrite Django's default behaviour.
+
+    request.session['cookie_path'] = "/userland/%s/%s" % (
+                base.user.username, base.name)
+    request.session.cycle_key()
+
+#### 7.4 Middleware
+
+The default behaviour in Django's `django.contrib.sessions.middleware.SessionMiddleware` (to set as Cookie-Path the path from `settings.SESSION_COOKIE_PATH`) is changed by adding a Middleware, which sets the path from the session's `cookie_path` object.
+
+
+See [CasSessionMiddleware](aaa/cas/middleware.py)
+
+### 8. Return with Session
+
+In 7. Ticket Verification the client received a HTTP Response with a `Set-Cookie` Header with the session.
+
+### 9. Use Service
+
+In 8. the response was a Redirect Response and the client can now use the service with the session (he is authentication).
+
+
+
+
+

minikube.ini

@@ -0,0 +1,11 @@
+[kubernetes]
+namespace=tumbo
+context=minikube
+
+[site]
+host=192.168.99.100
+password="aHVodWxhbGFsYQ=="
+frontend_host=192.168.99.100
+ADMIN_PASSWORD=hellohello
+ALLOWED_HOSTS=192.168.99.100:31999
+SERVER_NAME=192.168.99.100

requirements-tox.txt

@@ -2,7 +2,7 @@ APScheduler==3.3.1
 bumpversion==0.5.3
 bunch==1.0.1
 configobj==5.0.6
-coverage==4.2
+coverage==4.5.1
 django-compressor==1.5
 django-cors-headers==1.1.0
 django-debug-toolbar==1.6
@@ -38,8 +38,8 @@ Pygments==2.1.3
 PyJWT==1.6.1
 pyOpenSSL==16.2.0
 python-jose==1.1.0
-python-social-auth==0.3.6
 social-auth-app-django==2.1.0
+social-auth-core==1.7.0
 pytz==2016.7
 rcssmin==1.0.6
 redis==2.10.5

requirements.txt

@@ -2,7 +2,7 @@ APScheduler==3.3.1
 bumpversion==0.5.3
 bunch==1.0.1
 configobj==5.0.6
-coverage==4.2
+coverage==4.5.1
 Django==1.8.19
 django-compressor==1.5
 django-cors-headers==1.1.0
@@ -39,8 +39,8 @@ Pygments==2.1.3
 PyJWT==1.6.1
 pyOpenSSL==16.2.0
 python-jose==1.1.0
-python-social-auth==0.3.6
 social-auth-app-django==2.1.0
+social-auth-core==1.7.0
 pytz==2016.7
 rcssmin==1.0.6
 redis==2.10.5

selenium/tumbo-selenium-project.side

@@ -0,0 +1 @@
+{"id":"4d72f9d6-642c-48c5-a1f6-7e7aeb8bf797","name":"Untitled Project","url":"http://127.0.0.1:8000","tests":[{"id":"af7ee453-9cb4-492e-b490-e0f13fd2a0a8","name":"Untitled","commands":[{"id":"87a6934a-4c57-4546-beb9-86730dbca9c8","command":"open","target":"/search","value":""},{"id":"1ef312f0-12e0-4e7f-ad66-72c5e7091d3b","command":"clickAt","target":"css=h3.r > a","value":"228,8"},{"id":"6b7c595d-d87c-4112-ae10-16b3a26945f8","command":"clickAt","target":"css=p > a","value":"32,9"},{"id":"e3823f62-1ef2-4d88-b4e9-64321965faf6","command":"runScript","target":["window.scrollTo(0,670)"],"value":""},{"id":"413e0241-369c-4580-b1c0-87abb56fba8c","command":"clickAt","target":"css=h3.r > a","value":"151,7"},{"id":"bf254ee0-50fe-4432-b6d3-59604a257146","command":"clickAt","target":"//a[contains(text(),'Top 3 Selenium IDE alternatives for Firefox & Chrome | Katalon Studio')]","value":"322,0"},{"id":"bd0de052-00fb-4095-990c-c354fbb0956f","command":"clickAt","target":"css=h3.r > a","value":"234,5"},{"id":"d8f6a575-d83c-4d76-bfdf-5e398e046bd0","command":"clickAt","target":"css=span > a","value":"312,12"},{"id":"8ebb1e1a-4157-4ac1-9682-ea4cfeb4ae7d","command":"runScript","target":["window.scrollTo(0,187)"],"value":""}]},{"id":"5934e323-878b-43aa-822a-be33c34d77e1","name":"tumbo","commands":[{"id":"3f240f0e-352b-40d0-9dcc-e511f157abea","command":"open","target":"/","value":""},{"id":"7b5cc7c0-6ef6-4631-9889-4755c3542104","command":"clickAt","target":"id=inputUsername","value":"259,20"},{"id":"0b197145-5947-49b3-9101-fda5850c0b0c","command":"type","target":"id=inputUsername","value":"admin"},{"id":"a3649189-1560-47c5-9ee6-9d4ebaea85ce","command":"type","target":"id=inputPassword","value":"adminpw"},{"id":"857e133b-07f0-4367-b055-f642bb835972","command":"submit","target":"id=login_form","value":""},{"id":"281055ff-bfea-4676-9f54-c28a4e322776","command":"clickAt","target":"id=inputBaseName","value":"294,14"},{"id":"956d99d4-daab-434f-a47f-cd52c7664a9c","command":"type","target":"id=inputBaseName","value":"testbase3"},{"id":"334b86ca-ee3a-493a-9a80-218d37af3d8f","command":"clickAt","target":"//div[2]/div/h2","value":"190,15"},{"id":"8cbf16a1-c51c-429a-bb61-53b2092cb398","command":"clickAt","target":"name=create_new_base","value":"56,22"},{"id":"6df1d46a-a80a-4dee-8264-04ec107efa0f","command":"mouseOver","target":"name=create_new_base","value":""},{"id":"7787534a-355e-4a15-910a-0ec28f5432ea","command":"mouseOut","target":"name=create_new_base","value":""},{"id":"b00d6803-f261-4b5d-8266-e06e15f83cf1","command":"clickAt","target":"//a[contains(text(),'admin/testbase3')]","value":"88,15"},{"id":"896ab499-6250-49c6-ae6d-ed15473e471b","command":"clickAt","target":"css=button[name=\"state_cycle\"] > span","value":"24,2"},{"id":"2bc447db-973f-4251-8821-5e6b286251b3","command":"clickAt","target":"css=button[name=\"state_cycle\"] > span","value":"22,5"},{"id":"77a55534-9e16-4f6b-b3f2-687294eac53e","command":"clickAt","target":"css=button[name=\"state_cycle\"] > span","value":"4,7"},{"id":"dae26f06-0c0f-4c52-9b1a-0bd3579cdeb4","command":"clickAt","target":"css=button[name=\"state_cycle\"] > span","value":"15,11"},{"id":"df787a1a-7d63-4910-95e2-97dd6f62c9a4","command":"clickAt","target":"css=p.ng-binding","value":"705,14"},{"id":"91addc7c-a7f9-499f-805c-456ad9255c83","command":"clickAt","target":"name=state_cycle","value":"21,19"},{"id":"646bdf71-a936-4420-ab68-7a6db90a025e","command":"clickAt","target":"css=button[name=\"state_cycle\"] > span","value":"11,10"},{"id":"26f6f82e-c2b6-43cb-90d5-ba897f320bfe","command":"clickAt","target":"css=p.ng-binding","value":"741,5"},{"id":"11d91c58-00f6-419f-bf81-6800ecfd04b4","command":"clickAt","target":"css=div.col-md-12 > a","value":"283,11"},{"id":"5f91b803-55b4-4394-98d5-bbd4aed8fde9","command":"type","target":"id=username","value":"admin"},{"id":"6167394a-6f67-4edc-932e-47378e1d6628","command":"type","target":"id=password","value":"adminpw"},{"id":"f1e604a5-ee8b-4b51-87f6-c23e2cdeec99","command":"submit","target":"id=login_form","value":""}]}],"suites":[],"urls":["https://www.google.ch","http://127.0.0.1:8000"]}

tox.ini

@@ -2,6 +2,7 @@
 envlist = py{27}-django{18}
 
 [testenv]
+whitelist_externals = docker 
 basepython =
     py27: python2.7
 deps =
@@ -14,9 +15,20 @@ setenv = DIGITALOCEAN_ZONE = a.a.com
          DROPBOX_REDIRECT_URL = aa
          CI = true
 commands =
-    pip install selenium==3.11.0
     pip install -r requirements-tox.txt
+
+    - docker rm --force rabbitmq || true
+    - docker rm --force postgresql || true
+
+    docker login -u {env:DOCKER_USER} -p {env:DOCKER_PASS}
+
+    docker run -d -e RABBITMQ_PASS=rabbitmq -p 5672:5672 -p 15672:15672  --name rabbitmq tutum/rabbitmq
+    docker run -d -p 55432:5432 -e SUPERUSER=true -e DB_NAME=tumbo_fictitious -e DB_USER=tumbo -e PASSWORD=tumbodbpw --name postgresql philipsahli/postgresql-test
+
     coverage run --append --omit='*migrations*' --source=tumbo tumbo/manage.py test core --settings=tumbo.dev
     coverage run --append --omit='*migrations*' --source=tumbo tumbo/manage.py test aaa --settings=tumbo.dev
-    coverage run --append --omit='*migrations*' --source=tumbo tumbo/manage.py test ui --settings=tumbo.dev
+    coverage run --append --omit='*migrations*' --source=tumbo tumbo/manage.py test ui --settings=tumbo.dev -v 3
+
+    docker rm --force rabbitmq
+    docker rm --force postgresql
 

tumbo/aaa/cas/authentication.py

@@ -30,6 +30,8 @@ def cas_login(function):
     def wrapper(request, *args, **kwargs):
         user = request.user
 
+        logger.info("step:cas-1:user %s arrived for URL %s" % (user, request.get_full_path()))
+
         # if logged in
         if request.user.is_authenticated():
             logger.info("user.is_authenticated with user %s" %
@@ -46,7 +48,8 @@ def wrapper(request, *args, **kwargs):
 
         service = reverse(
             'userland-static', args=[kwargs['username'], kwargs['base'], "index.html"])
-        proto = request.META.get('HTTP_X_FORWARDED_PROTO', 'https')
+        # TODO: make it possible to use https
+        proto = request.META.get('HTTP_X_FORWARDED_PROTO', 'http')
         host = request.META.get('HTTP_X_FORWARDED_HOST',
                                 request.META.get('HTTP_HOST', None))
         if base.frontend_host:
@@ -58,34 +61,48 @@ def wrapper(request, *args, **kwargs):
         ticket = request.GET.get("ticket", None)
 
         if ticket:
+
             # if the service is called with a ticket, verify the ticket and redirect to the service
+            logger.info("step:cas-6:ticket received in CAS")
+
             cas_ticketverify = reverse('cas-ticketverify')
             cas_ticketverify += "?ticket=%s&service=%s" % (
                 ticket, service_full)
             host = urlparse(request.build_absolute_uri()).netloc
-            response = requests.get("https://%s%s" % (host, cas_ticketverify))
+            # TODO: normally with https
+            logger.info("step:cas-7:verify ticket -> request")
+            response = requests.get("http://%s%s" % (host, cas_ticketverify))
             logger.info("Response from verify: " + str(response.status_code))
             logger.info("Response from verify: " + response.text)
+            logger.info("step:cas-7:verify ticket -> request -> response(%s)" % response.status_code)
 
             # read jwt for identity
             username, decoded_dict = read_jwt(
                 response.text, settings.SECRET_KEY)
             logger.info("Identity from Token: %s" % username)
             logger.info("Identity from Token: %s" % str(decoded_dict))
 
+            logger.info("step:cas-7.2:verify ticket -> request -> response(%s)" % response.status_code)
+
             user = User.objects.get(username=username)
             user.backend = 'django.contrib.auth.backends.ModelBackend'
             auth_login(request, user)
 
+            logger.info("step:cas-7.3:verify ticket -> request -> response(%s) -> remember cookie_path" % response.status_code)
+
             request.session['cookie_path'] = "/userland/%s/%s" % (
                 base.user.username, base.name)
+            logger.info("Setting cookie_path to: %s" % request.session['cookie_path'])
             request.session.cycle_key()
 
+            from django.middleware.csrf import rotate_token
+            rotate_token(request)
+
             # user is logged in successfully, redirect to service URL
             return HttpResponseRedirect(service)
 
         # User need to authenticate first on cas
         url = reverse('cas-login') + "?service=%s" % service_full
-        logger.info("Redirecting to CAS login %s" % url)
+        logger.info("step:cas-2: redirecting to CAS login %s" % url)
         return HttpResponseRedirect(url)
     return wrapper