Issue #293: HTTP proxy support

geeknote/config.py

@@ -31,7 +31,6 @@
 
 # Application path
 APP_DIR = os.path.join(os.getenv("HOME") or os.getenv("USERPROFILE"),  ".geeknote")
-ERROR_LOG = os.path.join(APP_DIR, "error.log")
 
 # Set default system editor
 DEF_UNIX_EDITOR = "nano"
@@ -44,18 +43,22 @@
 # Url view the note
 NOTE_URL = "https://%domain%/Home.action?#n=%s"
 
+if DEV_MODE:
+    USER_STORE_URI = USER_STORE_URI_SANDBOX
+    CONSUMER_KEY = CONSUMER_KEY_SANDBOX
+    CONSUMER_SECRET = CONSUMER_SECRET_SANDBOX
+    USER_BASE_URL = USER_BASE_URL_SANDBOX
+    APP_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "config")
+    sys.stderr.write("Developer mode: using %s as application directory\n" % APP_DIR)
+
+ERROR_LOG = os.path.join(APP_DIR, "error.log")
+
 # validate config
 try:
     if not os.path.exists(APP_DIR):
         os.mkdir(APP_DIR)
 except Exception, e:
-    sys.stdout.write("Can not create application dirictory : %s" % APP_DIR)
+    sys.stderr.write("Can not create application directory : %s" % APP_DIR)
     exit(1)
 
-if DEV_MODE:
-    USER_STORE_URI = USER_STORE_URI_SANDBOX
-    CONSUMER_KEY = CONSUMER_KEY_SANDBOX
-    CONSUMER_SECRET = CONSUMER_SECRET_SANDBOX
-    USER_BASE_URL = USER_BASE_URL_SANDBOX
-
 NOTE_URL = NOTE_URL.replace('%domain%', USER_BASE_URL)

geeknote/editor.py

@@ -76,7 +76,7 @@ def checklistInENMLtoSoup(soup):
 
     @staticmethod
     def ENMLtoText(contentENML):
-        soup = BeautifulSoup(contentENML.decode('utf-8'))
+        soup = BeautifulSoup(contentENML.decode('utf-8'), "html.parser")
 
         for section in soup.select('li > p'):
             section.replace_with( section.contents[0] )

geeknote/geeknote.py

@@ -624,7 +624,8 @@ def _editWithEditorInThread(self, inputData, note = None):
             if not thread.isAlive():
                 # check if thread is alive here before sleep to avoid losing data saved during this 5 secs
                 break
-            time.sleep(5)
+            thread.join(timeout=5)
+
         return result
 
     def create(self, title, content=None, tags=None, notebook=None, resource=None):

geeknote/oauth.py

@@ -5,9 +5,9 @@
 import Cookie
 import uuid
 import re
-from urllib import urlencode, unquote
+import base64
+from urllib import urlencode, unquote, getproxies, proxy_bypass
 from urlparse import urlparse
-
 import out
 import tools
 import config
@@ -58,6 +58,27 @@ class GeekNoteAuth(object):
     incorrectCode = 0
     code = None
 
+    def __init__(self):
+        try:
+            proxy = getproxies()['https']
+        except KeyError:
+            proxy = None
+        if proxy is None:
+            self._proxy = None
+        else:
+            # This assumes that the proxy is given in URL form.
+            # A little simpler as _parse_proxy in urllib2.py
+            self._proxy = urlparse(proxy)
+            logging.debug("Using proxy: %s" % self._proxy.geturl())
+
+        if proxy is None or not self._proxy.username:
+            self._proxy_auth = None
+        else:
+            user_pass = "%s:%s" % (urlparse.unquote(self._proxy.username),
+                                   urlparse.unquote(self._proxy.password))
+            self._proxy_auth = { "Proxy-Authorization":
+                                 "Basic " + base64.b64encode(user_pass).strip() }
+
     def getTokenRequestData(self, **kwargs):
         params = {
             'oauth_consumer_key': self.consumerKey,
@@ -77,9 +98,11 @@ def loadPage(self, url, uri=None, method="GET", params=""):
             logging.error("Request URL undefined")
             tools.exitErr()
 
+        if not url.startswith("http"):
+            url = "https://" + url
+        urlData = urlparse(url)
         if not uri:
-            urlData = urlparse(url)
-            url = urlData.netloc
+            url = "%s://%s" (urlData.scheme, urlData.netloc)
             uri = urlData.path + '?' + urlData.query
 
         # prepare params, append to uri
@@ -97,11 +120,27 @@ def loadPage(self, url, uri=None, method="GET", params=""):
         if method == "POST":
             headers["Content-type"] = "application/x-www-form-urlencoded"
 
-        logging.debug("Request URL: %s:/%s > %s # %s", url,
+        if self._proxy is None or proxy_bypass(urlData.hostname):
+            host = urlData.hostname
+            port = urlData.port
+            real_host = real_port = None
+        else:
+            host = self._proxy.hostname
+            port = self._proxy.port
+            real_host = urlData.hostname
+            real_port = urlData.port
+
+        logging.debug("Request URL: %s%s > %s # %s", url,
                       uri, unquote(params), headers["Cookie"])
 
-        conn = httplib.HTTPSConnection(url)
-        conn.request(method, uri, params, headers)
+        conn = httplib.HTTPSConnection(host, port)
+
+        if real_host is not None:
+            conn.set_tunnel(real_host, real_port, headers=self._proxy_auth)
+        if config.DEBUG:
+            conn.set_debuglevel(1)
+
+        conn.request(method, url + uri, params, headers)
         response = conn.getresponse()
         data = response.read()
         conn.close()

geeknote/out.py

@@ -6,6 +6,7 @@
 import time
 import datetime
 import sys
+import os.path
 
 import tools
 from editor import Editor
@@ -94,10 +95,33 @@ def draw():
         except:
             pass
 
+def _getCredentialsFromFile():
+    # Get evernote credentials from file APP_DIR/credentials
+    # This is used only for sandbox mode (DEV_MODE=True) for security reasons
+    if config.DEV_MODE:
+        creds = os.path.join(config.APP_DIR, "credentials")
+        if os.path.exists(creds):
+            # execfile doesn't work reliably for assignments, see python docs
+            with open(creds, "r") as f:
+                # this sets "credentials" if correctly formatted
+                exec f.read()
+            try:
+                return credentials.split(":")
+            except:
+                sys.stderr.write("""Error reading credentials from %s. 
+Format should be:
+credentials="<username>:<password>:<two-factor auth code>"
+
+""" % creds)
+    return None
 
 @preloaderPause
 def GetUserCredentials():
     """Prompts the user for a username and password."""
+    creds = _getCredentialsFromFile()
+    if creds is not None:
+        return creds[:2]
+
     try:
         login = None
         password = None
@@ -117,6 +141,10 @@ def GetUserCredentials():
 @preloaderPause
 def GetUserAuthCode():
     """Prompts the user for a two factor auth code."""
+    creds = _getCredentialsFromFile()
+    if creds is not None:
+        return creds[2]
+
     try:
         code = None
         if code is None:
@@ -275,18 +303,11 @@ def rawInput(message, isPass=False):
 
 def printDate(timestamp):
 
-    # Author @ash-2000 https://github.com/ash-2000
-    # Check for crashing when timestamp is 13 digits on python2.7
-    # pull request #260
-
-    if len(str(timestamp)) == 13:
-        timestamp = int(str(timestamp)[0:-3])
-
-    # ---
-
     return datetime.date.strftime(datetime.date.fromtimestamp(timestamp / 1000), "%d.%m.%Y")
 
-def printLine(line, endLine="\n", out=sys.stdout):
+def printLine(line, endLine="\n", out=None):
+    if out is None:
+        out = sys.stdout
     message = line + endLine
     message = tools.stdoutEncode(message)
     try:

proxy_support.md

@@ -0,0 +1,45 @@
+HTTP proxy support for geeknote
+===============================
+
+I recommend to make this work with virtualenv, to avoid overwriting system files.
+The important part is to install in the order **thrift, then evernote, then geeknote**. This will make sure that path search order is correct for thrift.
+
+```
+# Download thrift and geeknote
+git clone https://github.com/apache/thrift.git
+git clone https://github.com/mwilck/geeknote.git
+
+# create and enter a virtual environment
+virtualenv /var/tmp/geeknote
+. /var/tmp/geeknote/bin/activate
+
+# Apply proxy-support patches for thrift
+cd thrift
+
+## If the patches don't apply, you may need to check out the state that I wrote the patches for:
+## git checkout -b proxy e363a34e63
+curl https://issues.apache.org/jira/secure/attachment/12801233/0001-python-THttpClient-Add-support-for-system-proxy-sett.patch | git am
+curl https://issues.apache.org/jira/secure/attachment/12801234/0002-Python-THttpClient-Support-proxy-authorization.patch | git am
+
+# Install thrift from the patched tree
+(cd lib/py; python setup.py install)
+cd ..
+
+# Install evernote
+pip install evernote
+
+# Install geeknote
+cd geeknote
+python setup.py install
+```
+
+Now `geeknote login`, `geeknote find`, etc. should work behind a proxy if the `http_proxy` environment variable is correctly set. You can now generate a script to activate the virtual environment:
+
+```
+cat >~/bin/geeknote <<\EOF
+#! /bin/bash
+. /var/tmp/geeknote/bin/activate
+exec geeknote "$@"
+EOF
+chmod a+x ~/bin/geeknote
+```

requirements.txt

@@ -3,3 +3,4 @@ html2text
 sqlalchemy
 markdown2
 thrift
+beautifulsoup4

tests/editorTest.py

@@ -16,7 +16,6 @@ def setUp(self):
 _Line 2_
 
 **Line 3**
-
 """
         self.HTML_TEXT = "<h1>Header 1</h1><h2>Header 2</h2><p>Line 1</p><p>"\
                          "<em>Line 2</em></p><p><strong>Line 3</strong></p>"

tests/geeknoteTest.py

@@ -1,26 +1,45 @@
 # -*- coding: utf-8 -*-
 
+import sys
 import time
 import unittest
 from geeknote.geeknote import *
 from geeknote import tools
 from geeknote.editor import Editor
+from geeknote.storage import Storage
 
 class GeekNoteOver(GeekNote):
     def __init__(self):
         pass
 
     def loadNoteContent(self, note):
-        note.content = "note content"
+        if "content" not in note.__dict__:
+            note.content = "note content"
 
+    def updateNote(self, guid=None, **inputData):
+        # HACK for testing: this assumes that the guid represents a "note" itself
+        # see do_test_editWithEditorInThread below
+        guid.content = inputData["content"]
 
 class NotesOver(Notes):
     def connectToEvertone(self):
         self.evernote = GeekNoteOver()
 
-
 class testNotes(unittest.TestCase):
 
+    @classmethod
+    def setUpClass(cls):
+        # Use our trivial "pseudoedit" program as editor to avoid user interaction
+        cls.storage = Storage()
+        cls.saved_editor = cls.storage.getUserprop('editor')
+        cls.storage.setUserprop('editor', sys.executable + " " +
+                                os.path.join(os.path.dirname(os.path.abspath(__file__)), "pseudoedit.py"))
+
+    @classmethod
+    def tearDownClass(cls):
+        if cls.saved_editor:
+            cls.storage.setUserprop('editor', cls.saved_editor)
+
     def setUp(self):
         self.notes = NotesOver()
         self.testNote = tools.Struct(title="note title")
@@ -49,17 +68,26 @@ def test_parseInput2(self):
         )
         self.assertEqual(testData["tags"], ["tag1", "tag2"])
 
+    def do_test_editWithEditorInThread(self, txt, expected):
+        testNote = tools.Struct(title="note title",
+                                content=txt)
+        # hack to make updateNote work - see above
+        testNote.guid = testNote
+        testData = self.notes._parseInput("title",
+                                          txt,
+                                          "tag1, tag2",
+                                          None, testNote)
+        result = self.notes._editWithEditorInThread(testData, testNote)
+        self.assertEqual(Editor.ENMLtoText(testNote.content), expected)
+
     def test_editWithEditorInThread(self):
-        testData = self.notes._parseInput("title", "WRITE", "tag1, tag2",
-                                          None, self.testNote)
-        print ('')
-        print ('')
-        print (testData)
-        print ('')
-        print ('')
-
-        self.notes._editWithEditorInThread(testData)
-
+        txt = "Please do not change this file"
+        self.do_test_editWithEditorInThread(txt, txt+'\n')
+
+    def test_editWithEditorInThread2(self):
+        txt = "Please delete this line, save, and quit the editor"
+        self.do_test_editWithEditorInThread(txt, "\n")
+
     def test_createSearchRequest1(self):
         testRequest = self.notes._createSearchRequest(
             search="test text",
@@ -88,6 +116,5 @@ def test_createSearchRequest2(self):
         self.assertEqual(testRequest, response)
 
     def testError_createSearchRequest1(self):
-        testRequest = self.notes._createSearchRequest(search="test text",
-                                                      date="12.31.1999")
-        self.assertEqual(testRequest, 'exit')
+        self.assertRaises(SystemExit, self.notes._createSearchRequest, 
+                          search="test text", date="12.31.1999")