Locking stats and lots of changes to make threads as fast as possible

Author: happz

src/api.py

@@ -7,11 +7,13 @@
 import json
 import threading
 import types
+import UserDict
 
 import hlib.error
 import hlib.handlers
 import hlib.http.session
 import hlib.input
+import hlib.locks
 import hlib.log
 
 # pylint: disable-msg=F0401
@@ -23,6 +25,26 @@
   lambda s: s.startswith('password')
 ]
 
+class APICookieStorage(UserDict.UserDict):
+  def __init__(self):
+    UserDict.UserDict.__init__(self)
+
+    self._lock = hlib.locks.RLock(name = 'API cookie storage')
+
+  def __getitem__(self, name):
+    with self._lock:
+      return UserDict.UserDict.__getitem__(self, name)
+
+  def __setitem__(self, name, value):
+    with self._lock:
+      UserDict.UserDict.__setitem__(self, name, value)
+
+  def __contains__(self, name):
+    with self._lock:
+      return UserDict.UserDict.__contains__(self, name)
+
+_COOKIES = APICookieStorage()
+
 class ApiIORegime(hlib.handlers.IORegime):
   @staticmethod
   def check_session():
@@ -252,7 +274,7 @@ def __init__(self, name, app):
     self.name			= name
     self.app			= app
 
-    self.lock = threading.Lock()
+    self.lock = hlib.locks.Lock(name = 'API token cache')
     self.token_to_user = {}
 
   def __getitem__(self, key):

src/cache.py

@@ -7,6 +7,8 @@
 import sys
 import threading
 
+import hlib.locks
+
 from hlib.stats import stats as STATS
 
 class Cache(object):
@@ -16,7 +18,7 @@ def __init__(self, name, app):
     self.name = name
     self.app = app
 
-    self.lock		= threading.RLock()
+    self.lock = hlib.locks.RLock(name = 'Cache')
     self.objects	= {}
 
     self.stats_name = 'Cache (%s - %s)' % (self.app.name, self.name)

src/console.py

@@ -7,6 +7,8 @@
 import json
 import os
 
+import hlib.locks
+
 class HelpAction(argparse._HelpAction):
   def __init__(self, *args, **kwargs):
     super(HelpAction, self).__init__(*args, **kwargs)
@@ -42,6 +44,8 @@ def __init__(self, console, parser):
     self.parser.add_argument('--version', action = 'store_const', dest = 'action', const = 'version')
     self.parser.add_argument('--crash', action = 'store_const', dest = 'action', const = 'crash')
 
+    self.parser.add_argument('--locks', action = 'store_const', dest = 'action', const = 'locks')
+
   def handler(self, args):
     if args.action == 'quit':
       self.console.stop()
@@ -51,6 +55,9 @@ def handler(self, args):
     if args.action == 'crash':
       os._exit(0)
 
+    if args.action == 'locks':
+      hlib.locks.save_stats('lock_debug.dat')
+
 class Command_Help(Command):
   def __init__(self, console, parser):
     super(Command_Help, self).__init__(console, parser)

src/database.py

@@ -15,6 +15,7 @@
 
 import hlib.console
 import hlib.error
+import hlib.locks
 import hlib.log
 from hlib.stats import stats as STATS
 
@@ -30,7 +31,7 @@ def __init__(self, console, parser):
     self.parser.add_argument('--enable-lt', action = 'store_const', dest = 'action', const = 'enable-lt')
     self.parser.add_argument('--disable-lt', action = 'store_const', dest = 'action', const = 'disable-lt')
     self.parser.add_argument('--pack', action = 'store_const', dest = 'action', const = 'pack')
-    self.parser.add_argument('--minimize-cache', action = 'store_const', dest = 'action', const = 'minimize-cache')
+    self.parser.add_argument('--gc', action = 'store_const', dest = 'action', const = 'gc')
 
     self.parser.add_argument('--name', action = 'store', dest = 'name')
 
@@ -57,8 +58,8 @@ def handler(self, args):
     if args.action == 'pack':
       return self.__get_db_by_name(args.name).pack()
 
-    if args.action == 'minimize-cache':
-      return self.__get_db_by_name(args.name).minimize_cache()
+    if args.action == 'gc':
+      return self.__get_db_by_name(args.name).globalGC()
 
     if args.action == 'update-stats':
       return self.__get_db_by_name(args.name).update_stats()
@@ -134,8 +135,8 @@ def __init__(self, name, address, **kwargs):
     self.db		= None
     self.root		= None
 
-    self.log_transaction_handler = self.__log_transaction
-    self.log_transaction_lock = threading.RLock()
+    self.log_transaction_handler = self.__nolog_transaction
+    self.log_transaction_lock = hlib.locks.RLock(name = 'Transaction logging setup')
 
     self.kwargs		= kwargs
 
@@ -167,7 +168,7 @@ def on_request_finished(self, _):
   def is_opened(self):
     return self.db != None
 
-  def set_transaction_logging(self, enabled = True):
+  def set_transaction_logging(self, enabled = False):
     with self.log_transaction_lock:
       if enabled:
         self.log_transaction_handler = self.__log_transaction
@@ -191,6 +192,9 @@ def connect(self):
     return (connection, self.root)
 
   def log_transaction(self, *args, **kwargs):
+    # Transaction logging is broken - because of locking - lock is required even for NOP action :(
+    return
+
     with self.log_transaction_lock:
       self.log_transaction_handler(*args, **kwargs)
 
@@ -259,6 +263,24 @@ def pack(self):
   def minimize_cache(self):
     self.db.minimizeCache()
 
+  def localGC(self):
+    hruntime.dbconn.cacheGC()
+    hruntime.dbconn.cacheMinimize()
+
+  def poolGC(self):
+    self.db._a()
+    self.db.pool.availableGC()
+    self.db.historical_pool.availableGC()
+    self.db._r()
+
+  def globalGC(self):
+    self.pack()
+
+    self.db._a()
+    self.db.pool.availableGC()
+    self.db.historical_pool.availableGC()
+    self.db._r()
+
   def update_stats(self):
     data = self.db.getActivityMonitor().getActivityAnalysis(divisions = 1)[0]
     connections = {}
@@ -271,18 +293,19 @@ def update_stats(self):
         'Info': data['info'],
         'Before': data['before']
       }
+      i += 1
 
     for data in self.db.cacheDetailSize():
       caches[data['connection']] = {
+        'Connection': data['connection'],
         'Non-ghost size': data['ngsize'],
         'Size': data['size']
       }
 
-    with STATS:
-      STATS.set(self.stats_name, 'Load', data['loads'] if 'loads' in data else 0)
-      STATS.set(self.stats_name, 'Stores', data['stores'] if 'stores' in data else 0)
-      STATS.set(self.stats_name, 'Connections', connections)
-      STATS.set(self.stats_name, 'Caches', caches)
+    STATS.set(self.stats_name, 'Load', data['loads'] if 'loads' in data else 0)
+    STATS.set(self.stats_name, 'Stores', data['stores'] if 'stores' in data else 0)
+    STATS.set(self.stats_name, 'Connections', connections)
+    STATS.set(self.stats_name, 'Caches', caches)
 
 class DBObject(persistent.Persistent):
   def __init__(self, *args, **kwargs):