diff --git a/server/INSTALL b/server/INSTALL index 7e414a8fa..edf8cff85 100644 --- a/server/INSTALL +++ b/server/INSTALL @@ -3,41 +3,29 @@ How to setup the MTT server 1) Setup the MTT Database ---------------------------------------------------- -- Install PostgreSQL - http://www.postgresql.org/ - -Note: We test with 8.4.13, but any later version should work fine. - -- Follow instructions in sql/INSTALL +See the MTT Wiki "Database" 2) Setup the MTT Reporter ---------------------------------------------------- -- Apache Setup - Apache must be setup to support PHP. - -- Optionally you might want to increase the file size limits for uploads -shell$ $EDITOR /etc/php.ini: -; Maximum allowed size for uploaded files. -upload_max_filesize = 20M +See the MTT Wiki "HTTPServer" -shell$ $EDITOR /etc/httpd/conf/httpd.conf: -# Limit Upload to 20Mb -LimitRequestBody 20971520 -- Follow instructions in php/INSTALL +3) Setup the cron scripts +---------------------------------------------------- +See the MTT Wiki "ServerMaintenance" -3) Setup the cron scripts +4.) Setup the Cherrypy Server ---------------------------------------------------- -- Follow the instructions in php/cron/INSTALL +See the MTT Wiki "MTT Cherrypy Server" -4) Test your setup +5) Test your setup ---------------------------------------------------- - Run a simple MTT client in debug mode and watch for errors -5) All done! +6) All done! ---------------------------------------------------- - Enjoy! diff --git a/server/gds/__init__.py b/server/gds/__init__.py deleted file mode 100755 index 9e3eae6e5..000000000 --- a/server/gds/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2009 Voltaire -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -""" -Application supported Google App Engine datastore for MTT. -""" - -VERSION = 0.1 -AUTHOR = "" \ No newline at end of file diff --git a/server/gds/admin/__init__.py b/server/gds/admin/__init__.py deleted file mode 100755 index 86bf28a5f..000000000 --- a/server/gds/admin/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# -# Copyright (c) 2009 Voltaire -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -"Admin place" \ No newline at end of file diff --git a/server/gds/admin/admin.py b/server/gds/admin/admin.py deleted file mode 100755 index bed4968d9..000000000 --- a/server/gds/admin/admin.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env python - -# -# Copyright (c) 2009 Voltaire -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# Python Standart modules -import os -import sys -import datetime -import types -import logging -import cStringIO -import mimetypes -import re - -# Python Google AppEngine SDK modules -import yaml -from google.appengine.api import users -from google.appengine.api import datastore -from google.appengine.api import datastore_errors -from google.appengine.api import datastore_types - -from google.appengine.ext import db -from google.appengine.ext import webapp -from google.appengine.ext.webapp.util import login_required -from google.appengine.ext.webapp.util import run_wsgi_app -from google.appengine.ext.webapp import template - -# Private modules -import conf -import auth - - -class AdminPage(webapp.RequestHandler): - "Admin place" - @login_required - def get(self): - logging.debug('%s: get=> %s' % (self.__class__.__name__, self.request.GET)) - status = 0 - - if not users.is_current_user_admin(): - status = 401 - else: - template_context = { - 'current_user': users.get_current_user() - } - - path = os.path.join(os.path.dirname(__file__), os.path.join('../templates', 'admin.html')) - self.response.out.write(template.render(path, template_context, debug=conf.DEBUG)) - - if (status == 0): status = 200 - self.response.set_status(status) - - return status - - -class AddUserHandler(webapp.RequestHandler): - def post(self): - logging.debug('%s: post=> %s' % (self.__class__.__name__, self.request.POST)) - status = 0 - - if not users.is_current_user_admin(): - status = 401 - else: - user = auth.add_user(username = self.request.get('username_'), - is_superuser = self.request.get('is_superuser_') - ) - - if (status == 0): status = 200 - self.response.set_status(status) - - self.redirect('/admin/') - - return status - - -application = webapp.WSGIApplication( - [('/admin/', AdminPage), - ('/admin/adduser', AddUserHandler) - ], debug=True) - -def main(): - if conf.DEBUG: - logging.getLogger().setLevel(logging.DEBUG) - run_wsgi_app(application) - -if __name__ == "__main__": - main() - diff --git a/server/gds/app.yaml b/server/gds/app.yaml deleted file mode 100644 index 845db2125..000000000 --- a/server/gds/app.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2009 Voltaire -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -application: open-mpi-mtt -version: 2 -runtime: python -api_version: 1 - -handlers: -- url: /favicon.ico - static_files: templates/images/favicon.ico - upload: templates/images/favicon.ico - -- url: /images - static_dir: templates/images - -- url: /admin/.* - script: admin/admin.py - login: admin - -- url: /.* - script: main.py - diff --git a/server/gds/auth/__init__.py b/server/gds/auth/__init__.py deleted file mode 100755 index e7197b423..000000000 --- a/server/gds/auth/__init__.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python - -# -# Copyright (c) 2009 Voltaire -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# Python Standart modules -import os -import re -import base64 -import urllib - -# Python Google AppEngine SDK modules -from google.appengine.api import users -from google.appengine.ext import db - -# Private modules -from models import User -import logging - - -def check_login(handler_method): - """ - A decorator to require that a user be logged in to access a handler. - """ - def check(self, *args): - logging.debug('check_login %s: <%s> admin = %s user = %s' % - (self.__class__.__name__, self.request.method, str(users.is_current_user_admin()), str(self.user))) - - user = users.get_current_user() - - if not user: - greeting = ("Sign in or register." % - users.create_login_url("/")) - self.response.set_status(401) # Unauthorized - self.response.out.write("%s" % greeting) - return - - if not users.is_current_user_admin() and self.user is None: - if self.request.method == 'GET': - credential = [user.email(), None] - self.user = authenticate(credential) - if self.user is None: - self.redirect('%s?%s=%s' % ('/login/', 'next', urllib.quote(self.request.uri))) - return - elif self.request.method == 'POST': - self.user = authenticate(get_credential()) - if self.user is None: - self.response.set_status(401) # Unauthorized - return - - handler_method(self, *args) - - return check - - -def get_credential(): - """ - Get credentials from http request. - """ - credential = None - if os.environ.has_key('HTTP_AUTHORIZATION'): - http_authorization=os.environ['HTTP_AUTHORIZATION'] - if http_authorization[:6].lower() == 'basic ': - try: decoded=base64.decodestring(http_authorization[6:]) - except base64.binascii.Error: decoded='' - credential = decoded.split(':') - else: - credential = ['', ''] - - logging.debug('get_credential: %s' % credential) - - return credential - - -def authenticate(credential): - """ - If the given credentials are valid, return a User object. - """ - if credential is None: - return None - - query = db.Query(User) - users = query.filter('username =', credential[0]) - - if users.count()>1: - logging.error("There are several users with username = '%s' " % credential[0]) - return None - - users = query.filter('is_active =', True) - user = users.get() - - if user and credential[1] is not None: - if (not user.check_password(raw_password = credential[1])): - user = None - - logging.debug('authenticate: %s' % str(user)) - - return user - - -def add_user(**credential): - """ - Create new User objects using passed credentials and return a User object. - """ - logging.debug('add_user: %s' % str(credential)) - - user = None - - query = db.Query(User) - query_users = query.filter('username =', credential['username']) - if query_users.count()>1: - logging.error("There are several users with username = '%s' and email = '%s'" % (credential['username'], credential['email'])) - user = query_users.get() - elif (credential.has_key('username')): - user = User(username = credential['username']) - if (credential.has_key('is_superuser') and - credential['is_superuser'].lower() in ('yes', 'true')): user.is_superuser = True - else: - user.is_superuser = False - user.is_active = True - - user.put() - - return user diff --git a/server/gds/auth/models.py b/server/gds/auth/models.py deleted file mode 100755 index fb0099c64..000000000 --- a/server/gds/auth/models.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python - -# -# Copyright (c) 2009 Voltaire -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# Python Standart modules -import datetime -import re - -# Python Google AppEngine SDK modules -from google.appengine.ext import db - -# Private modules - - -class User(db.Model): - username = db.StringProperty(required=True) - is_superuser = db.BooleanProperty(default=False) - is_active = db.BooleanProperty(default=True) - last_login = db.DateTimeProperty() - created = db.DateTimeProperty(auto_now_add=True) - modified = db.DateTimeProperty(auto_now=True) - - def __str__(self): - return str(self.username) - - def __unicode__(self): - return unicode(self.username) - - def get_full_name(self): - "Returns the first_name plus the last_name, with a space in between." - full_name = '%s %s' % (self.first_name, self.last_name) - return full_name.strip() - - def check_password(self, raw_password): - import urllib - import logging - from google.appengine.api import urlfetch - - request_body = urllib.urlencode({'Email': self.username, - 'Passwd': raw_password, - 'accountType': 'HOSTED_OR_GOOGLE', - 'service': 'ah', - 'source': 'test'}) - auth_response = urlfetch.fetch('https://www.google.com/accounts/ClientLogin', - method=urlfetch.POST, - headers={'Content-type':'application/x-www-form-urlencoded', - 'Content-Length': - str(len(request_body))}, - payload=request_body) - auth_dict = dict(x.split("=") for x in auth_response.content.split("\n") if x) - auth_token = auth_dict.has_key("Auth") - - return auth_token diff --git a/server/gds/conf.py b/server/gds/conf.py deleted file mode 100755 index 909060560..000000000 --- a/server/gds/conf.py +++ /dev/null @@ -1,18 +0,0 @@ -# -# Copyright (c) 2009 Voltaire -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -""" -Configuration settings for application -""" - -# Debug mode -DEBUG = True - -# Authorisation key -AUTH = "" \ No newline at end of file diff --git a/server/gds/index.yaml b/server/gds/index.yaml deleted file mode 100644 index d402d33cd..000000000 --- a/server/gds/index.yaml +++ /dev/null @@ -1,260 +0,0 @@ -# -# Copyright (c) 2009 Voltaire -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -indexes: - -######################################################## -# select * from SubmitInfo -######################################################## - -# where hostname(equality) and http_username(inequality) -- kind: SubmitInfo - properties: - - name: hostname - - name: http_username - -# where hostname(inequality) and http_username(equality) -- kind: SubmitInfo - properties: - - name: http_username - - name: hostname - -# where hostname(equality) and local_username(inequality) -- kind: SubmitInfo - properties: - - name: hostname - - name: local_username - -# where hostname(inequality) and local_username(equality) -- kind: SubmitInfo - properties: - - name: local_username - - name: hostname - -# where http_username(equality) and local_username(inequality) -- kind: SubmitInfo - properties: - - name: http_username - - name: local_username - -# where http_username(inequality) and local_username(equality) -- kind: SubmitInfo - properties: - - name: local_username - - name: http_username - -######################################################## -# select * from SuiteInfo -######################################################## - -# where suite_name(equality) and suite_version(inequality) -- kind: SuiteInfo - properties: - - name: suite_name - - name: suite_version - -# where suite_name(inequality) and suite_version(equality) -- kind: SuiteInfo - properties: - - name: suite_version - - name: suite_name - -######################################################## -# select * from MpiInfo -######################################################## - -# where mpi_name(equality) and mpi_version(inequality) -- kind: MpiInfo - properties: - - name: mpi_name - - name: mpi_version - -# where mpi_name(inequality) and mpi_version(equality) -- kind: MpiInfo - properties: - - name: mpi_version - - name: mpi_name - -# where mpi_name(equality) and oma_version(inequality) -- kind: MpiInfo - properties: - - name: mpi_name - - name: oma_version - -# where mpi_name(inequality) and oma_version(equality) -- kind: MpiInfo - properties: - - name: oma_version - - name: mpi_name - -# where mpi_version(equality) and oma_version(inequality) -- kind: MpiInfo - properties: - - name: mpi_version - - name: oma_version - -# where mpi_version(inequality) and oma_version(equality) -- kind: MpiInfo - properties: - - name: oma_version - - name: mpi_version - -######################################################## -# select * from CompilerInfo -######################################################## - -# where compiler_name(equality) and compiler_version(inequality) -- kind: CompilerInfo - properties: - - name: compiler_name - - name: compiler_version - -# where compiler_name(inequality) and compiler_version(equality) -- kind: CompilerInfo - properties: - - name: compiler_version - - name: compiler_name - -######################################################## -# select * from ClusterInfo -######################################################## - -# where cluster_name(equality) and node_count(inequality) -- kind: ClusterInfo - properties: - - name: cluster_name - - name: node_count - -# where cluster_name(inequality) and node_count(equality) -- kind: ClusterInfo - properties: - - name: node_count - - name: cluster_name - -# where node_hostname(equality) and node_count(inequality) -- kind: ClusterInfo - properties: - - name: node_hostname - - name: node_count - -# where node_hostname(inequality) and node_count(equality) -- kind: ClusterInfo - properties: - - name: node_count - - name: node_hostname - -######################################################## -# select * from MpiInstallPhase -######################################################## - -# where status(equality) and start_time(inequality) -- kind: MpiInstallPhase - properties: - - name: status - - name: start_time - -# where status(equality) and end_time(inequality) -- kind: MpiInstallPhase - properties: - - name: status - - name: end_time - -######################################################## -# select * from TestBuildPhase -######################################################## - -# where status(equality) and start_time(inequality) -- kind: TestBuildPhase - properties: - - name: status - - name: start_time - -# where status(equality) and end_time(inequality) -- kind: TestBuildPhase - properties: - - name: status - - name: end_time - -######################################################## -# select * from TestRunPhase -######################################################## - -# where status(equality) and start_time(inequality) -- kind: TestRunPhase - properties: - - name: status - - name: start_time - -# where status(equality) and end_time(inequality) -- kind: TestRunPhase - properties: - - name: status - - name: end_time - -# where status(equality) and tag(inequality) -- kind: TestRunPhase - properties: - - name: status - - name: tag - -# where status(equality) and tag(equality) and start_time(inequality) -- kind: TestRunPhase - properties: - - name: status - - name: tag - - name: start_time - -# where status(equality) and tag(equality) and end_time(inequality) -- kind: TestRunPhase - properties: - - name: status - - name: tag - - name: end_time - -# where status(equality) and test_name(equality) and start_time(inequality) -- kind: TestRunPhase - properties: - - name: status - - name: test_name - - name: start_time - -# where status(equality) and test_name(equality) and end_time(inequality) -- kind: TestRunPhase - properties: - - name: status - - name: test_name - - name: end_time - -# where status(equality) and tag(equality) and test_name(equality) and start_time(inequality) -- kind: TestRunPhase - properties: - - name: status - - name: tag - - name: test_name - - name: start_time - -# where status(equality) and tag(equality) and test_name(equality) and end_time(inequality) -- kind: TestRunPhase - properties: - - name: status - - name: tag - - name: test_name - - name: end_time - - - -# AUTOGENERATED - -# This index.yaml is automatically updated whenever the dev_appserver -# detects that a new type of query is run. If you want to manage the -# index.yaml file manually, remove the above marker line (the line -# saying "# AUTOGENERATED"). If you want to manage some indexes -# manually, move them above the marker line. The index.yaml file is -# automatically uploaded to the admin console when you next deploy -# your application using appcfg.py. diff --git a/server/gds/main.py b/server/gds/main.py deleted file mode 100755 index 2276c0246..000000000 --- a/server/gds/main.py +++ /dev/null @@ -1,1204 +0,0 @@ -#!/usr/bin/env python - -# -# Copyright (c) 2009 Voltaire -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# Python Standart modules -import os -import sys -import datetime -import types -import string -import logging -import cStringIO -import mimetypes -import re -import urllib - -# Python Google AppEngine SDK modules -import yaml -from google.appengine.api import users -from google.appengine.api import datastore -from google.appengine.api import datastore_errors -from google.appengine.api import datastore_types - -from google.appengine.ext import db -from google.appengine.ext import webapp -from google.appengine.ext.webapp.util import run_wsgi_app -from google.appengine.ext.webapp import template - -from google.appengine.runtime import DeadlineExceededError - -# Private modules -import conf -import models -import pager -import auth - - -class MainPage(webapp.RequestHandler): - "Output data collection view" - - user = None - - def get(self): - - if not users.is_current_user_admin(): - greeting = ("Sign in or register." % - users.create_login_url("/")) - self.response.set_status(401) # Unauthorized - self.response.out.write("%s" % greeting) - return 0 - - logging.debug('%s: get as a user %s=> %s' % (self.__class__.__name__, str(self.user), self.request.GET)) - status = 0 - - query_str = self.request.get('querytext') - result_count = 0 - result_set = None - try: - if (query_str == ''): - bookmark = self.request.get('bookmark') - query_collection = pager.PagerQuery(models.TestRunPhase) - prev, result_set, next = query_collection.fetch(15, bookmark) - result_count = query_collection._get_query().count() - - # query.count() is not used because of count() ignores the LIMIT clause on GQL queries. -# for result in query_collection._get_query(): -# result_count += 1 - else: - temp_query_str = query_str - (temp_query_str, status) = gql_ex(temp_query_str) - - query_collection = db.GqlQuery(temp_query_str) - result_set = query_collection - prev = '' - next = '' - - # query.count() is not used because of count() ignores the LIMIT clause on GQL queries. - for result in result_set: - result_count += 1 - - except datastore_errors.BadQueryError: - result_set = None - result_count = 0 - prev = '' - next = '' - query_str == '' - - if os.environ.has_key('APPLICATION_ID'): - app_id=os.environ['APPLICATION_ID'] - - template_context = { - 'query_collection': result_set, - 'query_count': result_count, - 'prev': prev, - 'next': next, - 'app_id': app_id, - 'query_str': query_str - } - - path = os.path.join(os.path.dirname(__file__), os.path.join('templates', 'index.html')) - self.response.out.write(template.render(path, template_context, debug=conf.DEBUG)) - - -class LoginHandler(webapp.RequestHandler): - def get(self): - """ It displays a login form that POSTs to the same URL. - - """ - - logging.debug('%s: get=> %s' % (self.__class__.__name__, self.request.GET)) - status = 0 - - template_context = { - 'action': self.request.uri, - 'next': self.request.get('next') - } - - path = os.path.join(os.path.dirname(__file__), os.path.join('templates', 'login.html')) - self.response.out.write(template.render(path, template_context, debug=conf.DEBUG)) - - if (status == 0): status = 200 - self.response.set_status(status) - - return status - - def post(self): - """ It tries to log the user in. If login is successful, - the view redirects to the URL specified in ``next``. If - ``next`` isn't provided, it redirects to ```` (which is - currently hard-coded). If login isn't successful, it redisplays the login form. - - """ - - logging.debug('%s: post=> %s' % (self.__class__.__name__, self.request.POST)) - status = 0 - - credential = [self.request.get('username_'), self.request.get('password_')] - user = auth.authenticate(credential) - - url = '' - if user is None: - url = self.request.uri - status = 401 # Unauthorized - - else: - url = '%s' % (self.request.get('next')) - status = 302 # Found - - self.response.set_status(status) - self.redirect(url) - - return status - - -class DownloadHandler(webapp.RequestHandler): - def get(self): - logging.debug('%s: get=> %s' % (self.__class__.__name__, self.request.GET)) - status = 0 - - id = self.request.get('data_file') - data_file = models.TestRunPhase.get_by_id(int(id)).data_file - if data_file: - self.response.headers['Content-Type'] = 'application/zip' - self.response.out.write(data_file) - else: - status = 404 # Not found - - if (status == 0): status = 200 - self.response.set_status(status) - - return status - - -class ClientHandler(webapp.RequestHandler): - user = None - - def _ping(self): - status = 0 - if 'PING' in self.request.arguments(): - self.response.headers['Content-Type'] = 'text/html' - self.response.out.write("Ping is successful.\n") - else: - status = 400 - - return status - - def _submit(self): - status = 0 - respond = '' - if 'SUBMIT' in self.request.arguments(): - req_data_file = yaml.load(self.request.get('data')) - data_file = req_data_file['modules'] - - if not status and data_file.has_key('SubmitInfo'): - if (data_file['SubmitInfo'].has_key('hostname') and - data_file['SubmitInfo'].has_key('local_username') and - data_file['SubmitInfo'].has_key('http_username') and - data_file['SubmitInfo'].has_key('mtt_version')): - - submits = models.SubmitInfo.all() - if (data_file['SubmitInfo']['hostname'] is not None): submits.filter('hostname =', str(data_file['SubmitInfo']['hostname'])) - if (data_file['SubmitInfo']['local_username'] is not None): submits.filter('local_username =', str(data_file['SubmitInfo']['local_username'])) - if (data_file['SubmitInfo']['http_username'] is not None): submits.filter('http_username =', str(data_file['SubmitInfo']['http_username'])) - if (data_file['SubmitInfo']['mtt_version'] is not None): submits.filter('mtt_version =', str(data_file['SubmitInfo']['mtt_version'])) - - logging.debug("SubmitInfo count = %d\n" % submits.count()) - if submits.count() == 0: - submit = models.SubmitInfo() - self.__fill_entity(submit, data_file['SubmitInfo']) - submit.put() - else: - submit = submits[0] - else: - logging.error("Incorrect format of 'SubmitInfo'") - respond = "Incorrect format of 'SubmitInfo'" - status = 400 - - if not status and data_file.has_key('MpiInfo'): - if (data_file['MpiInfo'].has_key('mpi_name') and - data_file['MpiInfo'].has_key('mpi_version') and - data_file['MpiInfo'].has_key('oma_version')): - - mpis = models.MpiInfo.all() - if (data_file['MpiInfo']['mpi_name'] is not None): mpis.filter('mpi_name =', str(data_file['MpiInfo']['mpi_name'])) - if (data_file['MpiInfo']['mpi_version'] is not None): mpis.filter('mpi_version =', str(data_file['MpiInfo']['mpi_version'])) - if (data_file['MpiInfo']['oma_version'] is not None): mpis.filter('oma_version =', str(data_file['MpiInfo']['oma_version'])) - - logging.debug("MpiInfo count = %d\n" % mpis.count()) - if mpis.count() == 0: - mpi = models.MpiInfo() - self.__fill_entity(mpi, data_file['MpiInfo']) - mpi.put() - else: - mpi = mpis[0] - else: - logging.error("Incorrect format of 'MpiInfo'") - respond = "Incorrect format of 'MpiInfo'" - status = 400 - - if not status and data_file.has_key('ClusterInfo'): - if (data_file['ClusterInfo'].has_key('cluster_name') and - data_file['ClusterInfo'].has_key('node_count') and - data_file['ClusterInfo'].has_key('node_hostname') and - data_file['ClusterInfo'].has_key('node_arch') and - data_file['ClusterInfo'].has_key('node_ncpu') and - data_file['ClusterInfo'].has_key('node_nsocket') and - data_file['ClusterInfo'].has_key('node_htt') and - data_file['ClusterInfo'].has_key('node_mem') and - data_file['ClusterInfo'].has_key('node_cache') and - data_file['ClusterInfo'].has_key('node_mhz') and - data_file['ClusterInfo'].has_key('node_os_kernel') and - data_file['ClusterInfo'].has_key('node_os_vendor') and - data_file['ClusterInfo'].has_key('node_os_release') and - data_file['ClusterInfo'].has_key('net_pci') and - data_file['ClusterInfo'].has_key('net_conf') and - data_file['ClusterInfo'].has_key('net_eth100') and - data_file['ClusterInfo'].has_key('net_eth1000') and - data_file['ClusterInfo'].has_key('net_eth10k') and - data_file['ClusterInfo'].has_key('net_iwarp') and - data_file['ClusterInfo'].has_key('net_ibddr') and - data_file['ClusterInfo'].has_key('net_ibqdr')): - - clusters = models.ClusterInfo.all() - if (data_file['ClusterInfo']['cluster_name'] is not None): clusters.filter('cluster_name =', data_file['ClusterInfo']['cluster_name']) - if (data_file['ClusterInfo']['node_count'] is not None): clusters.filter('node_count =', data_file['ClusterInfo']['node_count']) - if (data_file['ClusterInfo']['node_hostname'] is not None): clusters.filter('node_hostname =', data_file['ClusterInfo']['node_hostname']) - if (data_file['ClusterInfo']['node_arch'] is not None): clusters.filter('node_arch =', data_file['ClusterInfo']['node_arch']) - if (data_file['ClusterInfo']['node_ncpu'] is not None): clusters.filter('node_ncpu =', data_file['ClusterInfo']['node_ncpu']) - if (data_file['ClusterInfo']['node_nsocket'] is not None): clusters.filter('node_nsocket =', data_file['ClusterInfo']['node_nsocket']) - if (data_file['ClusterInfo']['node_htt'] is not None): clusters.filter('node_htt =', data_file['ClusterInfo']['node_htt']) - if (data_file['ClusterInfo']['node_mem'] is not None): clusters.filter('node_mem =', data_file['ClusterInfo']['node_mem']) - if (data_file['ClusterInfo']['node_cache'] is not None): clusters.filter('node_cache =', data_file['ClusterInfo']['node_cache']) - if (data_file['ClusterInfo']['node_mhz'] is not None): clusters.filter('node_mhz =', data_file['ClusterInfo']['node_mhz']) - if (data_file['ClusterInfo']['node_os_kernel'] is not None): clusters.filter('node_os_kernel =', str(data_file['ClusterInfo']['node_os_kernel'])) - if (data_file['ClusterInfo']['node_os_vendor'] is not None): clusters.filter('node_os_vendor =', str(data_file['ClusterInfo']['node_os_vendor'])) - if (data_file['ClusterInfo']['node_os_release'] is not None): clusters.filter('node_os_release =', str(data_file['ClusterInfo']['node_os_release'])) - if (data_file['ClusterInfo']['net_eth100'] is not None): clusters.filter('net_eth100 =', data_file['ClusterInfo']['net_eth100']) - if (data_file['ClusterInfo']['net_eth1000'] is not None): clusters.filter('net_eth1000 =', data_file['ClusterInfo']['net_eth1000']) - if (data_file['ClusterInfo']['net_eth10k'] is not None): clusters.filter('net_eth10k =', data_file['ClusterInfo']['net_eth10k']) - if (data_file['ClusterInfo']['net_iwarp'] is not None): clusters.filter('net_iwarp =', data_file['ClusterInfo']['net_iwarp']) - if (data_file['ClusterInfo']['net_ibddr'] is not None): clusters.filter('net_ibddr =', data_file['ClusterInfo']['net_ibddr']) - if (data_file['ClusterInfo']['net_ibqdr'] is not None): clusters.filter('net_ibqdr =', data_file['ClusterInfo']['net_ibqdr']) - - logging.debug("ClusterInfo count = %d\n" % clusters.count()) - if clusters.count() == 0: - cluster = models.ClusterInfo() - self.__fill_entity(cluster, data_file['ClusterInfo']) - cluster.put() - else: - cluster = clusters[0] - else: - logging.error("Incorrect format of 'ClusterInfo'") - respond = "Incorrect format of 'ClusterInfo'" - status = 400 - - if not status and data_file.has_key('CompilerInfo'): - if (data_file['CompilerInfo'].has_key('compiler_name') and - data_file['CompilerInfo'].has_key('compiler_version')): - - compilers = models.CompilerInfo.all() - if (data_file['CompilerInfo']['compiler_name'] is not None): compilers.filter('compiler_name =', str(data_file['CompilerInfo']['compiler_name'])) - if (data_file['CompilerInfo']['compiler_version'] is not None): compilers.filter('compiler_version =', str(data_file['CompilerInfo']['compiler_version'])) - - logging.debug("CompilerInfo count = %d\n" % compilers.count()) - if compilers.count() == 0: - compiler = models.CompilerInfo() - self.__fill_entity(compiler, data_file['CompilerInfo']) - compiler.put() - else: - compiler = compilers[0] - else: - logging.error("Incorrect format of 'CompilerInfo'") - respond = "Incorrect format of 'CompilerInfo'" - status = 400 - - if not status and data_file.has_key('SuiteInfo'): - if (data_file['SuiteInfo'].has_key('suite_name') and - data_file['SuiteInfo'].has_key('suite_version')): - - suites = models.SuiteInfo.all() - if (data_file['SuiteInfo']['suite_name'] is not None): suites.filter('suite_name =', str(data_file['SuiteInfo']['suite_name'])) - if (data_file['SuiteInfo']['suite_version'] is not None): suites.filter('suite_version =', str(data_file['SuiteInfo']['suite_version'])) - - logging.debug("SuiteInfo count = %d\n" % suites.count()) - if suites.count() == 0: - suite = models.SuiteInfo() - self.__fill_entity(suite, data_file['SuiteInfo']) - suite.put() - else: - suite = suites[0] - else: - logging.error("Incorrect format of 'SuiteInfo'") - respond = "Incorrect format of 'SuiteInfo'" - status = 400 - - if not status and data_file.has_key('MpiInstallPhase'): - try: - submit, cluster, mpi, compiler - - mpi_install_phases = models.MpiInstallPhase.all() - mpi_install_phases.filter('submitinfo =', submit) - mpi_install_phases.filter('clusterinfo =', cluster) - mpi_install_phases.filter('mpiinfo =', mpi) - mpi_install_phases.filter('compilerinfo =', compiler) - - logging.debug("MpiInstallPhase count = %d\n" % mpi_install_phases.count()) - if mpi_install_phases.count() == 0: - mpi_install_phase = models.MpiInstallPhase() - mpi_install_phase.submitinfo = submit - mpi_install_phase.clusterinfo = cluster - mpi_install_phase.mpiinfo = mpi - mpi_install_phase.compilerinfo = compiler - - excluded_list = [ - 'submitinfo', - 'clusterinfo', - 'mpiinfo', - 'compilerinfo' - ] - self.__fill_entity(mpi_install_phase, data_file['MpiInstallPhase'], excluded_list, True) - - mpi_install_phase.put() - else: - mpi_install_phase = mpi_install_phases[0] - - except (NameError), err: - logging.error("Fatal error: error='%s'\n" % (err)) - respond = "Fatal error" - status = 500 - - if not status and data_file.has_key('TestBuildPhase'): - try: - submit, cluster, mpi, compiler, suite, mpi_install_phase - - test_build_phases = models.TestBuildPhase.all() - test_build_phases.filter('submitinfo =', submit) - test_build_phases.filter('clusterinfo =', cluster) - test_build_phases.filter('mpiinfo =', mpi) - test_build_phases.filter('compilerinfo =', compiler) - test_build_phases.filter('suiteinfo =', suite) - test_build_phases.filter('mpiinstallphase =', mpi_install_phase) - - logging.debug("TestBuildPhase count = %d\n" % test_build_phases.count()) - if test_build_phases.count() == 0: - test_build_phase = models.TestBuildPhase() - test_build_phase.submitinfo = submit - test_build_phase.clusterinfo = cluster - test_build_phase.mpiinfo = mpi - test_build_phase.compilerinfo = compiler - test_build_phase.suiteinfo = suite - test_build_phase.mpiinstallphase = mpi_install_phase - - excluded_list = [ - 'submitinfo', - 'clusterinfo', - 'mpiinfo', - 'compilerinfo', - 'suiteinfo', - 'mpiinstallphase' - ] - self.__fill_entity(test_build_phase, data_file['TestBuildPhase'], excluded_list, True) - - test_build_phase.put() - else: - test_build_phase = test_build_phases[0] - - except (NameError), err: - logging.error("Fatal error: error='%s'\n" % (err)) - respond = "Fatal error" - status = 500 - - if not status and data_file.has_key('TestRunPhase'): - try: - submit, cluster, mpi, compiler, suite, mpi_install_phase, test_build_phase - - test_run_phase = models.TestRunPhase() - test_run_phase.submitinfo = submit - test_run_phase.clusterinfo = cluster - test_run_phase.mpiinfo = mpi - test_run_phase.compilerinfo = compiler - test_run_phase.suiteinfo = suite - test_run_phase.mpiinstallphase = mpi_install_phase - test_run_phase.testbuildphase = test_build_phase - - excluded_list = [ - 'submitinfo', - 'clusterinfo', - 'mpiinfo', - 'compilerinfo', - 'suiteinfo', - 'mpiinstallphase', - 'testbuildphase', - 'data_file' - ] - self.__fill_entity(test_run_phase, data_file['TestRunPhase'], excluded_list, True) - - if 'raw' in self.request.arguments() : test_run_phase.data_file = db.Blob(self.request.get('raw')) - - test_run_phase.cached_submitinfo_hostname = submit.hostname - test_run_phase.cached_submitinfo_local_username = submit.local_username - test_run_phase.cached_submitinfo_http_username = submit.http_username - test_run_phase.cached_submitinfo_mtt_version = submit.mtt_version - - test_run_phase.cached_clusterinfo_cluster_name = cluster.cluster_name - test_run_phase.cached_clusterinfo_node_count = cluster.node_count - test_run_phase.cached_clusterinfo_node_hostname = cluster.node_hostname - test_run_phase.cached_clusterinfo_node_arch = cluster.node_arch - test_run_phase.cached_clusterinfo_node_ncpu = cluster.node_ncpu - test_run_phase.cached_clusterinfo_node_nsocket = cluster.node_nsocket - test_run_phase.cached_clusterinfo_node_htt = cluster.node_htt - test_run_phase.cached_clusterinfo_node_mem = cluster.node_mem - test_run_phase.cached_clusterinfo_node_cache = cluster.node_cache - test_run_phase.cached_clusterinfo_node_mhz = cluster.node_mhz - test_run_phase.cached_clusterinfo_node_os_kernel = cluster.node_os_kernel - test_run_phase.cached_clusterinfo_node_os_vendor = cluster.node_os_vendor - test_run_phase.cached_clusterinfo_node_os_release = cluster.node_os_release - test_run_phase.cached_clusterinfo_net_eth100 = cluster.net_eth100 - test_run_phase.cached_clusterinfo_net_eth1000 = cluster.net_eth1000 - test_run_phase.cached_clusterinfo_net_eth10k = cluster.net_eth10k - test_run_phase.cached_clusterinfo_net_iwarp = cluster.net_iwarp - test_run_phase.cached_clusterinfo_net_ibddr = cluster.net_ibddr - test_run_phase.cached_clusterinfo_net_ibqdr = cluster.net_ibqdr - test_run_phase.cached_mpiinfo_mpi_name = mpi.mpi_name - test_run_phase.cached_mpiinfo_mpi_version = mpi.mpi_version - test_run_phase.cached_mpiinfo_oma_version = mpi.oma_version - test_run_phase.cached_compilerinfo_compiler_name = compiler.compiler_name - test_run_phase.cached_compilerinfo_compiler_version = compiler.compiler_version - test_run_phase.cached_suiteinfo_suite_name = suite.suite_name - test_run_phase.cached_suiteinfo_suite_version = suite.suite_version - test_run_phase.cached_mpiinstallphase_mpi_path = mpi_install_phase.mpi_path - - test_run_phase.put() - - except (NameError), err: - logging.error("Fatal error: error='%s'\n" % (err)) - respond = "Fatal error" - status = 500 - - self.response.headers['Content-Type'] = 'text/html' - self.response.out.write("Data is submitted.\n" ) - else: - status = 400 - - return status - - def _query(self): - status = 0 - respond = '' - if 'QUERY' in self.request.arguments(): - query_str = '' - if ('gql' in self.request.arguments()): - query_str = self.request.get('gql') - if (not query_str): - status = 400 - - if (not status): - match = re.search(r"\s+[Ff][Rr][Oo][Mm]\s+([\w]+)\s*", query_str) - available_set = [ 'TestRunPhase', 'TestBuildPhase', 'MpiInstallPhase' ] - if (not match is None and - not match.group(1) in available_set): - query_str = '' - status = 400 - - if (not status): - try: - (query_str, status) = gql_ex(query_str) - - if (not status and query_str): - query = db.GqlQuery(query_str) - result_set = query - except (datastore_errors.BadQueryError, datastore_errors.BadArgumentError, datastore_errors.BadRequestError, datastore_errors.BadRequestError, datastore_errors.BadFilterError, db.KindError), err: - logging.error("Incorrect GQL line: <_query> GQL='%s' error='%s'\n" % - (query_str, err)) - respond = str(err) - status = 400 - except (datastore_errors.NeedIndexError), err: - logging.error("No matching index found: <_query> GQL='%s' error='%s'\n" % - (query_str, err)) - respond = str(err) - status = 400 - - if not status and result_set: - result_count = 0 - data_file = {} - data_file['data'] = [] - for entity in result_set: - raw = None - data_entity = {} - data_entity['key'] = "key_%s" % str(entity.key().id()) - data_entity['modules'] = {} - entities_list = [ entity ] - - if 'no-ref' not in self.request.arguments(): - for prop in entity.properties().values(): - if (datastore.typename(prop) in ['ReferenceProperty']): - val = prop.get_value_for_datastore(entity) - if val is not None: - query_entity = db.GqlQuery("select * from %s where __key__=key('%s')" % (val.kind(), val)) - entities_list.append(query_entity.get()) - - for temp_entity in entities_list: - key_dict = {} - for prop in temp_entity.properties().values(): - if (datastore.typename(prop) not in ['BlobProperty', 'ReferenceProperty']): - val = prop.get_value_for_datastore(temp_entity) - - if (datastore.typename(prop) in ['EmailProperty', 'BooleanProperty']): - val = str(val) - elif (datastore.typename(prop) in ['TextProperty']): - val = unicode(val) - - if val is None or val == 'unknown': - val = '' - - key_dict[prop.name] = val - - if ( prop.name == 'data_file' and - 'no-raw' not in self.request.arguments() and - datastore.typename(prop) in ['BlobProperty']): - raw = str(prop.get_value_for_datastore(temp_entity)) - if (raw is not None): data_entity['raw'] = raw - - for prop in temp_entity.dynamic_properties(): - val = temp_entity.__getattr__(prop) - - if (type(val).__name__ not in ['Blob', 'Key']): - if (type(val).__name__ in ['Email', 'BooleanProperty']): - val = str(val) - elif (type(val).__name__ in ['Text']): - val = unicode(val) - - if val is None or val == 'unknown': - val = '' - - key_dict[prop] = val - - data_entity['modules'][temp_entity.kind()] = key_dict - - data_file['data'].append(data_entity) - data_file['last_key'] = str(entity.key()) - result_count += 1 - - data_file['count'] = result_count - respond = yaml.safe_dump(data_file, default_flow_style=False, canonical=False) - - self.response.headers['Content-Type'] = 'text/yaml' - self.response.out.write(respond) - else: - status = 400 - - return status - - def _view(self): - status = 0 - respond = '' - if 'VIEW' in self.request.arguments(): - - data = [] - query_str = '' - if ('gql' in self.request.arguments()): - query_str = self.request.get('gql') - if (not query_str): - status = 400 - - if (not status and not self.user.is_superuser): - match = re.search(r"\s+[Ff][Rr][Oo][Mm]\s+([\w]+)\s*", query_str) - exception_set = [ 'User' ] - if (not match is None and match.group(1) in exception_set): - query_str = '' - status = 400 - - elif (self.request.get('kind') == 'all'): - key_values = {} - model_set = [ - models.MpiInstallPhase, - models.TestBuildPhase, - models.TestRunPhase, - models.SubmitInfo, - models.SuiteInfo, - models.MpiInfo, - models.ClusterInfo, - models.CompilerInfo - ] - for model in model_set: - key_values = self.__get_info(model) - data.append({'tag': str(model.kind()), 'data': key_values}) - - elif (self.request.get('kind') == 'suite'): - query_str = 'select * from SuiteInfo' - - elif (self.request.get('kind') == 'mpi'): - query_str = 'select * from MpiInfo' - - elif (self.request.get('kind') == 'compiler'): - query_str = 'select * from CompilerInfo' - - elif (self.request.get('kind') == 'cluster'): - query_str = 'select * from ClusterInfo' - - else : - query_str = '' - - try: - (query_str, status) = gql_ex(query_str) - - if (not status and query_str): - if (re.search(r"\s*[Ss][Ee][Ll][Ee][Cc][Tt]\s+([Cc][Oo][Uu][Nn][Tt]\s*\(\s*\*\s*\)\s+)", - query_str)): - # Queries that return keys are faster and cost less CPU than queries that return entities, since the keys themselves are already in the index, - # so the query doesn't need to fetch the actual entities. - query_str = re.sub(r"\s*[Ss][Ee][Ll][Ee][Cc][Tt]\s+([Cc][Oo][Uu][Nn][Tt]\s*\(\s*\*\s*\)\s+)", - "select __key__ ", - query_str) - query = db.GqlQuery(query_str) - result_set = query - # query.count() is not used because of count() ignores the LIMIT clause on GQL queries. - result_count = 0 - for result in result_set: - result_count += 1 - - key_values = {'count': [str(result_count)]} - data.append({'tag': '', 'data': key_values}) - - elif (re.search(r"\s*[Ss][Ee][Ll][Ee][Cc][Tt]\s+([\w\,\s]+)\s+[Ff][Rr][Oo][Mm]", - query_str)): - match = re.search(r"\s*[Ss][Ee][Ll][Ee][Cc][Tt]\s+([\w\,\s]+)\s+[Ff][Rr][Oo][Mm]", - query_str) - query_str = re.sub(r"\s*[Ss][Ee][Ll][Ee][Cc][Tt]\s+([\w\,\s]+)\s+[Ff][Rr][Oo][Mm]", - "select * from", - query_str) - fields = [] - fields = re.split('\W+', match.group(1)) - - query = db.GqlQuery(query_str) - result_set = query - - key_values = {} - key_values = get_table_data(result_set) - - data.append({'tag': '', 'data': {}}) - for key in fields: - if key_values.has_key(key): - data[0]['data'][key] = key_values[key] - - elif (query_str): - query = db.GqlQuery(query_str) - result_set = query - - key_values = {} - key_values = get_table_data(result_set) - if key_values.has_key('_key_'): - del key_values['_key_'] - - data.append({'tag': '', 'data': key_values}) - - except (datastore_errors.BadQueryError, datastore_errors.BadArgumentError, datastore_errors.BadRequestError, datastore_errors.BadFilterError, db.KindError), err: - logging.error("Incorrect GQL line: <_view> GQL='%s' error='%s'\n" % - (query_str, err)) - respond = str(err) - status = 400 - except (datastore_errors.NeedIndexError), err: - logging.error("No matching index found: <_query> GQL='%s' error='%s'\n" % - (query_str, err)) - respond = str(err) - status = 400 - - if (not status and - 'format' in self.request.arguments()): - if (self.request.get('format') == 'txt'): - respond += self.__do_txt(data, len(data)) - elif (self.request.get('format') == 'html'): - respond += self.__do_html(data, len(data)) - elif (self.request.get('format') == 'yaml'): - respond += self.__do_yaml(data, len(data)) - elif (self.request.get('format') == 'raw'): - respond += str(data) - respond += '\n' - - self.response.headers['Content-Type'] = 'text/html' - self.response.out.write(respond) - else: - status = 400 - - return status - - def __fill_entity(self, entity, data, excluded_list = None, dynamic = False): - """Fill entity with values from data. - - """ - if (excluded_list is None): - excluded_list = [] - - for key, value in data.iteritems(): - # Set field values that are defined in model - if (key in entity.properties() and - value is not None and - key not in excluded_list): - try: - prop = entity.properties()[key] - if (datastore.typename(prop) in ['FloatProperty']): - entity.__setattr__(key, prop.validate(float(value))) - elif (datastore.typename(prop) in ['IntegerProperty']): - entity.__setattr__(key, prop.validate(int(value))) - elif (datastore.typename(prop) in ['StringProperty']): - entity.__setattr__(key, prop.validate(str(value))) - elif (datastore.typename(prop) in ['TextProperty']): - entity.__setattr__(key, prop.validate(unicode(value))) - elif (datastore.typename(prop) in ['DateTimeProperty'] and - type(value).__name__ in ['str', 'unicode']): - value = value.strip('\'') - value = datetime.datetime.strptime(str(value), '%Y-%m-%d %H:%M:%S') - entity.__setattr__(key, prop.validate(value)) - elif (datastore.typename(prop) in ['ListProperty', 'StringListProperty'] and - type(value).__name__ in ['str', 'unicode']): - value = value.lstrip('(') - value = value.rstrip(')') - value = value.strip() - value = re.sub(r"\'\s+\'", "\'\t\'", value) - value = re.sub(r"\'", "", value) - value = value.split('\t') - entity.__setattr__(key, prop.validate(value)) - else: - entity.__setattr__(key, prop.validate(value)) - except (datastore_errors.BadValueError, ValueError, TypeError), err: - logging.error("Incorrect value: <__fill_entity> entity=%s field=%s value=%s %s error='%s'\n" % - (entity.kind(), key, value, value.__class__, err)) - - # Dynamically set field values that are not defined - elif (dynamic == True and - key not in entity.properties() and - value is not None): - # This code gives possibility to limit adding dynamic fields by prefix - is_field_added = False - for field_prefix in ['data_', 'custom_']: - if (string.find(key, field_prefix) == 0): - entity.__setattr__(key, value) - is_field_added = True - logging.debug("Added dynamic field: <__fill_entity> entity=%s field=%s value=%s %s\n" % - (entity.kind(), key, value, value.__class__)) - break - if (is_field_added == False): - logging.error("Incorrect dynamic field: <__fill_entity> entity=%s field=%s value=%s %s\n" % - (entity.kind(), key, value, value.__class__)) - - # Undefined field - value pair - elif (value is not None): - logging.error("Invalid field: <__fill_entity> entity=%s field=%s value=%s %s\n" % - (entity.kind(), key, value, value.__class__)) - - - def __get_info(self, model): - """Returns the union of model names used by the given list of models. - - We return the union as a dictionary mapping the model names to a property - information. - """ - key_dict = {'name': [], 'type': []} - for key, prop in sorted(model.properties().iteritems()): - key_dict['name'].append(str(key)) - key_dict['type'].append(str(datastore.typename(prop))) - - return key_dict - - def __do_html(self, dataset, count): - """Returns the html-formated respond. - - """ - form = '' - # set head - form += """ - - MTT-VIEW - -""" - for j in range(count): - - data = dataset[j]['data'] - row_count = 0; - i = 0; - width = {}; - - # calculate size of data - for key, values in sorted(data.iteritems()): - if (len(data[key]) > row_count): row_count = len(values) - width[key] = len(key); - for i in range(len(values)): - if (len(data[key][i]) > width[key]): width[key] = len(data[key][i]) - - if (dataset[j].has_key('tag') and dataset[j]['tag']): - form += "

" + str(dataset[j]['tag']) + "

" - - # set table - form += "" - - # dispaly names of columns - form += "" - for key in sorted(data.keys()): - form += "" - form += "" - - form += "" - - # show data - for i in range(row_count): - if i % 2: - form += "" - else: - form += "" - for key, values in sorted(data.iteritems()): - form += "" - form += "" - - form += "" - - form += "
" + str(key) + "
" - if ( i < len(data[key]) and data[key][i] != ''): - form += data[key][i]; - else: - form += '' - form += "
" - form += "

total: " + str(row_count) + "

" - form += "
" - - return form - - def __do_txt(self, dataset, count): - """Returns the txt-formated respond. - - """ - form = '' - for j in range(count): - - data = dataset[j]['data'] - row_count = 0 - format = '' - i = 0 - width = {} - - # calculate size of data - for key, values in sorted(data.iteritems()): - if (len(data[key]) > row_count): row_count = len(values) - width[key] = len(key); - for i in range(len(values)): - if (len(data[key][i]) > width[key]): width[key] = len(data[key][i]) - - if (dataset[j].has_key('tag') and dataset[j]['tag']): - form += str(dataset[j]['tag']) - - # dispaly names of columns - form += "\n" - for key in sorted(data.keys()): - format = "%-" + str(width[key]) + "s " - form += format % key - - form += "\n" - for key in sorted(data.keys()): - form += "-" * width[key] - form += " " - - # show data - form += "\n" - for i in range(row_count): - for key, values in sorted(data.iteritems()): - if ( i < len(data[key]) and data[key][i] != ''): - format = "%-" + str(width[key]) + "s " - form += format % str(data[key][i]) - else: - format = "%-" + str(width[key]) + "s " - form += format % '' - form += "\n" - - form += "\n" - form += "total: " + str(row_count) - form += "\n" - - return form - - def __do_yaml(self, dataset, count): - """Returns the yaml-formated respond. - - """ - form = yaml.safe_dump(dataset, default_flow_style=False, canonical=False); - - return form - - def _admin(self): - status = 0 - if 'ADMIN' in self.request.arguments(): - if ('_NEWUSER_' in self.request.arguments()): - user = auth.add_user(username = self.request.get('username')) - if user is None: - status = 400 - else: - self.response.headers['Content-Type'] = 'text/html' - self.response.out.write("Operation has been completed successfully.\n") - else: - status = 400 - - return status - - def _update(self): - status = 0 - respond = '' - if 'UPDATE' in self.request.arguments(): - query_str = '' - if ('gql' in self.request.arguments()): - query_str = self.request.get('gql') - if (not query_str): - status = 400 - - if (not status): - match = re.search(r"^\s*[Uu][Pp][Dd][Aa][Tt][Ee]\s+(\w+)\s+[Ss][Ee][Tt]\s+([^\"]+)\s+[Ww][Hh][Ee][Rr][Ee]\s+([^\"]+)", query_str) - available_set = [ 'TestRunPhase', 'TestBuildPhase', 'MpiInstallPhase' ] - if (not match is None and - not match.group(1) in available_set): - query_str = '' - status = 400 - - if (not status): - fields = [] - fields = re.split(',', match.group(2)) - key_dict = {} - for set_value in fields: - (key, value) = re.split('=', set_value) - key_dict[key.strip()] = value.strip() - - query_str = 'select * from ' + match.group(1) + ' where ' + match.group(3) - try: - (query_str, status) = gql_ex(query_str) - - if (not status and query_str): - query = db.GqlQuery(query_str) - result_set = query - - excluded_list = [ - 'submitinfo', - 'clusterinfo', - 'mpiinfo', - 'compilerinfo', - 'suiteinfo', - 'mpiinstallphase', - 'testbuildphase', - 'data_file' - ] - for entity in result_set: - self.__fill_entity(entity, key_dict, excluded_list, True) - entity.put() - - except (datastore_errors.BadQueryError, datastore_errors.BadArgumentError, datastore_errors.BadRequestError, datastore_errors.BadRequestError, datastore_errors.BadFilterError, db.KindError), err: - logging.error("Incorrect GQL line: <_update> GQL='%s' error='%s'\n" % - (query_str, err)) - respond = str(err) - status = 400 - except (datastore_errors.NeedIndexError), err: - logging.error("No matching index found: <_update> GQL='%s' error='%s'\n" % - (query_str, err)) - respond = str(err) - status = 400 - - self.response.headers['Content-Type'] = 'text/html' - self.response.out.write(respond) - else: - status = 400 - - return status - - def post(self): - logging.debug('%s: post=> %s' % (self.__class__.__name__, self.request.POST)) - status = 0 - respond = '' - -########################################################################################### -# COMMENT FOLLOWING THREE LINES IN ORDER TO SKIP AUTHORIZATION WHEN GDS IS RUNNING LOCALLY - self.user = auth.authenticate(auth.get_credential()) - if self.user is None: - status = 401 # Unauthorized -############################################################################################ - try: - if not status and 'PING' in self.request.arguments(): - status = self._ping(); - elif not status and 'SUBMIT' in self.request.arguments(): - status = self._submit(); - elif not status and 'QUERY' in self.request.arguments(): - status = self._query(); - elif not status and 'VIEW' in self.request.arguments(): - status = self._view(); - elif not status and 'ADMIN' in self.request.arguments(): - if self.user.is_superuser: - status = self._admin() - else: - status = 403 # Forbidden - elif not status and 'UPDATE' in self.request.arguments(): - if self.user.is_superuser: - status = self._update() - else: - status = 403 # Forbidden - elif not status: - status = 400 # Bad Request - - except (datastore_errors.Timeout, DeadlineExceededError), err: - logging.error("The timeout exception handling: error='%s'\n" % (err)) - respond = "Response time deadline has been reached\n" + str(err) - self.response.out.write(respond) - status = 500 - - if (status == 0): status = 200 - self.response.set_status(status) - - return status - - -def gql_ex(query_str): - """GQL extended implementation. - - """ - status = 0 - - # The LIKE operator is used in a WHERE clause to search for a part of string value in a column. - # Example: select * from T where F like '' - (query_str, status) = re.subn(r"\s+(?P(?P\w+)\s+[Ll][Ii][Kk][Ee]\s+[\'](?P[^']+)[\'])\s*", - " \g >= \'\g\' and \g < \'\g" +"\xEF\xBF\xBD".decode('utf-8') + "\' ", query_str) - - # The usage of SELECT clause inside IN operation - # Example: select * from T where F in (select * from T1) - if (status == 0): - match = re.search(r"\s+((\w+)\s+[Ii][Nn]\s*\(\s*\"(\s*[Ss][Ee][Ll][Ee][Cc][Tt]\s+[^\"]+)\"\s*\))\s*", query_str) - if (match is not None): - status = 1 - try: - query = db.GqlQuery(match.group(3)) - result_set = query - except (datastore_errors.BadQueryError, datastore_errors.BadArgumentError, datastore_errors.BadRequestError, datastore_errors.BadFilterError, db.KindError), err: - logging.error("Incorrect GQL line: GQL='%s' error='%s'\n" % - (match.group(3), err)) - return (query_str, 400) - - key_values = {} - key_values = get_table_data(result_set) - # We use only one entity because it is not supported by GQL F in (key()[,key()]) construction - # so we should do F=key() only for single entity - if key_values.has_key('_key_'): - replace_str = ' ' + match.group(2) + '=' - for value in key_values['_key_']: - replace_str += "KEY('" + str(value) + "') " - break - replace_str += '' - query_str = query_str.replace(match.group(1), replace_str) - else: - return (query_str, 501) # Not Implemented - - if status: - (query_str, status) = gql_ex(query_str) - else : - logging.debug("GQL query: \'%s\' status = %d\n" % (query_str, status)) - - return (query_str, status) - - -def get_table_data(entities): - """Returns the union of key names used by the given list of entities. - - We return the union as a dictionary mapping the key names to a sample - value from one of the entities for the key name. - """ - key_dict = {} - key_dict['_id_'] = [] - key_dict['_key_'] = [] - for entity in entities: - for prop in entity.properties().values(): - if not key_dict.has_key(prop): - key_dict[prop.name] = [] - for prop in entity.dynamic_properties(): - if not key_dict.has_key(prop): - key_dict[prop] = [] - - for entity in entities: - for key in key_dict.keys(): - if key in ['_id_']: - key_dict['_id_'].append(str(entity.key().id())) - elif key in ['_key_']: - key_dict['_key_'].append(str(entity.key())) - - elif key in entity.properties(): - prop = entity.properties()[key] - - val = prop.get_value_for_datastore(entity) - - if (datastore.typename(prop) in ['BlobProperty']): - val = 'blob' - elif (datastore.typename(prop) in ['ReferenceProperty']): - val = str(val) - elif (datastore.typename(prop) in ['EmailProperty', 'BooleanProperty']): - val = str(val) - elif (datastore.typename(prop) in ['TextProperty']): - val = unicode(val) - - if val is None or val == 'unknown': - val = '' - - if key_dict.has_key(key): - key_dict[key].append(str(val)) - else: - key_dict[key] = [str(val)] - - elif key in entity.dynamic_properties(): - val = entity.__getattr__(key) - - if (type(val).__name__ in ['Blob']): - val = 'blob' - elif (type(val).__name__ in ['Key']): - val = str(val) - elif (type(val).__name__ in ['Email', 'bool']): - val = str(val) - elif (type(val).__name__ in ['Text']): - val = unicode(val) - - if val is None or val == 'unknown': - val = '' - - if key_dict.has_key(key): - key_dict[key].append(str(val)) - else: - key_dict[key] = [str(val)] - - else: - key_dict[key].append('') - - return key_dict - - -application = webapp.WSGIApplication( - [('/', MainPage), -# ('/login/*', LoginHandler), - ('/get', DownloadHandler), - ('/client', ClientHandler) - ], debug=True) - -def main(): - if conf.DEBUG: - logging.getLogger().setLevel(logging.DEBUG) - run_wsgi_app(application) - -if __name__ == "__main__": - main() - diff --git a/server/gds/models.py b/server/gds/models.py deleted file mode 100755 index 0ac5a0f15..000000000 --- a/server/gds/models.py +++ /dev/null @@ -1,168 +0,0 @@ -#! /usr/bin/env python - -# -# Copyright (c) 2009 Voltaire -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# Python Standart modules - -# Python Google AppEngine SDK modules -from google.appengine.ext import db - -# Private modules - -class SubmitInfo(db.Model): - # mpiinstallphase_set(back-reference) - # testbuildphase_set(back-reference) - # testrunphase_set(back-reference) - hostname = db.StringProperty(default='unknown') - local_username = db.StringProperty(default='unknown') - http_username = db.StringProperty(default='unknown') - mtt_version = db.StringProperty(default='unknown') - -class SuiteInfo(db.Model): - # testbuildphase_set(back-reference) - # testrunphase_set(back-reference) - suite_name = db.StringProperty(default='unknown') - suite_version = db.StringProperty(default='unknown') - -class MpiInfo(db.Model): - # mpiinstallphase_set(back-reference) - # testbuildphase_set(back-reference) - # testrunphase_set(back-reference) - mpi_name = db.StringProperty(default='unknown') - mpi_version = db.StringProperty(default='unknown') - oma_version = db.StringProperty(default='unknown') - -class ClusterInfo(db.Model): - # mpiinstallphase_set(back-reference) - # testbuildphase_set(back-reference) - # testrunphase_set(back-reference) - cluster_name = db.StringProperty(default='unknown') - node_count = db.IntegerProperty(default=0) - node_hostname = db.StringProperty(default='unknown') - node_arch = db.StringProperty(default='unknown') - node_ncpu = db.IntegerProperty(default=0) - node_nsocket = db.IntegerProperty(default=0) - node_htt = db.BooleanProperty(default=False) - node_mem = db.IntegerProperty(default=0) - node_cache = db.IntegerProperty(default=0) - node_mhz = db.IntegerProperty(default=0) - node_os_kernel = db.StringProperty(default='unknown') - node_os_vendor = db.StringProperty(default='unknown') - node_os_release = db.StringProperty(default='unknown') - net_pci = db.TextProperty() - net_conf = db.TextProperty() - net_eth100 = db.BooleanProperty(default=False) - net_eth1000 = db.BooleanProperty(default=False) - net_eth10k = db.BooleanProperty(default=False) - net_iwarp = db.BooleanProperty(default=False) - net_ibddr = db.BooleanProperty(default=False) - net_ibqdr = db.BooleanProperty(default=False) - -class CompilerInfo(db.Model): - # mpiinstallphase_set(back-reference) - # testbuildphase_set(back-reference) - # testrunphase_set(back-reference) - compiler_name = db.StringProperty(default='unknown') - compiler_version = db.StringProperty(default='unknown') - -class MpiInstallPhase(db.Expando): - # testbuildphase_set(back-reference) - # testrunphase_set(back-reference) - submitinfo = db.ReferenceProperty(SubmitInfo, collection_name='mpiinstallphase_set') - clusterinfo = db.ReferenceProperty(ClusterInfo, collection_name='mpiinstallphase_set') - mpiinfo = db.ReferenceProperty(MpiInfo, collection_name='mpiinstallphase_set') - compilerinfo = db.ReferenceProperty(CompilerInfo, collection_name='mpiinstallphase_set') - start_time = db.DateTimeProperty(auto_now_add=True) - end_time = db.DateTimeProperty(auto_now_add=True) - duration = db.IntegerProperty() - status = db.IntegerProperty(default=0) - description = db.TextProperty() - stdout = db.TextProperty() - stderr = db.TextProperty() - configuration = db.TextProperty() - mpi_path = db.StringProperty(default='unknown') - -class TestBuildPhase(db.Expando): - # testrunphase_set(back-reference) - submitinfo = db.ReferenceProperty(SubmitInfo, collection_name='testbuildphase_set') - clusterinfo = db.ReferenceProperty(ClusterInfo, collection_name='testbuildphase_set') - mpiinfo = db.ReferenceProperty(MpiInfo, collection_name='testbuildphase_set') - compilerinfo = db.ReferenceProperty(CompilerInfo, collection_name='testbuildphase_set') - suiteinfo = db.ReferenceProperty(SuiteInfo, collection_name='testbuildphase_set') - mpiinstallphase = db.ReferenceProperty(MpiInstallPhase, collection_name='testbuildphase_set') - start_time = db.DateTimeProperty(auto_now_add=True) - end_time = db.DateTimeProperty(auto_now_add=True) - duration = db.IntegerProperty() - status = db.IntegerProperty(default=0) - description = db.TextProperty() - stdout = db.TextProperty() - stderr = db.TextProperty() - -class TestRunPhase(db.Expando): - # id_testbuild(back-reference) - submitinfo = db.ReferenceProperty(SubmitInfo, collection_name='runtestphase_set') - clusterinfo = db.ReferenceProperty(ClusterInfo, collection_name='runtestphase_set') - mpiinfo = db.ReferenceProperty(MpiInfo, collection_name='runtestphase_set') - compilerinfo = db.ReferenceProperty(CompilerInfo, collection_name='runtestphase_set') - suiteinfo = db.ReferenceProperty(SuiteInfo, collection_name='runtestphase_set') - mpiinstallphase = db.ReferenceProperty(MpiInstallPhase, collection_name='runtestphase_set') - testbuildphase = db.ReferenceProperty(TestBuildPhase, collection_name='runtestphase_set') - - start_time = db.DateTimeProperty(auto_now_add=True) - end_time = db.DateTimeProperty(auto_now_add=True) - duration = db.IntegerProperty() - status = db.IntegerProperty(default=0) - description = db.TextProperty() - stdout = db.TextProperty() - stderr = db.TextProperty() - test_name = db.StringProperty() - test_case = db.StringProperty() - cmdline = db.TextProperty() - mpi_nproc = db.IntegerProperty() - mpi_hlist = db.StringProperty() - mpi_rlist = db.StringProperty() - mpi_mca = db.StringProperty() - mpi_btl = db.StringListProperty() - net_note = db.StringProperty() - tag = db.StringListProperty() - - data_file = db.BlobProperty(default=None) - - cached_submitinfo_hostname = db.StringProperty() - cached_submitinfo_local_username = db.StringProperty() - cached_submitinfo_http_username = db.StringProperty() - cached_submitinfo_mtt_version = db.StringProperty() - cached_clusterinfo_cluster_name = db.StringProperty() - cached_clusterinfo_node_count = db.IntegerProperty() - cached_clusterinfo_node_hostname = db.StringProperty() - cached_clusterinfo_node_arch = db.StringProperty() - cached_clusterinfo_node_ncpu = db.IntegerProperty() - cached_clusterinfo_node_nsocket = db.IntegerProperty() - cached_clusterinfo_node_htt = db.BooleanProperty(default=False) - cached_clusterinfo_node_mem = db.IntegerProperty() - cached_clusterinfo_node_cache = db.IntegerProperty() - cached_clusterinfo_node_mhz = db.IntegerProperty() - cached_clusterinfo_node_os_kernel = db.StringProperty() - cached_clusterinfo_node_os_vendor = db.StringProperty() - cached_clusterinfo_node_os_release = db.StringProperty() - cached_clusterinfo_net_eth100 = db.BooleanProperty(default=False) - cached_clusterinfo_net_eth1000 = db.BooleanProperty(default=False) - cached_clusterinfo_net_eth10k = db.BooleanProperty(default=False) - cached_clusterinfo_net_iwarp = db.BooleanProperty(default=False) - cached_clusterinfo_net_ibddr = db.BooleanProperty(default=False) - cached_clusterinfo_net_ibqdr = db.BooleanProperty(default=False) - cached_mpiinfo_mpi_name = db.StringProperty() - cached_mpiinfo_mpi_version = db.StringProperty() - cached_mpiinfo_oma_version = db.StringProperty() - cached_compilerinfo_compiler_name = db.StringProperty() - cached_compilerinfo_compiler_version = db.StringProperty() - cached_suiteinfo_suite_name = db.StringProperty() - cached_suiteinfo_suite_version = db.StringProperty() - cached_mpiinstallphase_mpi_path = db.StringProperty() diff --git a/server/gds/pager/__init__.py b/server/gds/pager/__init__.py deleted file mode 100755 index fa7ceba4f..000000000 --- a/server/gds/pager/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python -"Universal pagination class for App Engine models" - -# Python Standart modules - -# Python Google AppEngine SDK modules - -# Private modules -from pager import PagerQuery -import logging diff --git a/server/gds/pager/pager.py b/server/gds/pager/pager.py deleted file mode 100755 index e3683e3f7..000000000 --- a/server/gds/pager/pager.py +++ /dev/null @@ -1,514 +0,0 @@ -# -*- coding: utf-8 -*- -""" - tipfy.model.pager - ~~~~~~~~~~~~~~~~~ - - Universal pagination class for App Engine models. Supports "efficient paging - in any query, on any schema, purely in user land, without requiring extra - properties or other schema changes". This means that no property is - required to be used as index, and that it can easily build efficient - pagination for any existing model or query. - - PagerQuery wraps and has the same interface as db.Query, except that fetch() - takes a bookmark parameter instead of an offset. The query returns results - and bookmarks for the next or previous results, if available. - - A bookmark is the first or last result entity that was returned to the app - for a given query, serialized and base64-encoded so that apps can pass it - around in URL parameters, cookies, etc. - - It is based on the concept posted in the App Engine group: - http://google-appengine.googlegroups.com/web/efficient_paging_using_key_instead_of_a_dedicated_unique_property.txt - - Quick example: - - # Get the encoded bookmark from request. - bookmark = request.GET['bookmark'] - - # Build a paginated query. - query = PagerQuery(ModelClass).filter('foo >', 'bar') \ - .filter('baz =', 'ding') \ - .order('foo') - - # Fetch results for the current page and bookmarks for previous and next - # pages. - prev, results, next = query.fetch(10, bookmark) - - # For a searchable query, use: - query = SearchablePagerQuery(ModelClass).filter('foo >', 'bar') \ - .filter('baz =', 'ding') \ - .order('foo') \ - .search('my_string') - - prev, results, next = query.fetch(10, bookmark) - - # If 'prev' and/or 'next' are returned, use them to create links to the - # previous and/or next pages. - http://localhost:8080/?bookmark={{ prev }} - http://localhost:8080/?bookmark={{ next }} - - :copyright: 2009 by tipfy.org. - :license: BSD, see LICENSE.txt for more details. -""" -import re -from base64 import b64encode, b64decode -from urllib import urlencode -from cgi import parse_qsl -from datetime import datetime - -from google.appengine.ext import db -from google.appengine.ext.search import SearchableQuery, SearchableMultiQuery -from google.appengine.api import datastore_errors - -# Regex borrowed from google.appengine.api.datastore. -OPERATORS = ['<', '<=', '>', '>=', '=', '=='] -INEQUALITY_OPERATORS = ['<', '<=', '>', '>='] -FILTER_REGEX = re.compile( - '^\s*([^\s]+)(\s+(%s)\s*)?$' % '|'.join(OPERATORS), - re.IGNORECASE | re.UNICODE) - -def match_filter(prop_operator): - """Returns the property and operator given a value passed to filter().""" - matches = FILTER_REGEX.match(prop_operator) - return (matches.group(1), matches.group(3)) - -def encode_bookmark(values): - """Encodes a dictionary into a string.""" - return b64encode(urlencode(values)) - -def decode_bookmark(bookmark): - """Decodes a string into a bookmark dictionary.""" - return dict(parse_qsl(b64decode(bookmark))) - -class PagerQuery(object): - """Wraps db.Query to build bookmarkable queries, and to resume queries from - bookmarks. - """ - query_class = db.Query - - def __init__(self, model_class, keys_only=False): - """Constructs a bookmarkable query over instances of the given Model. - - Args: - model_class: Model class to build query for. - keys_only: Whether the query should return full entities or only keys. - """ - # Initialize query values. - self._model_class = model_class - self._keys_only = keys_only - self._ancestor = None - self._filters = {} - self._inequality_prop = None - self._inequality_filters = {} - self._orderings = [] - self._order_directions = {} - # Properties that should be encoded into a bookmark: inequalities - # and orderings. - self._bookmark_properties = [] - # Keep track of first result to know when we are on first page. - self._first_result = None - - def filter(self, property_operator, value): - """Adds a filter to query. - - Args: - property_operator: string with the property and operator to filter by. - value: the filter value. - - Returns: - Self to support method chaining. - - Raises: - BadArgumentError if invalid property is provided or two different - inequality properties are set for the same query. - """ - prop, operator = match_filter(property_operator) - if operator in INEQUALITY_OPERATORS: - if self._inequality_prop and self._inequality_prop != prop: - raise datastore_errors.BadArgumentError('Queries must have ' - 'only one inequality operator.') - self._inequality_prop = prop - self._inequality_filters[operator] = value - - # Store the property name to be used in bookmarks. - if prop not in self._bookmark_properties: - self._bookmark_properties.append(prop) - elif operator in OPERATORS: - self._filters[prop] = value - else: - raise datastore_errors.BadArgumentError('Filter operator is not ' - 'valid, received %s.' % operator) - return self - - def order(self, prop): - """Sets order of query result. - - To use descending order, prepend '-' (minus) to the property - name, e.g., '-date' rather than 'date'. - - Args: - property: Property to sort on. - - Returns: - Self to support method chaining. - """ - direction = '' - if prop.startswith('-'): - prop = prop[1:] - direction = '-' - self._orderings.append(prop) - self._order_directions[prop] = direction - - # Store the property name to be used in bookmarks. - if prop not in self._bookmark_properties: - self._bookmark_properties.append(prop) - return self - - def ancestor(self, ancestor): - """Sets an ancestor for this query. - - This restricts the query to only return results that descend from - a given model instance. In other words, all of the results will - have the ancestor as their parent, or parent's parent, etc. The - ancestor itself is also a possible result! - - Args: - ancestor: Model or Key (that has already been saved) - - Returns: - Self to support method chaining. - """ - self._ancestor = ancestor - return self - - def fetch(self, limit, bookmark=None): - """Fetches the query results, returning bookmarks for next and previous - pages if available. If bookmark is provided, the query is resumed from - that bookmark. - - Args: - limit: Maximum number of results to return. - bookmark: Encoded values of the query to be resumed. - - Returns: - A tuple (prev, res, next), where 'prev' and 'next' are bookmarks - for the next and previous pages and 'res' is a list of db.Model - instances for the current page. - """ - # If the query has an inequality filter but no sort order: - # appends an ASC sort order on the inequality property. - if self._inequality_prop and not self._orderings: - self.order(self._inequality_prop) - - # If the query doesn't have a sort order on __key__: - # append an ASC sort order on __key__. - if '__key__' not in self._orderings: - self.order('__key__') - - reverse = False - if bookmark: - # Resume the query from this bookmark. - # For reversed queries, invert all orderings. - if bookmark.startswith('-'): - reverse = True - bookmark = bookmark[1:] - - directions = {'': '-', '-': ''} - for prop, direction in self._order_directions.iteritems(): - self._order_directions[prop] = directions[direction] - - res = self._fetch_from_bookmark(limit + 1, bookmark) - else: - # Fetch bookmarkable results. - query = self._get_query(filters=self._filters, - inequality_prop=self._inequality_prop, - inequality_filters=self._inequality_filters, - orderings=self._orderings, - order_directions=self._order_directions) - res = query.fetch(limit + 1) - - # Build the next and previous bookmarks. - prev = None - next = None - if res: - has_prev = False - has_next = False - - # Prepare results, removing excedent and reversing if needed. - if len(res) > limit: - res.pop() - has_next = True - - if reverse: - res.reverse() - - # Track the very first result to avoid building the 'previous' - # bookmark for the first page. - first_result_key = str(res[0].key().id_or_name()) - if not bookmark: - self._first_result = first_result_key - elif self._first_result and self._first_result != first_result_key: - # Only show previous link if this is not the first page. - has_prev = True - - # Build the 'next' bookmark using the last result. - if reverse or has_next: - next = encode_bookmark(self._get_bookmark_values(res[-1])) - - # Build the 'previous' bookmark using the first result. - if bookmark and has_prev: - prev = '-' + encode_bookmark(self._get_bookmark_values(res[0])) - - return (prev, res, next) - - def _fetch_from_bookmark(self, limit, bookmark): - """Fetches results resuming a query from a bookmark. This may require - additional queries depending on the number of sort orders. - - Args: - limit: Maximum number of results to return. - bookmark: Encoded values of the query to be resumed. - - Returns: - A list of db.Model instances. There may be fewer than 'limit' - results if there aren't enough results to satisfy the request. - """ - bookmark = self._decode_bookmark(bookmark) - if not bookmark: - return [] - - # Add a new filter to the query: "prop = [bookmark value for prop]" - q_filters = dict(self._filters) - for prop in self._orderings: - q_filters[prop] = bookmark[prop] - - # Build the derived queries starting from bookmark. - queries = [] - q_orderings = [] - q_order_directions = {} - self._orderings.reverse() - for prop in self._orderings: - # Add order on prop to the beginning of the list. - q_orderings.insert(0, prop) - q_order_directions[prop] = self._order_directions[prop] - - # Replace the = filter on prop by a inequality one. - q_filters.pop(prop) - if q_order_directions[prop] == '': - operator = '>' - else: - operator = '<' - q_inequality_filters = {operator: bookmark[prop]} - - # If there are more inequality filters on prop, add them. - if prop == self._inequality_prop: - for operator, val in self._inequality_filters.iteritems(): - if operator not in q_inequality_filters: - q_inequality_filters[operator] = val - - query = self._get_query(filters=q_filters, inequality_prop=prop, - inequality_filters=q_inequality_filters, orderings=q_orderings, - order_directions=q_order_directions) - queries.append(query) - - # Fetch the results until reach limit. - results = [] - for query in queries: - results.extend(query.fetch(limit)) - limit -= len(results) - if limit == 0: - break - - return results - - def _get_query(self, filters={}, inequality_prop=None, - inequality_filters={}, orderings=[], order_directions={}): - """Builds a db.Query. - - Returns: - A google.appengine.ext.db.Query instance. - """ - query = self.__class__.query_class(self._model_class, - keys_only=self._keys_only) - - if self._ancestor: - query.ancestor(self._ancestor) - - if inequality_prop and inequality_filters: - for operator, value in inequality_filters.iteritems(): - query.filter(inequality_prop + ' ' + operator, value) - - for prop, value in filters.iteritems(): - query.filter(prop + ' =', value) - - for prop in orderings: - query.order(order_directions[prop] + prop) - - return query - - def _get_bookmark_values(self, entity): - """Returns a dictionary to build a bookmark. The properties used are - the filter inequalities and the query sort orders, plus the entity key - and the key_name or id of the very first result of the first page. - - Returns: - A dictionary of property names/values to build a bookmark. - """ - values = {} - for prop in self._bookmark_properties: - if prop == '__key__': - values[prop] = entity.key() - else: - values[prop] = str(getattr(entity, prop)) - if self._first_result: - values['_'] = self._first_result - return values - - def _decode_bookmark(self, bookmark): - """Decodes a bookmark and prepares the values to be used on queries. - Currently supported properties are: - DateProperty - DateTimeProperty - FloatProperty - IntegerProperty - StringProperty - ReferenceProperty - TimeProperty - - Returns: - A dictionary of property names/values to be used in queries. - """ - required = list(self._bookmark_properties) - required.append('_') - try: - bookmark = decode_bookmark(bookmark) - # Ensure that all required values are available. - for key in required: - if key not in bookmark: - return None - except: - return None - - for key in required: - value = bookmark[key] - if key == '_': - self._first_result = bookmark[key] - elif key == '__key__': - bookmark[key] = db.Key(value) - else: - prop = getattr(self._model_class, key) - if isinstance(prop, db.ReferenceProperty): - bookmark[key] = db.Key(value) - elif isinstance(prop, db.DateProperty): - bookmark[key] = \ - datetime.strptime(str(value), '%Y-%m-%d').date() - elif isinstance(prop, db.TimeProperty): - bookmark[key] = parse_datetime(value, '%H:%M:%S').time() - elif isinstance(prop, db.DateTimeProperty): - bookmark[key] = parse_datetime(value, '%Y-%m-%d %H:%M:%S') - else: - bookmark[key] = prop.data_type(value) - - return bookmark - - -class SearchablePagerQuery(PagerQuery): - class Query(db.Query): - """A subclass of db.Query that supports full text search.""" - _search_query = None - - def search(self, search_query): - """Adds a full text search to this query. - - Args: - search_query, a string containing the full text search query. - - Returns: - self - """ - self._search_query = search_query - return self - - def _get_query(self): - """Wraps db.Query._get_query() and injects SearchableQuery.""" - query = db.Query._get_query(self, - _query_class=SearchableQuery, - _multi_query_class=SearchableMultiQuery) - if self._search_query: - query.Search(self._search_query) - return query - - query_class = Query - _search_query = None - - def search(self, search_query): - self._search_query = search_query - return self - - def _get_query(self, **kargs): - query = super(SearchablePagerQuery, self)._get_query(**kargs) - if self._search_query: - query.search(self._search_query) - return query - - -# borrowed from -# http://kbyanc.blogspot.com/2007/09/python-reconstructing-datetimes-from.html -def parse_datetime(s, format): - """Create datetime object representing date/time - expressed in a string. - - This is required because converting microseconds using strptime() is only - supported in Python 2.6. - - Takes a string in the format produced by calling str() - on a python datetime or time objects and returns a datetime - instance that would produce that string. - - Acceptable formats are: "YYYY-MM-DD HH:MM:SS.ssssss+HH:MM", - "YYYY-MM-DD HH:MM:SS.ssssss", - "YYYY-MM-DD HH:MM:SS+HH:MM", - "YYYY-MM-DD HH:MM:SS" - Where ssssss represents fractional seconds. The timezone - is optional and may be either positive or negative - hours/minutes east of UTC. - """ - if s is None: - return None - # Split string in the form 2007-06-18 19:39:25.3300-07:00 - # into its constituent date/time, microseconds, and - # timezone fields where microseconds and timezone are - # optional. - m = re.match(r'(.*?)(?:\.(\d+))?(([-+]\d{1,2}):(\d{2}))?$', - str(s)) - datestr, fractional, tzname, tzhour, tzmin = m.groups() - - # Create tzinfo object representing the timezone - # expressed in the input string. The names we give - # for the timezones are lame: they are just the offset - # from UTC (as it appeared in the input string). We - # handle UTC specially since it is a very common case - # and we know its name. - if tzname is None: - tz = None - else: - tzhour, tzmin = int(tzhour), int(tzmin) - if tzhour == tzmin == 0: - tzname = 'UTC' - tz = FixedOffset(timedelta(hours=tzhour, - minutes=tzmin), tzname) - - # Convert the date/time field into a python datetime - # object. - x = datetime.strptime(datestr, format) - - # Convert the fractional second portion into a count - # of microseconds. - if fractional is None: - fractional = '0' - fracpower = 6 - len(fractional) - fractional = float(fractional) * (10 ** fracpower) - - # Return updated datetime object with microseconds and - # timezone information. - return x.replace(microsecond=int(fractional), tzinfo=tz) diff --git a/server/gds/run_local_gds.sh b/server/gds/run_local_gds.sh deleted file mode 100755 index f462601c4..000000000 --- a/server/gds/run_local_gds.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/sh -# -# This script downloads, installs and loading GDS server on local host -# -# - -HOME_PATH=`dirname $0` # exec name cutting -mkdir -p $HOME_PATH/build - - -cd ${HOME_PATH} -HOME_PATH=`pwd`/build -cd ${HOME_PATH} - -PORT=8080 -if [ $# -eq 1 ]; then - PORT=$1 -fi - -LOCAL_PATH=../tests -APP_PATH=../ -APPENGINE_PATH=${HOME_PATH}/google_appengine - -if [ -e ${APPENGINE_PATH} ] -then - echo " GDS server already installed" -else - echo "Installing GDS server" -# download and install the newest version -# wget http://code.google.com/appengine/downloads.html -# link=`cat downloads.html |grep google_appengine |awk '{ print $2}' | cut -d= -f2 |cut -d'"' -f2 ` -# rm -f downloads.html -# wget $link -# unzip google_appengine* - -# currently get the working version 1.2.7 - wget http://googleappengine.googlecode.com/files/google_appengine_1.2.7.zip - unzip google_appengine_1.2.7.zip -fi - - -file=../main.py -line_num=`grep -rn " COMMENT FOLLOWING THREE LINES" $file | awk '{ print $1}' |sed 's/:#//g'` -if [ ${line_num} ] -then - echo "Backing up $file to ${file}.bak" - cp $file ${file}.bak - echo "modifying $file to run locally" - sed -i $line_num,+4d $file -fi - -logfile=$HOME_PATH/gds_`date +%s`.log -echo "starting GDS server, all output goes to $logfile" - -python25 "${APPENGINE_PATH}/dev_appserver.py" -d --address="`hostname`" --port=$PORT --datastore_path="${LOCAL_PATH}/db$PORT" --history_path="${LOCAL_PATH}/db$PORT.history" "${APP_PATH}" 2>&1|tee > $logfile & - diff --git a/server/gds/templates/admin.html b/server/gds/templates/admin.html deleted file mode 100644 index 0fc7eaa8b..000000000 --- a/server/gds/templates/admin.html +++ /dev/null @@ -1,95 +0,0 @@ - - - - Admin - - - - - -
- - - - - - - - - - - - - - - - -
-

Add a new user

-
- User name (Google account): - - - - - - - - - - -
- -
- - - Is user an MTT administrator - -
-
- -
-   - - -
-

- To modify users data (view/edit/delete) you should use Dashboard service from the main page, select "Data Viewer" and "User" kind. -

-
- - - diff --git a/server/gds/templates/images/attachment.gif b/server/gds/templates/images/attachment.gif deleted file mode 100644 index 4400e61e9..000000000 Binary files a/server/gds/templates/images/attachment.gif and /dev/null differ diff --git a/server/gds/templates/images/favicon.ico b/server/gds/templates/images/favicon.ico deleted file mode 100644 index 93269c40f..000000000 Binary files a/server/gds/templates/images/favicon.ico and /dev/null differ diff --git a/server/gds/templates/images/searchbtn.gif b/server/gds/templates/images/searchbtn.gif deleted file mode 100644 index 8468a965c..000000000 Binary files a/server/gds/templates/images/searchbtn.gif and /dev/null differ diff --git a/server/gds/templates/index.html b/server/gds/templates/index.html deleted file mode 100644 index f307a68bb..000000000 --- a/server/gds/templates/index.html +++ /dev/null @@ -1,154 +0,0 @@ - - - - HPC Vbench collection data - - - - - - - - - - - -
- - - -
- - -
-
- - - - - - - - - - - -
- -
Main
-
About
- Open MPI
- MPI Testing Tool
- Google App Engine
-
Services
- Admin
- Dashboard
- 		   - - - - - - - - - - - - - - - - - - - {% for cur_collection in query_collection %} - {% if forloop.counter0|divisibleby:2 %} - - {% else %} - - {% endif %} - - - - - - - - {% if cur_collection.data_file %} - - {% else %} - - {% endif %} - - {% endfor %} -

Collection

IdSuiteMPIClusterStart timeUserStatus
{{cur_collection.key.id}}{{cur_collection.suiteinfo.suite_name}}({{cur_collection.suiteinfo.suite_version}})  {{cur_collection.mpiinfo.mpi_name}}({{cur_collection.mpiinfo.mpi_version}}){{cur_collection.clusterinfo.cluster_name}}{{cur_collection.start_time}}{{cur_collection.submit.http_username}}{{cur_collection.status}}
-
- {% if prev %} - prev - {% endif %} - {% if next %} - next - {% endif %} - {% if query_count %} -

Total results {{query_count}}

- {% endif %} -
- -
- - - - - -
- - Powered by Google App Engine - -
- - - diff --git a/server/gds/templates/login.html b/server/gds/templates/login.html deleted file mode 100644 index 7919415a2..000000000 --- a/server/gds/templates/login.html +++ /dev/null @@ -1,74 +0,0 @@ - - - - Login - - - - - -
- - - - - - - - - - - - - -
- User name (Google account): - - -
- Password: - - -
-   - - - -
-
- - - diff --git a/server/gds/test.py b/server/gds/test.py deleted file mode 100755 index 48328f0db..000000000 --- a/server/gds/test.py +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env python - -# -# Copyright (c) 2009 Voltaire -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -""" -PyUnit Test runner suitable for use with Google App Engine. -Drop this file into your project directory, create a directory called -tests in the same directory and create a blank file in that called -__init__.py. You directory structure should be something like this: - - - root - - app.yaml - - main.py - - test.py - - tests - - __init__.py - -You should now be able to just drop valid PyUnit test cases into the tests -directory and they should be run when you run test.py via the command line. -""" - -import os -import unittest -import coverage -import argparse - -def run_tests(verbosity): - "Run test suite" - - # list all the files in the top level directory - file_list = os.listdir(os.path.join(os.path.abspath( - os.path.dirname(os.path.realpath(__file__))))) - - # list all the files in the tests directory - test_list = os.listdir(os.path.join(os.path.abspath( - os.path.dirname(os.path.realpath(__file__))), 'tests')) - - code_modules = [] - # loop over all the file names - for file_name in file_list: - extension = os.path.splitext(file_name)[-1] - # if they are python files or the test runner - if extension == '.py' and file_name != 'test.py': - # work out the module name - code_module_name = os.path.splitext(file_name)[0:-1][0] - # now import the module - module = __import__(code_module_name, globals(), locals(), - code_module_name) - # and add it to the list of available modules - code_modules.append(module) - - test_modules = [] - # loop over all the file names - for file_name in test_list: - extension = os.path.splitext(file_name)[-1] - # if they are python files - if extension == '.py': - # work out the module name - test_module_name = "tests." + os.path.splitext(file_name)[0:-1][0] - # now import the module - module = __import__(test_module_name, globals(), locals(), - test_module_name) - # and add it to the list of available modules - test_modules.append(module) - - # populate a test suite from the individual tests from the list of modules - suite = unittest.TestSuite(map( - unittest.defaultTestLoader.loadTestsFromModule, test_modules)) - - # set up the test runner - runner = unittest.TextTestRunner(verbosity=int(verbosity)) - - # set up coverage reporting - coverage.use_cache(0) - coverage.start() - - # run the tests - runner.run(suite) - - # stop coverage reporting - coverage.stop() - - # output coverage report - coverage.report(code_modules, show_missing=1) - -if __name__ == '__main__': - # instantiate the arguments parser - PARSER = argparse.ArgumentParser() - # add an option so we can set the test runner verbosity - PARSER.add_argument('--verbosity', - action='store', - dest='verbosity', - default='1', - type=int, - choices=['0', '1', '2'], - help="""Verbosity level; 0=minimal output, - 1=normal output, 2=all output""" - ), - # parse the command arguments - OPTIONS = PARSER.parse_args() - - # run the tests with the passed verbosity - run_tests(OPTIONS.verbosity) diff --git a/server/gds/tests/__init__.py b/server/gds/tests/__init__.py deleted file mode 100755 index 2972d8a33..000000000 --- a/server/gds/tests/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# -# Copyright (c) 2009 Voltaire -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -"Test suite for Application supported Google App Engine datastore for MTT." \ No newline at end of file diff --git a/server/php/cherrypy/bin/mtt_server_install.py b/server/php/cherrypy/bin/mtt_server_install.py index 4132d9a5b..0cbe97832 100755 --- a/server/php/cherrypy/bin/mtt_server_install.py +++ b/server/php/cherrypy/bin/mtt_server_install.py @@ -6,6 +6,7 @@ """ from __future__ import print_function +from builtins import input import os import shutil import sys @@ -37,7 +38,7 @@ print('Warning: Server appears to be already installed.') print("=" * 70) - response = raw_input('(R)emove and re-install or (A)bort? ') + response = input('(R)emove and re-install or (A)bort? ') if response != 'R' and response != 'r': sys.exit('Aborted') shutil.rmtree(env_dir) @@ -67,7 +68,7 @@ print('Warning: Server configuration file present.') print(divider) - response = raw_input('(R)eplace or (K)eep? ') + response = input('(R)eplace or (K)eep? ') if response == 'R' or response == 'r': call(['rm', final_conf]) call(['cp', tmpl_conf, final_conf]) @@ -81,7 +82,7 @@ print('Warning: Server .htaccess file present.') print(divider) - response = raw_input('(R)eplace or (K)eep? ') + response = input('(R)eplace or (K)eep? ') if response == 'R' or response == 'r': call(['rm', final_htaccess]) call(['cp', tmpl_htaccess, final_htaccess]) @@ -102,13 +103,12 @@ # # Setup virtual environment -# +# The python version installed will be the system default. +# Any python version > 2.7 will work. call(['virtualenv', '-p', 'python', env_dir]) call([env_dir + '/bin/pip', 'install', '-r', source_dir + '/mtt_server_requirements.txt']) # Link Server to virtual environment -# FIXME: This assumes python2.7. Need to find a way with virtualenv/pip to -# enforce this. cwd = os.getcwd() for dir,_,_ in os.walk(env_dir): diff --git a/server/php/cherrypy/bin/mtt_server_service.py b/server/php/cherrypy/bin/mtt_server_service.py index 9ead4e9d6..8faed6acf 100755 --- a/server/php/cherrypy/bin/mtt_server_service.py +++ b/server/php/cherrypy/bin/mtt_server_service.py @@ -65,7 +65,7 @@ def _getPID(): def _start(): """If not already running, start the REST server.""" - print 'Starting REST server...' + print('Starting REST server...') pid = _getPID() if -2 == pid: @@ -98,7 +98,7 @@ def _restart(): def _stop(): """If running, stop the REST server.""" - print(Stopping REST server...') + print('Stopping REST server...') pid = _getPID() if -2 == pid: diff --git a/server/php/cherrypy/src/webapp/db_pgv3.py b/server/php/cherrypy/src/webapp/db_pgv3.py index 99dbf65bb..d0218f486 100644 --- a/server/php/cherrypy/src/webapp/db_pgv3.py +++ b/server/php/cherrypy/src/webapp/db_pgv3.py @@ -476,7 +476,7 @@ def _info_runtime(self, phase, search): ########################################################## def _find_value(self, metadata, data, field, aliases=None): - for (key, value) in metadata.iteritems(): + for (key, value) in metadata.items(): if key == field: return value if aliases is not None: @@ -484,7 +484,7 @@ def _find_value(self, metadata, data, field, aliases=None): if key == alias: return value - for (key, value) in data.iteritems(): + for (key, value) in data.items(): if key == field: return value if aliases is not None: diff --git a/server/sql/support/create-partitions-test-run.pl b/server/sql/support/create-partitions-test-run.pl index 4be76560c..03cd6010c 100755 --- a/server/sql/support/create-partitions-test-run.pl +++ b/server/sql/support/create-partitions-test-run.pl @@ -78,10 +78,10 @@ print " FOREIGN KEY (test_build_compiler_id) REFERENCES compiler(compiler_id),\n"; print " -- PARTITION/FK PROBLEM: FOREIGN KEY (test_build_id) REFERENCES test_build(test_build_id),\n"; print " FOREIGN KEY (test_name_id) REFERENCES test_names(test_name_id),\n"; - print " FOREIGN KEY (bios_id) REFERENCES bios(bios_id),\n"; - print " FOREIGN KEY (firmware_id) REFERENCES firmware(firmware_id),\n"; - print " FOREIGN KEY (provision_id) REFERENCES provision(provision_id),\n"; - print " FOREIGN KEY (harasser_id) REFERENCES harasser(harasser_id),\n"; +# print " FOREIGN KEY (bios_id) REFERENCES bios(bios_id),\n"; +# print " FOREIGN KEY (firmware_id) REFERENCES firmware(firmware_id),\n"; +# print " FOREIGN KEY (provision_id) REFERENCES provision(provision_id),\n"; +# print " FOREIGN KEY (harasser_id) REFERENCES harasser(harasser_id),\n"; print " FOREIGN KEY (performance_id) REFERENCES performance(performance_id),\n"; print " FOREIGN KEY (test_run_command_id) REFERENCES test_run_command(test_run_command_id),\n"; print " FOREIGN KEY (description_id) REFERENCES description(description_id),\n"; @@ -137,10 +137,10 @@ print " test_build_compiler_id,\n"; print " test_build_id,\n"; print " test_name_id,\n"; - print " bios_id,\n"; - print " firmware_id,\n"; - print " provision_id,\n"; - print " harasser_id,\n"; +# print " bios_id,\n"; +# print " firmware_id,\n"; +# print " provision_id,\n"; +# print " harasser_id,\n"; print " performance_id,\n"; print " test_run_command_id,\n"; print " np,\n"; @@ -172,10 +172,10 @@ print " NEW.test_build_compiler_id,\n"; print " NEW.test_build_id,\n"; print " NEW.test_name_id,\n"; - print " NEW.bios_id,\n"; - print " NEW.firmware_id,\n"; - print " NEW.provision_id,\n"; - print " NEW.harasser_id,\n"; +# print " NEW.bios_id,\n"; +# print " NEW.firmware_id,\n"; +# print " NEW.provision_id,\n"; +# print " NEW.harasser_id,\n"; print " NEW.performance_id,\n"; print " NEW.test_run_command_id,\n"; print " NEW.np,\n"; diff --git a/server/viewer/main/webapp/db_pgv3.py b/server/viewer/main/webapp/db_pgv3.py index cea01576f..8ec87e93c 100644 --- a/server/viewer/main/webapp/db_pgv3.py +++ b/server/viewer/main/webapp/db_pgv3.py @@ -193,7 +193,7 @@ def _query(self, table_name, filters=[], row_id=None, fields=None): ########################################################## def _find_value(self, metadata, data, field, aliases=None): - for (key, value) in metadata.iteritems(): + for (key, value) in metadata.items(): if key == field: return value if aliases is not None: @@ -201,7 +201,7 @@ def _find_value(self, metadata, data, field, aliases=None): if key == alias: return value - for (key, value) in data.iteritems(): + for (key, value) in data.items(): if key == field: return value if aliases is not None: