From 33ca8c026afae46c3eb9d1212d03ddd5d531b2f1 Mon Sep 17 00:00:00 2001 From: Nadav Goldin Date: Mon, 24 Apr 2017 12:11:45 +0300 Subject: [PATCH] ovirtlago: reconfigure timeouts in 'lago ovirt start' 1. Increased default timeout for waiting until the VMs connected to the Engine report 'up' status to 8 minutes. 2. Added '--vms-timeout' parameter to allow configuring that timeout. 3. Increased 'ovirt-engine'/'vdsmd' service timeout to 3 minutes. 4. Added default timeouts in all relevant functions headers, and set the timeouts explicitly in 'do_ovirt_start' for visibility. Signed-off-by: Nadav Goldin --- ovirtlago/cmd.py | 32 ++++++++++++++++++++++++-------- ovirtlago/virt.py | 20 ++++++++++---------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/ovirtlago/cmd.py b/ovirtlago/cmd.py index 6cc5f741..f1a2bb36 100755 --- a/ovirtlago/cmd.py +++ b/ovirtlago/cmd.py @@ -198,10 +198,18 @@ def do_ovirt_stop_vms(prefix, **kwargs): @cli_plugin(help='Start all VMs that are down') +@cli_plugin_add_argument( + '--vms-timeout', + help=('Time to wait until the Engine reports the VMs are up.'), + dest='vms_timeout', + default=8 * 60, + type=int, + action='store', +) @in_ovirt_prefix @with_logging -def do_ovirt_start_vms(prefix, **kwargs): - prefix.virt_env.engine_vm().start_all_vms() +def do_ovirt_start_vms(prefix, vms_timeout, **kwargs): + prefix.virt_env.engine_vm().start_all_vms(timeout=vms_timeout) @cli_plugin(help='Print oVirt setup status') @@ -223,22 +231,30 @@ def do_ovirt_status(prefix, **kwargs): dest='with_vms', action='store_true', ) +@cli_plugin_add_argument( + '--vms-timeout', + help=('Time to wait until the Engine reports the VMs are up.'), + dest='vms_timeout', + default=8 * 60, + type=int, + action='store', +) @in_ovirt_prefix @with_logging -def do_ovirt_start(prefix, with_vms, **kwargs): +def do_ovirt_start(prefix, with_vms, vms_timeout, **kwargs): with LogTask('Starting oVirt environment'): prefix.start() with LogTask('Waiting for ovirt-engine status'): - prefix.virt_env.assert_engine_alive() + prefix.virt_env.assert_engine_alive(timeout=3 * 60) with LogTask('Waiting for vdsmd status'): - prefix.virt_env.assert_vdsm_alive() + prefix.virt_env.assert_vdsm_alive(timeout=3 * 60) with LogTask('Activating Engine Hosts'): - prefix.virt_env.engine_vm().start_all_hosts() + prefix.virt_env.engine_vm().start_all_hosts(timeout=5 * 60) if with_vms: with LogTask('Waiting for Storage domains to be in active mode'): - prefix.virt_env.engine_vm().check_sds_status() + prefix.virt_env.engine_vm().check_sds_status(timeout=5 * 60) with LogTask('Starting Engine VMs'): - prefix.virt_env.engine_vm().start_all_vms() + prefix.virt_env.engine_vm().start_all_vms(timeout=vms_timeout) @cli_plugin( diff --git a/ovirtlago/virt.py b/ovirtlago/virt.py index 7708c93b..65005ad1 100644 --- a/ovirtlago/virt.py +++ b/ovirtlago/virt.py @@ -340,7 +340,7 @@ def _search_vms(self, api, query): return [vm.id for vm in vms_service.list(search=query)] @require_sdk(version='4') - def start_all_vms(self): + def start_all_vms(self, timeout=8 * 60): api = self.get_api_v4(check=True) vms_service = api.system_service().vms_service() ids = self._search_vms(api, query='status=down') @@ -355,11 +355,11 @@ def _vm_is_up(id): for id in ids: testlib.assert_true_within( - partial(_vm_is_up, id=id), timeout=5 * 60 + partial(_vm_is_up, id=id), timeout=timeout ) @require_sdk(version='4') - def stop_all_vms(self): + def stop_all_vms(self, timeout=5 * 60): api = self.get_api_v4(check=True) vms_service = api.system_service().vms_service() ids = self._search_vms(api, query='status=up') @@ -374,11 +374,11 @@ def _vm_is_down(id): for id in ids: testlib.assert_true_within( - partial(_vm_is_down, id=id), timeout=5 * 60 + partial(_vm_is_down, id=id), timeout=timeout ) @require_sdk(version='4') - def stop_all_hosts(self): + def stop_all_hosts(self, timeout=5 * 60): api = self.get_api_v4(check=True) hosts_service = api.system_service().hosts_service() hosts = hosts_service.list(search='status=up') @@ -406,10 +406,10 @@ def _host_is_maint(): ) for h in hosts: - testlib.assert_true_within(_host_is_maint, timeout=5 * 60) + testlib.assert_true_within(_host_is_maint, timeout=timeout) @require_sdk(version='4') - def start_all_hosts(self): + def start_all_hosts(self, timeout=5 * 60): api = self.get_api_v4(check=True) hosts_service = api.system_service().hosts_service() hosts = hosts_service.list(search='status=maintenance') @@ -436,11 +436,11 @@ def _host_is_up(host): for host in hosts: testlib.assert_true_within( - partial(_host_is_up, host), timeout=5 * 60 + partial(_host_is_up, host), timeout=timeout ) @require_sdk(version='4') - def check_sds_status(self, status=None): + def check_sds_status(self, status=None, timeout=5 * 60): # the default status cannot be used in the function header, because # the v4 sdk might not be available. if status is None: @@ -455,7 +455,7 @@ def _sds_state(dc_id): return all(sd.status == status for sd in sds.list()) testlib.assert_true_within( - partial(_sds_state, dc_id=dc.id), timeout=5 * 60 + partial(_sds_state, dc_id=dc.id), timeout=timeout ) @require_sdk(version='4')