diff --git a/grl/agent/analytical.py b/grl/agent/analytical.py index f9b2b34d..4c0aab74 100644 --- a/grl/agent/analytical.py +++ b/grl/agent/analytical.py @@ -8,9 +8,11 @@ import optax from grl.mdp import POMDP -from grl.utils.loss import policy_discrep_loss, pg_objective_func -from grl.utils.loss import mem_discrep_loss, mem_magnitude_td_loss, obs_space_mem_discrep_loss -from grl.utils.math import glorot_init +from grl.utils.augment_policy import construct_aug_policy +from grl.utils.loss import policy_discrep_loss, pg_objective_func, \ + mem_pg_objective_func, unrolled_mem_pg_objective_func +from grl.utils.loss import mem_discrep_loss, mem_bellman_loss, mem_tde_loss, obs_space_mem_discrep_loss +from grl.utils.math import glorot_init, reverse_softmax from grl.utils.optimizer import get_optimizer from grl.vi import policy_iteration_step @@ -29,6 +31,7 @@ def __init__(self, value_type: str = 'v', error_type: str = 'l2', objective: str = 'discrep', + residual: bool = False, lambda_0: float = 0., lambda_1: float = 1., alpha: float = 1., @@ -43,7 +46,7 @@ def __init__(self, :param mem_params: Memory parameters (optional) :param value_type: If we optimize lambda discrepancy, what type of lambda discrepancy do we optimize? (v | q) :param error_type: lambda discrepancy error type (l2 | abs) - :param objective: What objective are we trying to minimize? (discrep | magnitude) + :param objective: What objective are we trying to minimize? (discrep | bellman | tde) :param pi_softmax_temp: When we take the softmax over pi_params, what is the softmax temperature? :param policy_optim_alg: What type of policy optimization do we do? (pi | pg) (discrep_max: discrepancy maximization | discrep_min: discrepancy minimization @@ -58,6 +61,10 @@ def __init__(self, self.og_n_obs = self.pi_params.shape[0] self.pg_objective_func = jit(pg_objective_func) + if self.policy_optim_alg == 'policy_mem_grad': + self.pg_objective_func = jit(mem_pg_objective_func) + elif self.policy_optim_alg == 'policy_mem_grad_unrolled': + self.pg_objective_func = jit(unrolled_mem_pg_objective_func) self.policy_iteration_update = jit(policy_iteration_step, static_argnames=['eps']) self.epsilon = epsilon @@ -65,6 +72,7 @@ def __init__(self, self.val_type = value_type self.error_type = error_type self.objective = objective + self.residual = residual self.lambda_0 = lambda_0 self.lambda_1 = lambda_1 self.alpha = alpha @@ -77,19 +85,29 @@ def __init__(self, self.new_mem_pi = new_mem_pi - self.optim_str = optim_str - # initialize optimizers - self.pi_lr = pi_lr - self.pi_optim = get_optimizer(optim_str, self.pi_lr) - self.pi_optim_state = self.pi_optim.init(self.pi_params) - self.mem_params = None if mem_params is not None: self.mem_params = mem_params + + if self.policy_optim_alg in ['policy_mem_grad', 'policy_mem_grad_unrolled']: + mem_probs, pi_probs = softmax(self.mem_params, -1), softmax(self.pi_params, -1) + aug_policy = construct_aug_policy(mem_probs, pi_probs) + self.pi_aug_params = reverse_softmax(aug_policy) + self.mi_lr = mi_lr self.mem_optim = get_optimizer(optim_str, self.mi_lr) self.mem_optim_state = self.mem_optim.init(self.mem_params) + # initialize optimizers + self.optim_str = optim_str + self.pi_lr = pi_lr + self.pi_optim = get_optimizer(optim_str, self.pi_lr) + + pi_params_to_optimize = self.pi_params + if self.policy_optim_alg in ['policy_mem_grad', 'policy_mem_grad_unrolled']: + pi_params_to_optimize = self.pi_aug_params + self.pi_optim_state = self.pi_optim.init(pi_params_to_optimize) + self.pi_softmax_temp = pi_softmax_temp self.rand_key = rand_key @@ -113,19 +131,25 @@ def init_and_jit_objectives(self): self.policy_discrep_objective_func = jit(partial_policy_discrep_loss) mem_loss_fn = mem_discrep_loss + partial_kwargs = { + 'value_type': self.val_type, + 'error_type': self.error_type, + 'lambda_0': self.lambda_0, + 'lambda_1': self.lambda_1, + 'alpha': self.alpha, + 'flip_count_prob': self.flip_count_prob + } if hasattr(self, 'objective'): - if self.objective == 'magnitude': - mem_loss_fn = mem_magnitude_td_loss + if self.objective == 'bellman': + mem_loss_fn = mem_bellman_loss + partial_kwargs['residual'] = self.residual + elif self.objective == 'tde': + mem_loss_fn = mem_tde_loss + partial_kwargs['residual'] = self.residual elif self.objective == 'obs_space': mem_loss_fn = obs_space_mem_discrep_loss - partial_mem_discrep_loss = partial(mem_loss_fn, - value_type=self.val_type, - error_type=self.error_type, - lambda_0=self.lambda_0, - lambda_1=self.lambda_1, - alpha=self.alpha, - flip_count_prob=self.flip_count_prob) + partial_mem_discrep_loss = partial(mem_loss_fn, **partial_kwargs) self.memory_objective_func = jit(partial_mem_discrep_loss) @property @@ -143,6 +167,7 @@ def reset_pi_params(self, pi_shape: Sequence[int] = None): if pi_shape is None: pi_shape = self.pi_params.shape self.pi_params = glorot_init(pi_shape) + self.pi_optim_state = self.pi_optim.init(self.pi_params) def new_pi_over_mem(self): if self.pi_params.shape[0] != self.og_n_obs: @@ -169,7 +194,7 @@ def policy_gradient_update(self, params: jnp.ndarray, optim_state: jnp.ndarray, params_grad = -params_grad updates, optimizer_state = self.pi_optim.update(params_grad, optim_state, params) params = optax.apply_updates(params, updates) - return v_0, td_v_vals, td_q_vals, params + return v_0, td_v_vals, td_q_vals, params, optimizer_state @partial(jit, static_argnames=['self', 'sign']) def policy_discrep_update(self, @@ -187,12 +212,15 @@ def policy_discrep_update(self, updates, optimizer_state = self.pi_optim.update(params_grad, optim_state, params) params = optax.apply_updates(params, updates) - return loss, mc_vals, td_vals, params + return loss, mc_vals, td_vals, params, optimizer_state def policy_improvement(self, pomdp: POMDP): - if self.policy_optim_alg == 'policy_grad': - v_0, prev_td_v_vals, prev_td_q_vals, new_pi_params = \ - self.policy_gradient_update(self.pi_params, self.pi_optim_state, pomdp) + if self.policy_optim_alg in ['policy_grad', 'policy_mem_grad', 'policy_mem_grad_unrolled']: + policy_params = self.pi_params + if self.policy_optim_alg in ['policy_mem_grad', 'policy_mem_grad_unrolled']: + policy_params = self.pi_aug_params + v_0, prev_td_v_vals, prev_td_q_vals, new_pi_params, new_optim_state= \ + self.policy_gradient_update(policy_params, self.pi_optim_state, pomdp) output = { 'v_0': v_0, 'prev_td_q_vals': prev_td_q_vals, @@ -201,9 +229,10 @@ def policy_improvement(self, pomdp: POMDP): elif self.policy_optim_alg == 'policy_iter': new_pi_params, prev_td_v_vals, prev_td_q_vals = self.policy_iteration_update( self.pi_params, pomdp, eps=self.epsilon) + new_optim_state = self.pi_optim_state output = {'prev_td_q_vals': prev_td_q_vals, 'prev_td_v_vals': prev_td_v_vals} elif self.policy_optim_alg == 'discrep_max' or self.policy_optim_alg == 'discrep_min': - loss, mc_vals, td_vals, new_pi_params = self.policy_discrep_update( + loss, mc_vals, td_vals, new_pi_params, new_optim_state = self.policy_discrep_update( self.pi_params, self.pi_optim_state, pomdp, @@ -211,7 +240,12 @@ def policy_improvement(self, pomdp: POMDP): output = {'loss': loss, 'mc_vals': mc_vals, 'td_vals': td_vals} else: raise NotImplementedError - self.pi_params = new_pi_params + + if self.policy_optim_alg in ['policy_mem_grad', 'policy_mem_grad_unrolled']: + self.pi_aug_params = new_pi_params + else: + self.pi_params = new_pi_params + self.pi_optim_state = new_optim_state return output @partial(jit, static_argnames=['self']) @@ -224,13 +258,14 @@ def memory_update(self, params: jnp.ndarray, optim_state: jnp.ndarray, pi_params updates, optimizer_state = self.mem_optim.update(params_grad, optim_state, params) params = optax.apply_updates(params, updates) - return loss, params + return loss, params, optimizer_state def memory_improvement(self, pomdp: POMDP): assert self.mem_params is not None, 'I have no memory params' - loss, new_mem_params = self.memory_update(self.mem_params, self.mem_optim_state, + loss, new_mem_params, new_mem_optim_state = self.memory_update(self.mem_params, self.mem_optim_state, self.pi_params, pomdp) self.mem_params = new_mem_params + self.mem_optim_state = new_mem_optim_state return loss def __getstate__(self) -> dict: @@ -254,6 +289,10 @@ def __setstate__(self, state: dict): # restore jitted functions self.pg_objective_func = jit(pg_objective_func) + if self.policy_optim_alg == 'policy_mem_grad': + self.pg_objective_func = jit(mem_pg_objective_func) + elif self.policy_optim_alg == 'policy_mem_grad_unrolled': + self.pg_objective_func = jit(unrolled_mem_pg_objective_func) self.policy_iteration_update = jit(policy_iteration_step, static_argnames=['eps']) if 'optim_str' not in state: diff --git a/grl/environment/__init__.py b/grl/environment/__init__.py index 8b1ab7b8..e5559b4a 100644 --- a/grl/environment/__init__.py +++ b/grl/environment/__init__.py @@ -1,28 +1,77 @@ from argparse import Namespace -import jax +import gymnasium as gym import numpy as np +from numpy import random +import popgym +from popgym.wrappers import Flatten from .rocksample import RockSample from .spec import load_spec, load_pomdp -from .wrappers import OneHotObservationWrapper, OneHotActionConcatWrapper +from .wrappers import OneHotObservationWrapper, OneHotActionConcatWrapper, \ + FlattenMultiDiscreteActionWrapper, DiscreteObservationWrapper, \ + ContinuousToDiscrete, ArrayObservationWrapper + +def get_popgym_env(args: Namespace, rand_key: random.RandomState = None, **kwargs): + # check to see if name exists + env_names = set([e["id"] for e in popgym.envs.ALL.values()]) + if args.spec not in env_names: + raise AttributeError(f"spec {args.spec} not found") + # wrappers fail unless disable_env_checker=True + env = gym.make(args.spec, disable_env_checker=True) + env.reset(seed=args.seed) + env.rand_key = rand_key + env.gamma = args.gamma + + return env def get_env(args: Namespace, rand_state: np.random.RandomState = None, - rand_key: jax.random.PRNGKey = None, + action_bins: int = 6, **kwargs): + """ + :param action_bins: If we have a continous action space, how many bins do we discretize to? + """ # First we check our POMDP specs try: env, _ = load_pomdp(args.spec, rand_key=rand_state, **kwargs) - - # TODO: some features are already encoded in a one-hot manner. - if args.feature_encoding == 'one_hot': - env = OneHotObservationWrapper(env) except AttributeError: - if args.spec == 'rocksample': - env = RockSample(rand_key=rand_key, **kwargs) - else: - raise NotImplementedError + # try to load from popgym + # validate input: we need a custom gamma for popgym args as they don't come with a gamma + if args.gamma is None: + raise AttributeError("Can't load non-native environments without passing in gamma!") + try: + env, _ = load_pomdp(args.spec, rand_key=rand_state, **kwargs) + + except AttributeError: + # try to load from popgym + # validate input: we need a custom gamma for popgym args as they don't come with a gamma + if args.gamma is None: + raise AttributeError( + "Can't load non-native environments without passing in gamma!") + try: + env = get_popgym_env(args, rand_key=rand_state, **kwargs) + + env = Flatten(env) + # also might need to preprocess our observation spaces + if isinstance(env.observation_space, gym.spaces.Discrete)\ + and args.feature_encoding != 'one_hot': + env = DiscreteObservationWrapper(env) + if isinstance(env.observation_space, gym.spaces.Tuple): + env = ArrayObservationWrapper(env) + + # preprocess continous action spaces + if isinstance(env.action_space, gym.spaces.Box): + env = ContinuousToDiscrete(env, action_bins) + elif isinstance(env.action_space, gym.spaces.MultiDiscrete): + env = FlattenMultiDiscreteActionWrapper(env) + + except AttributeError: + # don't have anything else implemented + raise NotImplementedError + + if args.feature_encoding == 'one_hot': + env = OneHotObservationWrapper(env) if args.action_cond == 'cat': env = OneHotActionConcatWrapper(env) diff --git a/grl/environment/pomdp_files/4x3.95-pomdp-solver-results.npy b/grl/environment/pomdp_files/4x3.95-pomdp-solver-results.npy index 361a0b21..4e298f90 100644 Binary files a/grl/environment/pomdp_files/4x3.95-pomdp-solver-results.npy and b/grl/environment/pomdp_files/4x3.95-pomdp-solver-results.npy differ diff --git a/grl/environment/pomdp_files/cheese.95-pomdp-solver-results.npy b/grl/environment/pomdp_files/cheese.95-pomdp-solver-results.npy index 8d625c1b..8a8cfc4c 100644 Binary files a/grl/environment/pomdp_files/cheese.95-pomdp-solver-results.npy and b/grl/environment/pomdp_files/cheese.95-pomdp-solver-results.npy differ diff --git a/grl/environment/pomdp_files/example_7-pomdp-solver-results.npy b/grl/environment/pomdp_files/example_7-pomdp-solver-results.npy index 67ee9c47..d88c0aef 100644 Binary files a/grl/environment/pomdp_files/example_7-pomdp-solver-results.npy and b/grl/environment/pomdp_files/example_7-pomdp-solver-results.npy differ diff --git a/grl/environment/pomdp_files/network-pomdp-solver-results.npy b/grl/environment/pomdp_files/network-pomdp-solver-results.npy index 61cbe1db..cb44d45e 100644 Binary files a/grl/environment/pomdp_files/network-pomdp-solver-results.npy and b/grl/environment/pomdp_files/network-pomdp-solver-results.npy differ diff --git a/grl/environment/pomdp_files/paint.95-action-cross-16583.alpha b/grl/environment/pomdp_files/paint.95-action-cross-16583.alpha new file mode 100644 index 00000000..bcb639e0 --- /dev/null +++ b/grl/environment/pomdp_files/paint.95-action-cross-16583.alpha @@ -0,0 +1,36 @@ +1 +1.9094990164038767588294832 1.9094990164038767588294832 1.9094990164038767588294832 1.9094990164038767588294832 3.1316167519690454312808470 3.1316167519690454312808470 3.1316167519690454312808470 3.1316167519690454312808470 2.1953745101033312892013782 2.1953745101033312892013782 2.1953745101033312892013782 2.1953745101033312892013782 3.6856125050351686844862797 3.6856125050351686844862797 3.6856125050351686844862797 3.6856125050351686844862797 2.7975557607195224996132765 2.7975557607195224996132765 2.7975557607195224996132765 2.7975557607195224996132765 + +1 +1.9703595428384403920318846 1.9703595428384403920318846 1.9703595428384403920318846 1.9703595428384403920318846 3.6856125050351686844862797 3.6856125050351686844862797 3.6856125050351686844862797 3.6856125050351686844862797 2.0382549725920608985063609 2.0382549725920608985063609 2.0382549725920608985063609 2.0382549725920608985063609 3.1316167519690454312808470 3.1316167519690454312808470 3.1316167519690454312808470 3.1316167519690454312808470 2.5509881474037428006340633 2.5509881474037428006340633 2.5509881474037428006340633 2.5509881474037428006340633 + +1 +2.0224713685980355570848133 2.0224713685980355570848133 2.0224713685980355570848133 2.0224713685980355570848133 3.4474713685980362676275490 3.4474713685980362676275490 3.4474713685980362676275490 3.4474713685980362676275490 2.2599713685980358235383392 2.2599713685980358235383392 2.2599713685980358235383392 2.2599713685980358235383392 3.4474713685980362676275490 3.4474713685980362676275490 3.4474713685980362676275490 3.4474713685980362676275490 2.7349713685980359123561811 2.7349713685980359123561811 2.7349713685980359123561811 2.7349713685980359123561811 + +3 +2.1289172301284891020145551 2.1289172301284891020145551 2.1289172301284891020145551 2.1289172301284891020145551 2.1289172301284891020145551 2.1289172301284891020145551 2.1289172301284891020145551 2.1289172301284891020145551 3.1289172301284895461037650 3.1289172301284895461037650 3.1289172301284895461037650 3.1289172301284895461037650 4.1289172301284899901929748 4.1289172301284899901929748 4.1289172301284899901929748 4.1289172301284899901929748 3.1289172301284895461037650 3.1289172301284895461037650 3.1289172301284895461037650 3.1289172301284895461037650 + +2 +2.1289172301284891020145551 2.1289172301284891020145551 2.1289172301284891020145551 2.1289172301284891020145551 4.1289172301284899901929748 4.1289172301284899901929748 4.1289172301284899901929748 4.1289172301284899901929748 2.1289172301284891020145551 2.1289172301284891020145551 2.1289172301284891020145551 2.1289172301284891020145551 2.1289172301284891020145551 2.1289172301284891020145551 2.1289172301284891020145551 2.1289172301284891020145551 3.1289172301284895461037650 3.1289172301284895461037650 3.1289172301284895461037650 3.1289172301284895461037650 + +1 +2.8893649926851554532447608 2.8893649926851554532447608 2.8893649926851554532447608 2.8893649926851554532447608 3.0054661775638464860094246 3.0054661775638464860094246 3.0054661775638464860094246 3.0054661775638464860094246 2.1303298224763906887346820 2.1303298224763906887346820 2.1303298224763906887346820 2.1303298224763906887346820 3.6783853175210641239800680 3.6783853175210641239800680 3.6783853175210641239800680 3.6783853175210641239800680 3.2838751551031100106570193 3.2838751551031100106570193 3.2838751551031100106570193 3.2838751551031100106570193 + +1 +3.1650036922575885967034992 3.1650036922575885967034992 3.1650036922575885967034992 3.1650036922575885967034992 3.3003786922575883977515332 3.3003786922575883977515332 3.3003786922575883977515332 3.3003786922575883977515332 2.1841286922575884332786700 2.1841286922575884332786700 2.1841286922575884332786700 2.1841286922575884332786700 3.4221904764845541002671325 3.4221904764845541002671325 3.4221904764845541002671325 3.4221904764845541002671325 3.2935970843710711264407109 3.2935970843710711264407109 3.2935970843710711264407109 3.2935970843710711264407109 + +1 +3.3456100358647971049208536 3.3456100358647971049208536 3.3456100358647971049208536 3.3456100358647971049208536 3.5085590672734863204595968 3.5085590672734863204595968 3.5085590672734863204595968 3.5085590672734863204595968 1.9469641829402153376804563 1.9469641829402153376804563 1.9469641829402153376804563 1.9469641829402153376804563 3.1011864887517637257019487 3.1011864887517637257019487 3.1011864887517637257019487 3.1011864887517637257019487 3.2233982623082804153114012 3.2233982623082804153114012 3.2233982623082804153114012 3.2233982623082804153114012 + +1 +3.4110742270132501374746425 3.4110742270132501374746425 3.4110742270132501374746425 3.4110742270132501374746425 3.5786007895132496692269797 3.5786007895132496692269797 3.5786007895132496692269797 3.5786007895132496692269797 1.9597414145132496798851207 1.9597414145132496798851207 1.9597414145132496798851207 1.9597414145132496798851207 2.9186476645132501950286041 2.9186476645132501950286041 2.9186476645132501950286041 2.9186476645132501950286041 3.1648609457632499442070184 3.1648609457632499442070184 3.1648609457632499442070184 3.1648609457632499442070184 + +1 +3.4539682336199621914829549 3.4539682336199621914829549 3.4539682336199621914829549 3.4539682336199621914829549 3.6280436285795252260300003 3.6280436285795252260300003 3.6280436285795252260300003 3.6280436285795252260300003 1.9034148435503734919649332 1.9034148435503734919649332 1.9034148435503734919649332 1.9034148435503734919649332 2.6899323232536365146927437 2.6899323232536365146927437 2.6899323232536365146927437 2.6899323232536365146927437 3.0719502784367991310432444 3.0719502784367991310432444 3.0719502784367991310432444 3.0719502784367991310432444 + +0 +3.7082978001441064819232452 3.7082978001441064819232452 3.7082978001441064819232452 3.7082978001441064819232452 3.7263478001441061593368431 3.7263478001441061593368431 3.7263478001441061593368431 3.7263478001441061593368431 1.9213478001441059994647276 1.9213478001441059994647276 1.9213478001441059994647276 1.9213478001441059994647276 1.9213478001441059994647276 1.9213478001441059994647276 1.9213478001441059994647276 1.9213478001441059994647276 2.7335978001441065821097709 2.7335978001441065821097709 2.7335978001441065821097709 2.7335978001441065821097709 + +0 +3.7324713685980364097360962 3.7324713685980364097360962 3.7324713685980364097360962 3.7324713685980364097360962 3.9224713685980363564453910 3.9224713685980363564453910 3.9224713685980363564453910 3.9224713685980363564453910 2.0224713685980355570848133 2.0224713685980355570848133 2.0224713685980355570848133 2.0224713685980355570848133 2.0224713685980355570848133 2.0224713685980355570848133 2.0224713685980355570848133 2.0224713685980355570848133 2.0224713685980355570848133 2.0224713685980355570848133 2.0224713685980355570848133 2.0224713685980355570848133 + diff --git a/grl/environment/pomdp_files/paint.95-action-cross-16583.pg b/grl/environment/pomdp_files/paint.95-action-cross-16583.pg new file mode 100644 index 00000000..d83f866d --- /dev/null +++ b/grl/environment/pomdp_files/paint.95-action-cross-16583.pg @@ -0,0 +1,12 @@ +0 1 1 3 - +1 1 4 0 - +2 1 4 3 - +3 3 6 - - +4 2 6 - - +5 1 7 3 - +6 1 11 3 - +7 1 11 5 - +8 1 11 6 - +9 1 11 7 - +10 0 11 - - +11 0 4 - - diff --git a/grl/environment/pomdp_files/paint.95-action-cross.POMDP b/grl/environment/pomdp_files/paint.95-action-cross.POMDP new file mode 100644 index 00000000..4a138a4f --- /dev/null +++ b/grl/environment/pomdp_files/paint.95-action-cross.POMDP @@ -0,0 +1,575 @@ +# Converted POMDP file for paint.95 + +discount: 0.95 +values: reward +states: 20 +actions: 4 +observations: 3 + +start: +0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.25 0.25 0.25 0.25 + +T: 0 +0.1 0.0 0.0 0.0 0.9 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.1 0.0 0.0 0.0 0.9 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.1 0.0 0.0 0.0 0.9 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.1 0.0 0.0 0.0 0.9 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.9 0.0 0.0 0.0 0.1 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.9 0.0 0.0 0.0 0.1 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.9 0.0 0.0 0.0 0.1 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.9 0.0 0.0 0.0 0.1 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 + +T: 1 +0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 +0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 + +T: 2 +0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 +0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 + +T: 3 +0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 +0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.0 + + +O: * +1.0 0.0 0.0 +0.75 0.25 0.0 +1.0 0.0 0.0 +1.0 0.0 0.0 +1.0 0.0 0.0 +0.75 0.25 0.0 +1.0 0.0 0.0 +1.0 0.0 0.0 +1.0 0.0 0.0 +0.75 0.25 0.0 +1.0 0.0 0.0 +1.0 0.0 0.0 +1.0 0.0 0.0 +0.25 0.75 0.0 +1.0 0.0 0.0 +1.0 0.0 0.0 +0.0 0.0 1.0 +0.0 0.0 1.0 +0.0 0.0 1.0 +0.0 0.0 1.0 + + + +R: 2 : 0 : 0 : * -1.0 +R: 2 : 0 : 1 : * -1.0 +R: 2 : 0 : 2 : * -1.0 +R: 2 : 0 : 3 : * -1.0 +R: 2 : 0 : 4 : * -1.0 +R: 2 : 0 : 5 : * -1.0 +R: 2 : 0 : 6 : * -1.0 +R: 2 : 0 : 7 : * -1.0 +R: 2 : 0 : 8 : * -1.0 +R: 2 : 0 : 9 : * -1.0 +R: 2 : 0 : 10 : * -1.0 +R: 2 : 0 : 11 : * -1.0 +R: 2 : 0 : 12 : * -1.0 +R: 2 : 0 : 13 : * -1.0 +R: 2 : 0 : 14 : * -1.0 +R: 2 : 0 : 15 : * -1.0 +R: 2 : 1 : 0 : * -1.0 +R: 2 : 1 : 1 : * -1.0 +R: 2 : 1 : 2 : * -1.0 +R: 2 : 1 : 3 : * -1.0 +R: 2 : 1 : 4 : * -1.0 +R: 2 : 1 : 5 : * -1.0 +R: 2 : 1 : 6 : * -1.0 +R: 2 : 1 : 7 : * -1.0 +R: 2 : 1 : 8 : * -1.0 +R: 2 : 1 : 9 : * -1.0 +R: 2 : 1 : 10 : * -1.0 +R: 2 : 1 : 11 : * -1.0 +R: 2 : 1 : 12 : * -1.0 +R: 2 : 1 : 13 : * -1.0 +R: 2 : 1 : 14 : * -1.0 +R: 2 : 1 : 15 : * -1.0 +R: 2 : 2 : 0 : * -1.0 +R: 2 : 2 : 1 : * -1.0 +R: 2 : 2 : 2 : * -1.0 +R: 2 : 2 : 3 : * -1.0 +R: 2 : 2 : 4 : * -1.0 +R: 2 : 2 : 5 : * -1.0 +R: 2 : 2 : 6 : * -1.0 +R: 2 : 2 : 7 : * -1.0 +R: 2 : 2 : 8 : * -1.0 +R: 2 : 2 : 9 : * -1.0 +R: 2 : 2 : 10 : * -1.0 +R: 2 : 2 : 11 : * -1.0 +R: 2 : 2 : 12 : * -1.0 +R: 2 : 2 : 13 : * -1.0 +R: 2 : 2 : 14 : * -1.0 +R: 2 : 2 : 15 : * -1.0 +R: 2 : 3 : 0 : * -1.0 +R: 2 : 3 : 1 : * -1.0 +R: 2 : 3 : 2 : * -1.0 +R: 2 : 3 : 3 : * -1.0 +R: 2 : 3 : 4 : * -1.0 +R: 2 : 3 : 5 : * -1.0 +R: 2 : 3 : 6 : * -1.0 +R: 2 : 3 : 7 : * -1.0 +R: 2 : 3 : 8 : * -1.0 +R: 2 : 3 : 9 : * -1.0 +R: 2 : 3 : 10 : * -1.0 +R: 2 : 3 : 11 : * -1.0 +R: 2 : 3 : 12 : * -1.0 +R: 2 : 3 : 13 : * -1.0 +R: 2 : 3 : 14 : * -1.0 +R: 2 : 3 : 15 : * -1.0 +R: 2 : 4 : 0 : * 1.0 +R: 2 : 4 : 1 : * 1.0 +R: 2 : 4 : 2 : * 1.0 +R: 2 : 4 : 3 : * 1.0 +R: 2 : 4 : 4 : * 1.0 +R: 2 : 4 : 5 : * 1.0 +R: 2 : 4 : 6 : * 1.0 +R: 2 : 4 : 7 : * 1.0 +R: 2 : 4 : 8 : * 1.0 +R: 2 : 4 : 9 : * 1.0 +R: 2 : 4 : 10 : * 1.0 +R: 2 : 4 : 11 : * 1.0 +R: 2 : 4 : 12 : * 1.0 +R: 2 : 4 : 13 : * 1.0 +R: 2 : 4 : 14 : * 1.0 +R: 2 : 4 : 15 : * 1.0 +R: 2 : 5 : 0 : * 1.0 +R: 2 : 5 : 1 : * 1.0 +R: 2 : 5 : 2 : * 1.0 +R: 2 : 5 : 3 : * 1.0 +R: 2 : 5 : 4 : * 1.0 +R: 2 : 5 : 5 : * 1.0 +R: 2 : 5 : 6 : * 1.0 +R: 2 : 5 : 7 : * 1.0 +R: 2 : 5 : 8 : * 1.0 +R: 2 : 5 : 9 : * 1.0 +R: 2 : 5 : 10 : * 1.0 +R: 2 : 5 : 11 : * 1.0 +R: 2 : 5 : 12 : * 1.0 +R: 2 : 5 : 13 : * 1.0 +R: 2 : 5 : 14 : * 1.0 +R: 2 : 5 : 15 : * 1.0 +R: 2 : 6 : 0 : * 1.0 +R: 2 : 6 : 1 : * 1.0 +R: 2 : 6 : 2 : * 1.0 +R: 2 : 6 : 3 : * 1.0 +R: 2 : 6 : 4 : * 1.0 +R: 2 : 6 : 5 : * 1.0 +R: 2 : 6 : 6 : * 1.0 +R: 2 : 6 : 7 : * 1.0 +R: 2 : 6 : 8 : * 1.0 +R: 2 : 6 : 9 : * 1.0 +R: 2 : 6 : 10 : * 1.0 +R: 2 : 6 : 11 : * 1.0 +R: 2 : 6 : 12 : * 1.0 +R: 2 : 6 : 13 : * 1.0 +R: 2 : 6 : 14 : * 1.0 +R: 2 : 6 : 15 : * 1.0 +R: 2 : 7 : 0 : * 1.0 +R: 2 : 7 : 1 : * 1.0 +R: 2 : 7 : 2 : * 1.0 +R: 2 : 7 : 3 : * 1.0 +R: 2 : 7 : 4 : * 1.0 +R: 2 : 7 : 5 : * 1.0 +R: 2 : 7 : 6 : * 1.0 +R: 2 : 7 : 7 : * 1.0 +R: 2 : 7 : 8 : * 1.0 +R: 2 : 7 : 9 : * 1.0 +R: 2 : 7 : 10 : * 1.0 +R: 2 : 7 : 11 : * 1.0 +R: 2 : 7 : 12 : * 1.0 +R: 2 : 7 : 13 : * 1.0 +R: 2 : 7 : 14 : * 1.0 +R: 2 : 7 : 15 : * 1.0 +R: 2 : 8 : 0 : * -1.0 +R: 2 : 8 : 1 : * -1.0 +R: 2 : 8 : 2 : * -1.0 +R: 2 : 8 : 3 : * -1.0 +R: 2 : 8 : 4 : * -1.0 +R: 2 : 8 : 5 : * -1.0 +R: 2 : 8 : 6 : * -1.0 +R: 2 : 8 : 7 : * -1.0 +R: 2 : 8 : 8 : * -1.0 +R: 2 : 8 : 9 : * -1.0 +R: 2 : 8 : 10 : * -1.0 +R: 2 : 8 : 11 : * -1.0 +R: 2 : 8 : 12 : * -1.0 +R: 2 : 8 : 13 : * -1.0 +R: 2 : 8 : 14 : * -1.0 +R: 2 : 8 : 15 : * -1.0 +R: 2 : 9 : 0 : * -1.0 +R: 2 : 9 : 1 : * -1.0 +R: 2 : 9 : 2 : * -1.0 +R: 2 : 9 : 3 : * -1.0 +R: 2 : 9 : 4 : * -1.0 +R: 2 : 9 : 5 : * -1.0 +R: 2 : 9 : 6 : * -1.0 +R: 2 : 9 : 7 : * -1.0 +R: 2 : 9 : 8 : * -1.0 +R: 2 : 9 : 9 : * -1.0 +R: 2 : 9 : 10 : * -1.0 +R: 2 : 9 : 11 : * -1.0 +R: 2 : 9 : 12 : * -1.0 +R: 2 : 9 : 13 : * -1.0 +R: 2 : 9 : 14 : * -1.0 +R: 2 : 9 : 15 : * -1.0 +R: 2 : 10 : 0 : * -1.0 +R: 2 : 10 : 1 : * -1.0 +R: 2 : 10 : 2 : * -1.0 +R: 2 : 10 : 3 : * -1.0 +R: 2 : 10 : 4 : * -1.0 +R: 2 : 10 : 5 : * -1.0 +R: 2 : 10 : 6 : * -1.0 +R: 2 : 10 : 7 : * -1.0 +R: 2 : 10 : 8 : * -1.0 +R: 2 : 10 : 9 : * -1.0 +R: 2 : 10 : 10 : * -1.0 +R: 2 : 10 : 11 : * -1.0 +R: 2 : 10 : 12 : * -1.0 +R: 2 : 10 : 13 : * -1.0 +R: 2 : 10 : 14 : * -1.0 +R: 2 : 10 : 15 : * -1.0 +R: 2 : 11 : 0 : * -1.0 +R: 2 : 11 : 1 : * -1.0 +R: 2 : 11 : 2 : * -1.0 +R: 2 : 11 : 3 : * -1.0 +R: 2 : 11 : 4 : * -1.0 +R: 2 : 11 : 5 : * -1.0 +R: 2 : 11 : 6 : * -1.0 +R: 2 : 11 : 7 : * -1.0 +R: 2 : 11 : 8 : * -1.0 +R: 2 : 11 : 9 : * -1.0 +R: 2 : 11 : 10 : * -1.0 +R: 2 : 11 : 11 : * -1.0 +R: 2 : 11 : 12 : * -1.0 +R: 2 : 11 : 13 : * -1.0 +R: 2 : 11 : 14 : * -1.0 +R: 2 : 11 : 15 : * -1.0 +R: 2 : 12 : 0 : * -1.0 +R: 2 : 12 : 1 : * -1.0 +R: 2 : 12 : 2 : * -1.0 +R: 2 : 12 : 3 : * -1.0 +R: 2 : 12 : 4 : * -1.0 +R: 2 : 12 : 5 : * -1.0 +R: 2 : 12 : 6 : * -1.0 +R: 2 : 12 : 7 : * -1.0 +R: 2 : 12 : 8 : * -1.0 +R: 2 : 12 : 9 : * -1.0 +R: 2 : 12 : 10 : * -1.0 +R: 2 : 12 : 11 : * -1.0 +R: 2 : 12 : 12 : * -1.0 +R: 2 : 12 : 13 : * -1.0 +R: 2 : 12 : 14 : * -1.0 +R: 2 : 12 : 15 : * -1.0 +R: 2 : 13 : 0 : * -1.0 +R: 2 : 13 : 1 : * -1.0 +R: 2 : 13 : 2 : * -1.0 +R: 2 : 13 : 3 : * -1.0 +R: 2 : 13 : 4 : * -1.0 +R: 2 : 13 : 5 : * -1.0 +R: 2 : 13 : 6 : * -1.0 +R: 2 : 13 : 7 : * -1.0 +R: 2 : 13 : 8 : * -1.0 +R: 2 : 13 : 9 : * -1.0 +R: 2 : 13 : 10 : * -1.0 +R: 2 : 13 : 11 : * -1.0 +R: 2 : 13 : 12 : * -1.0 +R: 2 : 13 : 13 : * -1.0 +R: 2 : 13 : 14 : * -1.0 +R: 2 : 13 : 15 : * -1.0 +R: 2 : 14 : 0 : * -1.0 +R: 2 : 14 : 1 : * -1.0 +R: 2 : 14 : 2 : * -1.0 +R: 2 : 14 : 3 : * -1.0 +R: 2 : 14 : 4 : * -1.0 +R: 2 : 14 : 5 : * -1.0 +R: 2 : 14 : 6 : * -1.0 +R: 2 : 14 : 7 : * -1.0 +R: 2 : 14 : 8 : * -1.0 +R: 2 : 14 : 9 : * -1.0 +R: 2 : 14 : 10 : * -1.0 +R: 2 : 14 : 11 : * -1.0 +R: 2 : 14 : 12 : * -1.0 +R: 2 : 14 : 13 : * -1.0 +R: 2 : 14 : 14 : * -1.0 +R: 2 : 14 : 15 : * -1.0 +R: 2 : 15 : 0 : * -1.0 +R: 2 : 15 : 1 : * -1.0 +R: 2 : 15 : 2 : * -1.0 +R: 2 : 15 : 3 : * -1.0 +R: 2 : 15 : 4 : * -1.0 +R: 2 : 15 : 5 : * -1.0 +R: 2 : 15 : 6 : * -1.0 +R: 2 : 15 : 7 : * -1.0 +R: 2 : 15 : 8 : * -1.0 +R: 2 : 15 : 9 : * -1.0 +R: 2 : 15 : 10 : * -1.0 +R: 2 : 15 : 11 : * -1.0 +R: 2 : 15 : 12 : * -1.0 +R: 2 : 15 : 13 : * -1.0 +R: 2 : 15 : 14 : * -1.0 +R: 2 : 15 : 15 : * -1.0 + +R: 3 : 0 : 0 : * -1.0 +R: 3 : 0 : 1 : * -1.0 +R: 3 : 0 : 2 : * -1.0 +R: 3 : 0 : 3 : * -1.0 +R: 3 : 0 : 4 : * -1.0 +R: 3 : 0 : 5 : * -1.0 +R: 3 : 0 : 6 : * -1.0 +R: 3 : 0 : 7 : * -1.0 +R: 3 : 0 : 8 : * -1.0 +R: 3 : 0 : 9 : * -1.0 +R: 3 : 0 : 10 : * -1.0 +R: 3 : 0 : 11 : * -1.0 +R: 3 : 0 : 12 : * -1.0 +R: 3 : 0 : 13 : * -1.0 +R: 3 : 0 : 14 : * -1.0 +R: 3 : 0 : 15 : * -1.0 +R: 3 : 1 : 0 : * -1.0 +R: 3 : 1 : 1 : * -1.0 +R: 3 : 1 : 2 : * -1.0 +R: 3 : 1 : 3 : * -1.0 +R: 3 : 1 : 4 : * -1.0 +R: 3 : 1 : 5 : * -1.0 +R: 3 : 1 : 6 : * -1.0 +R: 3 : 1 : 7 : * -1.0 +R: 3 : 1 : 8 : * -1.0 +R: 3 : 1 : 9 : * -1.0 +R: 3 : 1 : 10 : * -1.0 +R: 3 : 1 : 11 : * -1.0 +R: 3 : 1 : 12 : * -1.0 +R: 3 : 1 : 13 : * -1.0 +R: 3 : 1 : 14 : * -1.0 +R: 3 : 1 : 15 : * -1.0 +R: 3 : 2 : 0 : * -1.0 +R: 3 : 2 : 1 : * -1.0 +R: 3 : 2 : 2 : * -1.0 +R: 3 : 2 : 3 : * -1.0 +R: 3 : 2 : 4 : * -1.0 +R: 3 : 2 : 5 : * -1.0 +R: 3 : 2 : 6 : * -1.0 +R: 3 : 2 : 7 : * -1.0 +R: 3 : 2 : 8 : * -1.0 +R: 3 : 2 : 9 : * -1.0 +R: 3 : 2 : 10 : * -1.0 +R: 3 : 2 : 11 : * -1.0 +R: 3 : 2 : 12 : * -1.0 +R: 3 : 2 : 13 : * -1.0 +R: 3 : 2 : 14 : * -1.0 +R: 3 : 2 : 15 : * -1.0 +R: 3 : 3 : 0 : * -1.0 +R: 3 : 3 : 1 : * -1.0 +R: 3 : 3 : 2 : * -1.0 +R: 3 : 3 : 3 : * -1.0 +R: 3 : 3 : 4 : * -1.0 +R: 3 : 3 : 5 : * -1.0 +R: 3 : 3 : 6 : * -1.0 +R: 3 : 3 : 7 : * -1.0 +R: 3 : 3 : 8 : * -1.0 +R: 3 : 3 : 9 : * -1.0 +R: 3 : 3 : 10 : * -1.0 +R: 3 : 3 : 11 : * -1.0 +R: 3 : 3 : 12 : * -1.0 +R: 3 : 3 : 13 : * -1.0 +R: 3 : 3 : 14 : * -1.0 +R: 3 : 3 : 15 : * -1.0 +R: 3 : 4 : 0 : * -1.0 +R: 3 : 4 : 1 : * -1.0 +R: 3 : 4 : 2 : * -1.0 +R: 3 : 4 : 3 : * -1.0 +R: 3 : 4 : 4 : * -1.0 +R: 3 : 4 : 5 : * -1.0 +R: 3 : 4 : 6 : * -1.0 +R: 3 : 4 : 7 : * -1.0 +R: 3 : 4 : 8 : * -1.0 +R: 3 : 4 : 9 : * -1.0 +R: 3 : 4 : 10 : * -1.0 +R: 3 : 4 : 11 : * -1.0 +R: 3 : 4 : 12 : * -1.0 +R: 3 : 4 : 13 : * -1.0 +R: 3 : 4 : 14 : * -1.0 +R: 3 : 4 : 15 : * -1.0 +R: 3 : 5 : 0 : * -1.0 +R: 3 : 5 : 1 : * -1.0 +R: 3 : 5 : 2 : * -1.0 +R: 3 : 5 : 3 : * -1.0 +R: 3 : 5 : 4 : * -1.0 +R: 3 : 5 : 5 : * -1.0 +R: 3 : 5 : 6 : * -1.0 +R: 3 : 5 : 7 : * -1.0 +R: 3 : 5 : 8 : * -1.0 +R: 3 : 5 : 9 : * -1.0 +R: 3 : 5 : 10 : * -1.0 +R: 3 : 5 : 11 : * -1.0 +R: 3 : 5 : 12 : * -1.0 +R: 3 : 5 : 13 : * -1.0 +R: 3 : 5 : 14 : * -1.0 +R: 3 : 5 : 15 : * -1.0 +R: 3 : 6 : 0 : * -1.0 +R: 3 : 6 : 1 : * -1.0 +R: 3 : 6 : 2 : * -1.0 +R: 3 : 6 : 3 : * -1.0 +R: 3 : 6 : 4 : * -1.0 +R: 3 : 6 : 5 : * -1.0 +R: 3 : 6 : 6 : * -1.0 +R: 3 : 6 : 7 : * -1.0 +R: 3 : 6 : 8 : * -1.0 +R: 3 : 6 : 9 : * -1.0 +R: 3 : 6 : 10 : * -1.0 +R: 3 : 6 : 11 : * -1.0 +R: 3 : 6 : 12 : * -1.0 +R: 3 : 6 : 13 : * -1.0 +R: 3 : 6 : 14 : * -1.0 +R: 3 : 6 : 15 : * -1.0 +R: 3 : 7 : 0 : * -1.0 +R: 3 : 7 : 1 : * -1.0 +R: 3 : 7 : 2 : * -1.0 +R: 3 : 7 : 3 : * -1.0 +R: 3 : 7 : 4 : * -1.0 +R: 3 : 7 : 5 : * -1.0 +R: 3 : 7 : 6 : * -1.0 +R: 3 : 7 : 7 : * -1.0 +R: 3 : 7 : 8 : * -1.0 +R: 3 : 7 : 9 : * -1.0 +R: 3 : 7 : 10 : * -1.0 +R: 3 : 7 : 11 : * -1.0 +R: 3 : 7 : 12 : * -1.0 +R: 3 : 7 : 13 : * -1.0 +R: 3 : 7 : 14 : * -1.0 +R: 3 : 7 : 15 : * -1.0 +R: 3 : 12 : 0 : * 1.0 +R: 3 : 12 : 1 : * 1.0 +R: 3 : 12 : 2 : * 1.0 +R: 3 : 12 : 3 : * 1.0 +R: 3 : 12 : 4 : * 1.0 +R: 3 : 12 : 5 : * 1.0 +R: 3 : 12 : 6 : * 1.0 +R: 3 : 12 : 7 : * 1.0 +R: 3 : 12 : 8 : * 1.0 +R: 3 : 12 : 9 : * 1.0 +R: 3 : 12 : 10 : * 1.0 +R: 3 : 12 : 11 : * 1.0 +R: 3 : 12 : 12 : * 1.0 +R: 3 : 12 : 13 : * 1.0 +R: 3 : 12 : 14 : * 1.0 +R: 3 : 12 : 15 : * 1.0 +R: 3 : 13 : 0 : * 1.0 +R: 3 : 13 : 1 : * 1.0 +R: 3 : 13 : 2 : * 1.0 +R: 3 : 13 : 3 : * 1.0 +R: 3 : 13 : 4 : * 1.0 +R: 3 : 13 : 5 : * 1.0 +R: 3 : 13 : 6 : * 1.0 +R: 3 : 13 : 7 : * 1.0 +R: 3 : 13 : 8 : * 1.0 +R: 3 : 13 : 9 : * 1.0 +R: 3 : 13 : 10 : * 1.0 +R: 3 : 13 : 11 : * 1.0 +R: 3 : 13 : 12 : * 1.0 +R: 3 : 13 : 13 : * 1.0 +R: 3 : 13 : 14 : * 1.0 +R: 3 : 13 : 15 : * 1.0 +R: 3 : 14 : 0 : * 1.0 +R: 3 : 14 : 1 : * 1.0 +R: 3 : 14 : 2 : * 1.0 +R: 3 : 14 : 3 : * 1.0 +R: 3 : 14 : 4 : * 1.0 +R: 3 : 14 : 5 : * 1.0 +R: 3 : 14 : 6 : * 1.0 +R: 3 : 14 : 7 : * 1.0 +R: 3 : 14 : 8 : * 1.0 +R: 3 : 14 : 9 : * 1.0 +R: 3 : 14 : 10 : * 1.0 +R: 3 : 14 : 11 : * 1.0 +R: 3 : 14 : 12 : * 1.0 +R: 3 : 14 : 13 : * 1.0 +R: 3 : 14 : 14 : * 1.0 +R: 3 : 14 : 15 : * 1.0 +R: 3 : 15 : 0 : * 1.0 +R: 3 : 15 : 1 : * 1.0 +R: 3 : 15 : 2 : * 1.0 +R: 3 : 15 : 3 : * 1.0 +R: 3 : 15 : 4 : * 1.0 +R: 3 : 15 : 5 : * 1.0 +R: 3 : 15 : 6 : * 1.0 +R: 3 : 15 : 7 : * 1.0 +R: 3 : 15 : 8 : * 1.0 +R: 3 : 15 : 9 : * 1.0 +R: 3 : 15 : 10 : * 1.0 +R: 3 : 15 : 11 : * 1.0 +R: 3 : 15 : 12 : * 1.0 +R: 3 : 15 : 13 : * 1.0 +R: 3 : 15 : 14 : * 1.0 +R: 3 : 15 : 15 : * 1.0 + + diff --git a/grl/environment/pomdp_files/paint.95-pomdp-solver-results.npy b/grl/environment/pomdp_files/paint.95-pomdp-solver-results.npy index 310f4067..6d4ed322 100644 Binary files a/grl/environment/pomdp_files/paint.95-pomdp-solver-results.npy and b/grl/environment/pomdp_files/paint.95-pomdp-solver-results.npy differ diff --git a/grl/environment/pomdp_files/shuttle.95-pomdp-solver-results.npy b/grl/environment/pomdp_files/shuttle.95-pomdp-solver-results.npy index b66a937e..c3d9e4be 100644 Binary files a/grl/environment/pomdp_files/shuttle.95-pomdp-solver-results.npy and b/grl/environment/pomdp_files/shuttle.95-pomdp-solver-results.npy differ diff --git a/grl/environment/pomdp_files/tiger-alt-start-pomdp-solver-results.npy b/grl/environment/pomdp_files/tiger-alt-start-pomdp-solver-results.npy index b00be609..dc992fb3 100644 Binary files a/grl/environment/pomdp_files/tiger-alt-start-pomdp-solver-results.npy and b/grl/environment/pomdp_files/tiger-alt-start-pomdp-solver-results.npy differ diff --git a/grl/environment/pomdp_files/tmaze5-pomdp-solver-results.npy b/grl/environment/pomdp_files/tmaze5-pomdp-solver-results.npy index 9292526f..8504ea63 100644 Binary files a/grl/environment/pomdp_files/tmaze5-pomdp-solver-results.npy and b/grl/environment/pomdp_files/tmaze5-pomdp-solver-results.npy differ diff --git a/grl/environment/wrappers/__init__.py b/grl/environment/wrappers/__init__.py index 909cfc13..13dda8e1 100644 --- a/grl/environment/wrappers/__init__.py +++ b/grl/environment/wrappers/__init__.py @@ -1 +1,6 @@ from .one_hot import OneHotObservationWrapper, OneHotActionConcatWrapper +from .gamma_terminal import GammaTerminalWrapper +from .flat_multi_discrete import FlattenMultiDiscreteActionWrapper +from .discrete import DiscreteObservationWrapper +from .discrete_action import ContinuousToDiscrete +from .array import ArrayObservationWrapper \ No newline at end of file diff --git a/grl/environment/wrappers/array.py b/grl/environment/wrappers/array.py new file mode 100644 index 00000000..359f83dd --- /dev/null +++ b/grl/environment/wrappers/array.py @@ -0,0 +1,25 @@ +import numpy as np +from typing import Union, Tuple +import gymnasium as gym +from gymnasium.spaces import Tuple, MultiDiscrete + +class ArrayObservationWrapper(gym.ObservationWrapper): + """ + A wrapper for a gym env for discrete observations. + The wrapper casts the multi discrete observation in a + single element, 1D numpy array. + """ + def __init__(self, env: Union[gym.Env, gym.Wrapper]) -> None: + super().__init__(env) + assert isinstance(self.env.observation_space, Tuple) + + obs_space = [] + + for sp in self.env.observation_space: + assert isinstance(sp, gym.spaces.Discrete) + obs_space.append(sp.n) + + self.observation_space = gym.spaces.MultiDiscrete(obs_space) + + def observation(self, observation: int) -> np.ndarray: + return np.array(observation) diff --git a/grl/environment/wrappers/discrete.py b/grl/environment/wrappers/discrete.py new file mode 100644 index 00000000..ccb27680 --- /dev/null +++ b/grl/environment/wrappers/discrete.py @@ -0,0 +1,19 @@ +import numpy as np +from typing import Union, Tuple +import gymnasium as gym +from gymnasium.spaces import Discrete, MultiDiscrete + +class DiscreteObservationWrapper(gym.ObservationWrapper): + """ + A wrapper for a gym env for discrete observations. + The wrapper wraps the discrete observation in a + single element, 1D numpy array. + """ + def __init__(self, env: Union[gym.Env, gym.Wrapper]) -> None: + super().__init__(env) + assert isinstance(self.env.observation_space, Discrete) + + self.observation_space = MultiDiscrete([self.env.observation_space.n]) + + def observation(self, observation: int) -> np.ndarray: + return np.array([observation]) diff --git a/grl/environment/wrappers/discrete_action.py b/grl/environment/wrappers/discrete_action.py new file mode 100644 index 00000000..26119dbe --- /dev/null +++ b/grl/environment/wrappers/discrete_action.py @@ -0,0 +1,37 @@ +from typing import Union, List + +import gymnasium as gym +import numpy as np + +class ContinuousToDiscrete(gym.ActionWrapper): + """ + Taken (and modified) from https://github.com/thu-ml/tianshou/blob/master/tianshou/env/gym_wrappers.py + Gym environment wrapper to take discrete action in a continuous environment. + + :param gym.Env env: gym environment with continuous action space. + :param int action_per_dim: number of discrete actions in each dimension + of the action space. + """ + + def __init__(self, env: gym.Env, action_per_dim: Union[int, List[int]]) -> None: + super().__init__(env) + assert isinstance(env.action_space, gym.spaces.Box) + low, high = env.action_space.low, env.action_space.high + if isinstance(action_per_dim, int): + action_per_dim = [action_per_dim] * env.action_space.shape[0] + assert len(action_per_dim) == env.action_space.shape[0] + self.action_space = gym.spaces.MultiDiscrete(action_per_dim) + if len(action_per_dim) == 1: + self.action_space = gym.spaces.Discrete(action_per_dim[0]) + self.mesh = np.array( + [np.linspace(lo, hi, a) for lo, hi, a in zip(low, high, action_per_dim)], + dtype=object + ) + + def action(self, act: Union[np.ndarray, int]) -> np.ndarray: + if isinstance(act, int): + return np.array([self.mesh[0][act]]) + + if len(act.shape) == 1: + return np.array([self.mesh[i][a] for i, a in enumerate(act)]) + return np.array([[self.mesh[i][a] for i, a in enumerate(a_)] for a_ in act]) diff --git a/grl/environment/wrappers/flat_multi_discrete.py b/grl/environment/wrappers/flat_multi_discrete.py new file mode 100644 index 00000000..e51de130 --- /dev/null +++ b/grl/environment/wrappers/flat_multi_discrete.py @@ -0,0 +1,28 @@ +import numpy as np +from typing import Union, Tuple +import gymnasium as gym +from gymnasium.spaces import Discrete, MultiDiscrete + +class FlattenMultiDiscreteActionWrapper(gym.ActionWrapper): + """ + A wrapper for a gym env for multi-discrete actions. + It flattens this action space into a gym.spaces.Discrete action space + """ + def __init__(self, env: Union[gym.Env, gym.Wrapper]) -> None: + super().__init__(env) + assert isinstance(self.env.action_space, MultiDiscrete) + n_total_actions = 1 + for act_space in self.env.action_space: + n_total_actions *= act_space.n + + self.action_space = Discrete(int(n_total_actions)) + + def action(self, action: int) -> np.ndarray: + multi_action = [] + act_so_far = action + for act_space in self.env.action_space[:-1]: + multi_action.append(act_so_far // act_space.n) + act_so_far = act_so_far % act_space.n + + multi_action.append(act_so_far) + return np.array(multi_action) diff --git a/grl/environment/wrappers/gamma_terminal.py b/grl/environment/wrappers/gamma_terminal.py new file mode 100644 index 00000000..31dc4f57 --- /dev/null +++ b/grl/environment/wrappers/gamma_terminal.py @@ -0,0 +1,36 @@ +import numpy as np +from typing import Union, Tuple +import gymnasium as gym + +class GammaTerminalWrapper(gym.Wrapper): + """ + A wrapper for a gymnasium.Env that makes it compatible with our step() API, + in particular the gamma_terminal feature. + """ + def __init__(self, env: Union[gym.Env, gym.Wrapper], gamma: float = None) -> None: + super().__init__(env) + if gamma is None: + if hasattr(self.env, 'gamma'): + self.gamma_terminal_prob = 1. - self.env.gamma + self.gamma = 1. + else: + raise AttributeError( + "If environment does not come with a gamma, need to set gamma for gamma termination" + ) + else: + self.gamma_terminal_prob = 1. - gamma + self.gamma = 1. + + def step(self, action: int, **kwargs) -> Tuple[np.ndarray, float, bool, bool, dict]: + observation, reward, terminal, truncated, info = self.env.step(action, **kwargs) + # if the env happens to have a rand_key, use that for reproducibility. + # Otherwise, just use random.uniform. + # (By the way, this will not error because python evaluates boolean statements lazily.) + if hasattr(self.env, 'rand_key') and self.env.rand_key is not None: + unif = self.rand_key.uniform() + else: + unif = np.random.uniform() + if unif < self.gamma_terminal_prob: + terminal = True + + return observation, reward, terminal, truncated, info diff --git a/grl/environment/wrappers/one_hot.py b/grl/environment/wrappers/one_hot.py index dc7ee917..e362516d 100644 --- a/grl/environment/wrappers/one_hot.py +++ b/grl/environment/wrappers/one_hot.py @@ -2,11 +2,12 @@ import gymnasium as gym from gymnasium import spaces +from popgym.wrappers import PreviousAction import numpy as np from grl.utils.data import one_hot -class OneHotObservationWrapper(gym.Wrapper): +class OneHotObservationWrapper(gym.ObservationWrapper): """ One-hot observation wrapper. Assumes that the env passed into this wrapper has discrete-integer valued observations. @@ -17,21 +18,13 @@ def __init__(self, env: Union[gym.Env, gym.Wrapper]): assert isinstance(self.env.observation_space, spaces.Discrete), \ "Cannot call One-hot wrapper on non-discrete observation space." + def observation(self, observation: int) -> np.ndarray: + return one_hot(observation, self.env.observation_space.n) + @property def observation_space(self) -> spaces.MultiBinary: return spaces.MultiBinary(self.env.observation_space.n) - def reset(self, **kwargs) -> Tuple[np.ndarray, dict]: - obs_idx, info = self.env.reset() - observation = one_hot(obs_idx, self.env.observation_space.n) - return observation, info - - def step(self, action: int, **kwargs): - obs_idx, reward, terminal, truncated, info = self.env.step(action, **kwargs) - - observation = one_hot(obs_idx, self.env.observation_space.n) - return observation, reward, terminal, truncated, info - class OneHotActionConcatWrapper(gym.Wrapper): """ One-hot action concatenation wrapper for observations. @@ -48,16 +41,24 @@ def observation_space(self) -> spaces.Space: return spaces.MultiDiscrete(1 + self.action_space.n) elif isinstance(obs_space, spaces.MultiBinary): return spaces.MultiBinary(obs_space.n + self.action_space.n) + elif isinstance(obs_space, spaces.Box): + return spaces.Box(low=np.concatenate( + [obs_space.low, + np.zeros(self.action_space.n, dtype=obs_space.low.dtype)]), + high=np.concatenate([ + obs_space.high, + np.ones(self.action_space.n, dtype=obs_space.low.dtype) + ])) else: return NotImplementedError def reset(self, **kwargs) -> Tuple[np.ndarray, dict]: # If we are at the start of an episode, - # our action encoding is just a vector of zeros - action_encoding = np.zeros(self.env.action_space.n) - observation, info = self.env.reset() + # get null encoding based on the environment + action_encoding = one_hot(PreviousAction.get_null_action(self.env.action_space), + self.action_space.n) + observation, info = self.env.reset(**kwargs) action_concat_obs = np.concatenate([observation, action_encoding]) - return action_concat_obs, info def step(self, action: int, **kwargs) -> Tuple[np.ndarray, float, bool, bool, dict]: diff --git a/grl/evaluation.py b/grl/evaluation.py index 8717b8d1..f79015f7 100644 --- a/grl/evaluation.py +++ b/grl/evaluation.py @@ -6,21 +6,23 @@ from grl.agent.rnn import RNNAgent from grl.mdp import MDP, POMDP -from grl.utils.data import compress_episode_rewards +from grl.utils.loss import mse +from grl.utils.mdp import all_t_discounted_returns def eval_episodes(agent: RNNAgent, network_params: dict, env: Union[MDP, POMDP], rand_key: random.PRNGKey, n_episodes: int = 1, test_eps: float = 0., max_episode_steps: int = 1000)\ -> Tuple[dict, random.PRNGKey]: - all_ep_rews = [] - all_ep_qs = [] + all_ep_returns = [] + all_ep_discounted_returns = [] + all_ep_q_errs = [] original_epsilon = agent.eps agent.eps = test_eps for ep in trange(n_episodes): - ep_rews = [] + ep_rewards = [] ep_qs = [] hs, rand_key = agent.reset(rand_key) @@ -32,14 +34,13 @@ def eval_episodes(agent: RNNAgent, network_params: dict, ep_qs.append(qs[0, 0, action]) for t in range(max_episode_steps): - # TODO: we have gamma_terminal as False here. Is that what we want? - next_obs, reward, done, _, info = env.step(action, gamma_terminal=False) + next_obs, reward, done, _, info = env.step(action) next_action, rand_key, next_hs, qs = agent.act(network_params, next_obs, hs, rand_key) next_action = next_action.item() ep_qs.append(qs[0, 0, next_action]) - ep_rews.append(reward) + ep_rewards.append(reward) if done: break @@ -47,12 +48,31 @@ def eval_episodes(agent: RNNAgent, network_params: dict, hs = next_hs action = next_action - all_ep_rews.append(compress_episode_rewards(ep_rews)) - all_ep_qs.append(np.array(ep_qs, dtype=np.half)) + ep_qs = np.array(ep_qs) + relevant_ep_qs = ep_qs[:-1] + episode_rewards = np.array(ep_rewards) + + discounts = np.ones(episode_rewards.shape[0]) + if agent.args.no_gamma_terminal: + discounts *= env.gamma + + discounts[-1] = 0. + + t_discounted_returns = all_t_discounted_returns(discounts, episode_rewards) + all_ep_q_errs.append(mse(relevant_ep_qs, t_discounted_returns).item()) + all_ep_returns.append(episode_rewards.sum()) + if agent.args.no_gamma_terminal: + all_ep_discounted_returns.append(t_discounted_returns[0]) # reset original epsilon agent.eps = original_epsilon + all_ep_returns = np.array(all_ep_returns, dtype=np.float16) + all_ep_q_errs = np.array(all_ep_q_errs, dtype=np.float16) + + info = {'episode_returns': all_ep_returns, 'episode_q_errs': all_ep_q_errs} - info = {'episode_rewards': all_ep_rews, 'episode_qs': all_ep_qs} + if agent.args.no_gamma_terminal: + all_ep_discounted_returns = np.array(all_ep_discounted_returns, dtype=np.float16) + info['episode_discounted_returns'] = all_ep_discounted_returns return info, rand_key diff --git a/grl/mdp.py b/grl/mdp.py index 15a1d537..569ba368 100644 --- a/grl/mdp.py +++ b/grl/mdp.py @@ -67,7 +67,8 @@ def random_observation_fn(n_states, n_obs_per_block): @register_pytree_node_class class MDP(gym.Env): - def __init__(self, T, R, p0, gamma=0.9, rand_key: np.random.RandomState = None): + def __init__(self, T, R, p0, gamma=0.9, terminal_mask: np.ndarray = None, + rand_key: np.random.RandomState = None): self.gamma = gamma self.T = T self.R = R @@ -84,8 +85,15 @@ def __init__(self, T, R, p0, gamma=0.9, rand_key: np.random.RandomState = None): self.current_state = None self.rand_key = rand_key + # Terminal mask is a boolean mask across all states that indicates + # whether the state is a terminal(/absorbing) state. + self.terminal_mask = terminal_mask if terminal_mask is not None else self.get_terminal_mask() + + def get_terminal_mask(self): + return jnp.array([jnp.all(self.T[:, i, i] == 1.) for i in range(self.state_space.n)]) + def tree_flatten(self): - children = (self.T, self.R, self.p0, self.gamma) + children = (self.T, self.R, self.p0, self.gamma, self.terminal_mask) aux_data = None return (children, aux_data) @@ -133,7 +141,7 @@ def stationary_distribution(self, pi=None, p0=None, max_steps=200): old_distr = state_distr return state_distr - def reset(self, state=None): + def reset(self, state=None, **kwargs): if state is None: if self.rand_key is not None: state = self.rand_key.choice(self.state_space.n, p=self.p0) @@ -153,14 +161,6 @@ def step(self, action: int, gamma_terminal: bool = True): # Check if next_state is absorbing state is_absorbing = (self.T[:, next_state, next_state] == 1) terminal = is_absorbing.all() # absorbing for all actions - # Discounting: end episode with probability 1-gamma - if gamma_terminal: - if self.rand_key is not None: - unif = self.rand_key.uniform() - else: - unif = np.random.uniform() - if unif < (1 - self.gamma): - terminal = True truncated = False observation = self.observe(next_state) info = {'state': next_state} @@ -249,6 +249,7 @@ def __repr__(self): return base_str + '\n' + repr(self.phi) def observe(self, s): + assert self.phi[s].sum() == 1 if self.rand_key is not None: return self.rand_key.choice(self.observation_space.n, p=self.phi[s]) diff --git a/grl/memory/analytical.py b/grl/memory/analytical.py index b89b1b48..e241ff55 100644 --- a/grl/memory/analytical.py +++ b/grl/memory/analytical.py @@ -31,5 +31,7 @@ def memory_cross_product(mem_params: jnp.ndarray, pomdp: POMDP): p0_x = jnp.zeros(n_states_x) p0_x = p0_x.at[::n_states_m].set(pomdp.p0) - mem_aug_mdp = MDP(T_x, R_x, p0_x, gamma=pomdp.gamma) + new_terminal_mask = pomdp.terminal_mask.repeat(n_states_m) + mem_aug_mdp = MDP(T_x, R_x, p0_x, gamma=pomdp.gamma, terminal_mask=new_terminal_mask) + return POMDP(mem_aug_mdp, phi_x) diff --git a/grl/memory/lib.py b/grl/memory/lib.py index d3704e1a..85b2c62b 100644 --- a/grl/memory/lib.py +++ b/grl/memory/lib.py @@ -12,6 +12,7 @@ def get_memory(memory_id: str, leakiness: float = 0.1) -> np.ndarray: current_module = globals() mem_name = f'memory_{memory_id}' + mem_func, mem_params = None, None if memory_id.isdigit(): if int(memory_id) == 0: if n_obs is None or n_actions is None: @@ -19,9 +20,7 @@ def get_memory(memory_id: str, mem_params = glorot_init((n_actions, n_obs, n_mem_states, n_mem_states)) else: if mem_name in current_module: - T_mem = current_module[mem_name] - # smooth out for softmax - mem_params = reverse_softmax(T_mem) + mem_func = current_module[mem_name] else: raise NotImplementedError(f'{mem_name} not found in memory_lib.py') from None elif memory_id == 'fuzzy': @@ -32,10 +31,20 @@ def get_memory(memory_id: str, # 1 x 1 x n_mem_states x n_mem_states fuzzy_expanded_identity = np.expand_dims(np.expand_dims(fuzzy_identity, 0), 0) mem_func = fuzzy_expanded_identity.repeat(n_obs, axis=1).repeat(n_actions, axis=0) - - mem_params = reverse_softmax(mem_func) + elif memory_id == 'random_uniform': + mem_func = generate_random_uniform_memory_fn(n_mem_states, n_obs, n_actions) + elif memory_id == 'random_discrete': + mem_func = generate_random_discrete_memory_fn(n_mem_states, n_obs, n_actions) + elif memory_id == 'tiger_alt_start_1bit_optimal': + mem_func = tiger_alt_start_1bit_optimal() + elif memory_id == 'tiger_alt_start_counting': + mem_func = tiger_alt_start_counting(n_mem_states) else: raise NotImplementedError(f"No memory of id {memory_id} exists.") + + if mem_params is None and mem_func is not None: + # we need to softmax if mem_params is None + mem_params = reverse_softmax(mem_func) return mem_params def generate_1bit_mem_fns(n_obs, n_actions): @@ -98,6 +107,69 @@ def all_n_state_deterministic_memory(n_mem_states: int): all_mem_funcs = id[all_idxes] return all_mem_funcs +def unif_simplex(n: int): + """ + Samples uniformly random vector from a simplex in n-dimensions. + Taken from https://stackoverflow.com/questions/65154622/sample-uniformly-at-random-from-a-simplex-in-python + """ + logits = np.random.exponential(scale=1., size=n) + return logits / sum(logits) + +def generate_random_uniform_memory_fn(n_mem_states: int, n_obs: int, n_actions: int): + T_mem = np.zeros((n_actions, n_obs, n_mem_states, n_mem_states)) + + for a in range(n_actions): + for ob in range(n_obs): + for m in range(n_mem_states): + T_mem[a, ob, m] = unif_simplex(n_mem_states) + return T_mem + +def generate_random_discrete_memory_fn(n_mem_states: int, n_obs: int, n_actions: int): + unif_mem = generate_random_uniform_memory_fn(n_mem_states, n_obs, n_actions) + discrete_mem = (np.expand_dims(np.max(unif_mem, axis=-1), -1) == unif_mem).astype(float) + return discrete_mem + +def tiger_alt_start_counting(n_mem_states: int, *args): + """ + The counting memory function simply increments the memory state at every step. + """ + n_obs = 4 + + m_to_next_m_idx = (np.arange(n_mem_states) + 1) % n_mem_states + m_to_m = np.zeros((n_mem_states, n_mem_states)) + m_to_m[np.arange(n_mem_states), m_to_next_m_idx] = 1 + + # we increment for every observation + T_mem_listen = m_to_m[None, :].repeat(n_obs, axis=0) + + T_mem = np.stack([T_mem_listen, T_mem_listen, T_mem_listen]) + return T_mem + +def tiger_alt_start_1bit_optimal(): + T_mem_listen = np.array([ + [ # init + [1, 0], + [0, 1] + ], + [ # tiger-left + [0, 1], + [0, 1] + ], + [ # tiger-right + [1, 0], + [1, 0] + ], + [ # terminal + [1, 0], + [0, 1] + ], + ]) + + # Other two options don't matter, since you terminate after taking them. + T_mem = np.stack([T_mem_listen, T_mem_listen, T_mem_listen]) + return T_mem + + """ 1 bit memory functions with three obs: r, b, t and 2 actions: up, down diff --git a/grl/memory_iteration.py b/grl/memory_iteration.py index d8b17eb2..ec307f64 100644 --- a/grl/memory_iteration.py +++ b/grl/memory_iteration.py @@ -5,13 +5,15 @@ from jax.nn import softmax from tqdm import trange from functools import partial +from typing import Callable from grl.agent.analytical import AnalyticalAgent -from grl.utils.lambda_discrep import lambda_discrep_measures -from grl.mdp import POMDP, MDP +from grl.mdp import POMDP from grl.memory import memory_cross_product -from grl.utils.math import glorot_init, greedify -from grl.utils.loss import discrep_loss +from grl.utils.augment_policy import deconstruct_aug_policy +from grl.utils.math import glorot_init, greedify, reverse_softmax +from grl.utils.lambda_discrep import lambda_discrep_measures +from grl.utils.loss import discrep_loss, pg_objective_func from grl.vi import td_pe def run_memory_iteration(pomdp: POMDP, @@ -27,10 +29,12 @@ def run_memory_iteration(pomdp: POMDP, error_type: str = 'l2', value_type: str = 'q', objective: str = 'discrep', + residual: bool = False, lambda_0: float = 0., lambda_1: float = 1., alpha: float = 1., pi_params: jnp.ndarray = None, + kitchen_sink_policies: int = 0, epsilon: float = 0.1, flip_count_prob: bool = False): """ @@ -71,23 +75,27 @@ def run_memory_iteration(pomdp: POMDP, error_type=error_type, value_type=value_type, objective=objective, + residual=residual, lambda_0=lambda_0, lambda_1=lambda_1, alpha=alpha, epsilon=epsilon, flip_count_prob=flip_count_prob) + discrep_loss_fn = partial(discrep_loss, + value_type=value_type, + error_type=error_type, + alpha=alpha) + info, agent = memory_iteration(agent, pomdp, mi_iterations=mi_iterations, pi_per_step=pi_steps, mi_per_step=mi_steps, - init_pi_improvement=init_pi_improvement) + init_pi_improvement=init_pi_improvement, + kitchen_sink_policies=kitchen_sink_policies, + discrep_loss=discrep_loss_fn) - discrep_loss_fn = partial(discrep_loss, - value_type=value_type, - error_type=error_type, - alpha=alpha) get_measures = partial(lambda_discrep_measures, discrep_loss_fn=discrep_loss_fn) info['initial_policy'] = initial_policy @@ -95,7 +103,9 @@ def run_memory_iteration(pomdp: POMDP, # initial policy lambda-discrepancy info['initial_policy_stats'] = get_measures(pomdp, initial_policy) info['initial_improvement_stats'] = get_measures(pomdp, info['initial_improvement_policy']) - greedy_initial_improvement_policy = greedify(info['initial_improvement_policy']) + greedy_initial_improvement_policy = info['initial_improvement_policy'] + if policy_optim_alg == 'policy_iter': + greedy_initial_improvement_policy = greedify(info['initial_improvement_policy']) info['greedy_initial_improvement_stats'] = get_measures(pomdp, greedy_initial_improvement_policy) @@ -113,7 +123,9 @@ def run_memory_iteration(pomdp: POMDP, # Final memory w/ final policy discrep final_mem_pomdp = memory_cross_product(agent.mem_params, pomdp) info['final_mem_stats'] = get_measures(final_mem_pomdp, agent.policy) - greedy_final_policy = greedify(agent.policy) + greedy_final_policy = agent.policy + if policy_optim_alg == 'policy_iter': + greedy_final_policy = greedify(agent.policy) info['greedy_final_mem_stats'] = get_measures(final_mem_pomdp, greedy_final_policy) def perf_from_stats(stats: dict) -> float: @@ -139,6 +151,8 @@ def memory_iteration( mi_per_step: int = 50000, mi_iterations: int = 1, init_pi_improvement: bool = True, + kitchen_sink_policies: int = 0, + discrep_loss: Callable = None, log_every: int = 1000, ): """ @@ -153,13 +167,15 @@ def memory_iteration( :param mi_per_step: Number of memory improvement steps PER memory iteration step. :param mi_iterations: Number of steps of memory iteration to perform. :param init_pi_improvement: Do we start out with an initial policy improvement step? + :param kitchen_sink_policies: Do we select our initial policy based on randomly selected policies + TD optimal? :param log_every: How often do we log stats? """ info = {'policy_improvement_outputs': [], 'mem_loss': []} td_v_vals, td_q_vals = td_pe(agent.policy, init_pomdp) info['initial_values'] = {'v': td_v_vals, 'q': td_q_vals} - if agent.policy_optim_alg in ['discrep_max', 'discrep_min'] or not init_pi_improvement: + if agent.policy_optim_alg in ['discrep_max', 'discrep_min'] or \ + (not init_pi_improvement and agent.policy_optim_alg not in ['policy_grad', 'policy_mem_grad']): initial_pi_params = agent.pi_params.copy() og_policy_optim_algo = agent.policy_optim_alg @@ -177,17 +193,52 @@ def memory_iteration( if init_pi_improvement: # initial policy improvement + poa = agent.policy_optim_alg + prev_optim = agent.pi_optim_state + prev_obj_func = agent.pg_objective_func + if agent.policy_optim_alg in ['policy_mem_grad', 'policy_mem_grad_unrolled']: + agent.policy_optim_alg = 'policy_grad' + agent.pg_objective_func = jax.jit(pg_objective_func) + agent.pi_optim_state = agent.pi_optim.init(agent.pi_params) + print("Initial policy improvement step") initial_outputs = pi_improvement(agent, init_pomdp, iterations=pi_per_step, log_every=log_every) + info['policy_improvement_outputs'].append(initial_outputs) + info['initial_improvement_policy'] = agent.policy.copy() + if poa != agent.policy_optim_alg: + agent.policy_optim_alg = poa + agent.pg_objective_func = prev_obj_func + agent.pi_optim_state = prev_optim + + # here we test random policies + if kitchen_sink_policies > 0: + def get_measure(): + constant_mem = jnp.zeros_like(agent.mem_params) + constant_mem = constant_mem.at[..., 0].set(1) + repeated_pi = softmax(agent.pi_params.repeat(constant_mem.shape[-1], axis=0), axis=-1) + return agent.memory_objective_func(constant_mem, repeated_pi, init_pomdp).item() + best_measure = get_measure() + # best_lambda_discrep = discrep_loss(agent.policy, init_pomdp)[0].item() + best_pi_params = agent.pi_params + + print(f"Finding the argmax {agent.objective} over {kitchen_sink_policies} random policies") + for i in range(kitchen_sink_policies): + agent.reset_pi_params() + measure = get_measure() + if measure > best_measure: + best_pi_params = agent.pi_params + best_measure = measure + agent.pi_params = best_pi_params + + info['starting_policy'] = agent.policy.copy() - info['initial_improvement_policy'] = agent.policy.copy() print(f"Starting (unexpanded) policy: \n{agent.policy}\n") - if agent.mem_params is not None: + if agent.mem_params is not None and agent.policy_optim_alg not in ['policy_mem_grad', 'policy_mem_grad_unrolled']: # we have to set our policy over our memory MDP now # we do so with a random/copied policy given new memory bit info['initial_mem_params'] = agent.mem_params @@ -200,7 +251,7 @@ def memory_iteration( pomdp = copy.deepcopy(init_pomdp) for mem_it in range(mi_iterations): - if agent.mem_params is not None and mi_per_step > 0: + if agent.mem_params is not None and agent.policy_optim_alg not in ['policy_mem_grad', 'policy_mem_grad_unrolled']: print(f"Start MI {mem_it}") mem_loss = mem_improvement(agent, init_pomdp, @@ -216,8 +267,9 @@ def memory_iteration( pomdp = memory_cross_product(agent.mem_params, init_pomdp) if pi_per_step > 0: - # reset our policy parameters - agent.reset_pi_params((pomdp.observation_space.n, pomdp.action_space.n)) + if agent.policy_optim_alg not in ['policy_mem_grad', 'policy_mem_grad_unrolled']: + # reset our policy parameters + agent.reset_pi_params((pomdp.observation_space.n, pomdp.action_space.n)) # Now we improve our policy again policy_output = pi_improvement(agent, @@ -226,9 +278,21 @@ def memory_iteration( log_every=log_every) info['policy_improvement_outputs'].append(policy_output) + policy = agent.policy + if agent.policy_optim_alg in ['policy_mem_grad', 'policy_mem_grad_unrolled']: + agent.mem_params, unflat_policy = deconstruct_aug_policy(softmax(agent.pi_aug_params, axis=-1)) + + O, M, A = unflat_policy.shape + policy = unflat_policy.reshape(O * M, A) + agent.pi_params = reverse_softmax(policy) + + # Plotting for memory iteration + print(f"Learnt memory for iteration {mem_it}: \n" + f"{agent.memory}") + # Plotting for policy improvement print(f"Learnt policy for iteration {mem_it}: \n" - f"{agent.policy}") + f"{policy}") final_pomdp = pomdp @@ -251,6 +315,9 @@ def memory_iteration( # change our mode back agent.policy_optim_alg = og_policy_optim_algo + if agent.policy_optim_alg in ['policy_mem_grad', 'policy_mem_grad_unrolled']: + final_pomdp = memory_cross_product(agent.mem_params, init_pomdp) + # here we calculate our value function for our final policy and final memory td_v_vals, td_q_vals = td_pe(agent.policy, final_pomdp) info['final_outputs'] = {'v': td_v_vals, 'q': td_q_vals} @@ -277,7 +344,7 @@ def pi_improvement(agent: AnalyticalAgent, for it in to_iterate: output = agent.policy_improvement(pomdp) if it % log_every == 0: - if agent.policy_optim_alg == 'policy_grad': + if 'v_0' in output: print(f"initial state value for iteration {it}: {output['v_0'].item():.4f}") if agent.policy_optim_alg in ['discrep_min', 'discrep_max']: print(f"discrep for pi iteration {it}: {output['loss'].item():.4f}") diff --git a/grl/run.py b/grl/run.py index be391302..aa0b4d02 100644 --- a/grl/run.py +++ b/grl/run.py @@ -31,6 +31,7 @@ def add_tmaze_hyperparams(parser: argparse.ArgumentParser): if __name__ == '__main__': start_time = time() + # jax.disable_jit(True) # Args parser = argparse.ArgumentParser() @@ -49,7 +50,10 @@ def add_tmaze_hyperparams(parser: argparse.ArgumentParser): parser.add_argument('--optimizer', type=str, default='adam', help='What optimizer do we use? (sgd | adam | rmsprop)') parser.add_argument('--init_pi', default=None, type=str, - help='Do we initialize our policy to something?') + help='Do we initialize our policy to something? if kitchen_sink, randomly choose a policy,' + 'and take max over randomly chosen/TD optimal.') + parser.add_argument('--kitchen_sink_policies', default=0, type=int, + help='Do we initialize the policy to max LD over n random policies + TD optimal?') parser.add_argument('--use_memory', default=None, type=str, help='use memory function during policy eval if set') parser.add_argument('--mem_leakiness', default=0.1, type=float, @@ -69,7 +73,9 @@ def add_tmaze_hyperparams(parser: argparse.ArgumentParser): parser.add_argument('--error_type', default='l2', type=str, help='Do we use (l2 | abs) for our discrepancies?') parser.add_argument('--objective', default='discrep', type=str, - help='What objective are we trying to optimize? (discrep | magnitude | obs_space)') + help='What objective are we trying to optimize? (discrep | bellman | tde | obs_space)') + parser.add_argument('--residual', action='store_true', + help='For Bellman and TD errors, do we add the residual term?') parser.add_argument('--lr', default=1, type=float) parser.add_argument('--epsilon', default=0.1, type=float, help='(POLICY ITERATION AND TMAZE_EPS_HYPERPARAMS ONLY) What epsilon do we use?') @@ -142,7 +148,9 @@ def add_tmaze_hyperparams(parser: argparse.ArgumentParser): if 'Pi_phi' in pi_dict and pi_dict['Pi_phi'] is not None: logging.info(f'Pi_phi:\n {pi_dict["Pi_phi"]}') if args.init_pi is not None: - pi_params = get_start_pi(args.init_pi, pi_phi=pi_dict['Pi_phi'][0]) + pi_params = get_start_pi(args.init_pi, + pi_phi=pi_dict['Pi_phi'][0], + pomdp=pomdp) results_path = results_path(args) @@ -159,11 +167,13 @@ def add_tmaze_hyperparams(parser: argparse.ArgumentParser): value_type=args.value_type, error_type=args.error_type, objective=args.objective, + residual=args.residual, lambda_0=args.lambda_0, lambda_1=args.lambda_1, alpha=args.alpha, epsilon=args.epsilon, pi_params=pi_params, + kitchen_sink_policies=args.kitchen_sink_policies, flip_count_prob=args.flip_count_prob) info = {'logs': logs, 'args': args.__dict__} diff --git a/grl/run_sample_based.py b/grl/run_sample_based.py index aa4dbf4d..479e0832 100644 --- a/grl/run_sample_based.py +++ b/grl/run_sample_based.py @@ -7,12 +7,11 @@ from grl.agent import get_agent from grl.environment import get_env from grl.evaluation import eval_episodes -from grl.mdp import POMDP, MDP from grl.model import get_network from grl.sample_trainer import Trainer -from grl.utils.data import uncompress_episode_rewards from grl.utils.optimizer import get_optimizer from grl.utils.file_system import results_path, numpyify_and_save +from grl.environment.wrappers import GammaTerminalWrapper def parse_arguments(return_defaults: bool = False): parser = argparse.ArgumentParser() @@ -22,10 +21,12 @@ def parse_arguments(return_defaults: bool = False): help='name of POMDP spec; evals Pi_phi policies by default') parser.add_argument('--no_gamma_terminal', action='store_true', help='Do we turn OFF gamma termination?') + parser.add_argument('--gamma', default=None, type=float, + help='Gamma value: overrides environment gamma for our environments, required for Gym environments.') parser.add_argument('--max_episode_steps', default=1000, type=int, help='Maximum number of episode steps') - parser.add_argument('--feature_encoding', default='one_hot', type=str, - choices=['one_hot', 'discrete'], + parser.add_argument('--feature_encoding', default='none', type=str, + choices=['one_hot', 'discrete', 'none'], help='What feature encoding do we use?') # Agent params @@ -130,10 +131,12 @@ def parse_arguments(return_defaults: bool = False): # Load environment and env wrappers env_key, test_env_key, rand_key = jax.random.split(rand_key, num=3) - env = get_env(args, rand_state=np_rand_key, rand_key=env_key) + env = get_env(args, rand_state=np_rand_key) + if not args.no_gamma_terminal: + env = GammaTerminalWrapper(env, args.gamma) test_env = None if args.offline_eval_freq is not None: - test_env = get_env(args, rand_state=np_rand_key, rand_key=test_env_key) + test_env = get_env(args, rand_state=np_rand_key) # TODO: this is here b/c rock_positions are randomly generated in the env __init__ -- # refactor this! if args.spec == 'rocksample': @@ -149,7 +152,10 @@ def parse_arguments(return_defaults: bool = False): optimizer = get_optimizer(args.optimizer, step_size=args.lr) - agent = get_agent(network, optimizer, env.observation_space.shape, env, args) + obs_size = env.observation_space.shape + if not obs_size: + obs_size = (1, ) + agent = get_agent(network, optimizer, obs_size, env, args) trainer_key, rand_key = jax.random.split(rand_key) trainer = Trainer(env, agent, trainer_key, args, test_env=test_env, checkpoint_dir=agents_dir) @@ -163,18 +169,14 @@ def parse_arguments(return_defaults: bool = False): test_eps=args.offline_eval_epsilon, max_episode_steps=args.max_episode_steps) - summed_perf = 0 - for ep in final_eval_info['episode_rewards']: - full_ep = uncompress_episode_rewards(ep['episode_length'], ep['most_common_reward'], ep['compressed_rewards']) - summed_perf += sum(full_ep) + avg_perf = final_eval_info['episode_returns'].mean() - print(f"Final (averaged) greedy evaluation performance: {summed_perf / args.offline_eval_episodes}") + print(f"Final (averaged) greedy evaluation performance: {avg_perf}") info = { 'episodes_info': episodes_info, 'args': vars(args), - 'final_eval_rewards': final_eval_info['episode_rewards'], - 'final_eval_qs': final_eval_info['episode_qs'] + 'final_eval_info': final_eval_info } print(f"Saving results to {results_path}") diff --git a/grl/sample_trainer.py b/grl/sample_trainer.py index 9f5a217a..edd4de92 100644 --- a/grl/sample_trainer.py +++ b/grl/sample_trainer.py @@ -24,21 +24,28 @@ def __init__(self, rand_key: random.PRNGKey, args: Namespace, test_env: gym.Env = None, - checkpoint_dir: Path = None): + checkpoint_dir: Path = None, + seq_len_bins: int = 25): + """ + :param seq_len_bins: for when args.trunc == -1. For online training, + we'll have variable length episode lengths, which cause a lot of cache misses + for the `update` function. seq_len_bins bins episode lengths into 20 bins, based on + max_episode_length // seq_len_bins intervals between bins. + """ self.env = env self.test_env = test_env self.agent = agent self.args = args self._rand_key = rand_key - self.gamma_terminal = not self.args.no_gamma_terminal - self.discounting = self.env.gamma if not self.gamma_terminal else 1. + self.discounting = self.env.gamma self.max_episode_steps = self.args.max_episode_steps self.total_steps = self.args.total_steps # For RNNs self.trunc = self.args.trunc + self.interval = self.max_episode_steps / seq_len_bins self.action_cond = self.args.action_cond # we train at the end of an episode only if we also need to learn an MC head. @@ -75,9 +82,13 @@ def __init__(self, if self.args.arch == 'lstm': state_size = (2, ) + state_size + obs_size = self.env.observation_space.shape + if not obs_size: + obs_size = (1, ) + self.buffer = EpisodeBuffer(replay_capacity, rand_key, - self.env.observation_space.shape, + obs_size, state_size=state_size, unpack_state=self.args.arch == 'lstm') @@ -138,7 +149,12 @@ def sample_and_update(self, network_params: dict, optimizer_params: optax.Params -> Tuple[dict, optax.Params, dict]: seq_len = self.agent.trunc if self.online_training: # online training - sample = self.buffer.sample_idx(np.arange(len(self.buffer))[None, :]) + seq_len = int(np.ceil(len(self.buffer) / self.interval) * self.interval) \ + if len(self.buffer) > 10 else len(self.buffer) + sample = self.buffer.sample_idx(np.arange(seq_len)[None, :]) + elif self.trunc < 0 and self.args.replay_size > 0: + # if this is the case, we do complete episode rollouts + sample = self.buffer.sample_full_episodes(self.batch_size) else: # sample a sequence from our buffer! sample = self.buffer.sample(self.batch_size, seq_len=seq_len) @@ -190,8 +206,7 @@ def train(self): action = action.astype(int).item() for t in range(self.max_episode_steps): - next_obs, reward, done, _, info = self.env.step(action, - gamma_terminal=self.gamma_terminal) + next_obs, reward, done, _, info = self.env.step(action) next_action, self._rand_key, next_hs, qs = self.agent.act( network_params, next_obs, hs, self._rand_key) next_action = next_action.astype(int).item() @@ -215,7 +230,8 @@ def train(self): self.buffer.push(experience) # If we have enough things in the buffer to update - if self.batch_size < len(self.buffer) and self.trunc < len(self.buffer): + if self.trunc > 0 and \ + (self.batch_size < len(self.buffer) and self.trunc < len(self.buffer)): network_params, optimizer_params, info = self.sample_and_update( network_params, optimizer_params) @@ -246,14 +262,17 @@ def train(self): for b in batch_with_returns: self.buffer.push(b) - # Online MC training - if self.online_training and self.include_returns_in_batch: + # # Online MC training + if self.online_training: + # Only update at the end of an episode network_params, optimizer_params, info = self.sample_and_update( network_params, optimizer_params) episode_info['total_episode_loss'] += info['total_loss'].item() episode_info['episode_updates'] += 1 - episode_info |= compress_episode_rewards(episode_reward) + episode_reward = np.array(episode_reward) + episode_info['episode_returns'] = episode_reward.sum() + episode_info['discounted_returns'] = np.dot(episode_reward, self.discounting ** np.arange(len(episode_reward))) # save all our episode info in lists for k, v in episode_info.items(): @@ -278,4 +297,8 @@ def train(self): if 'total_loss' in all_logs: all_logs['total_loss'] = np.array(all_logs['total_loss'], dtype=np.half) + all_logs['online_info']['total_episode_loss'] = np.array(all_logs['online_info']['total_episode_loss'], dtype=np.float16) + all_logs['online_info']['episode_returns'] = np.array(all_logs['online_info']['episode_returns'], dtype=np.float16) + all_logs['online_info']['discounted_returns'] = np.array(all_logs['online_info']['discounted_returns'], dtype=np.float16) + return network_params, optimizer_params, all_logs diff --git a/grl/utils/augment_policy.py b/grl/utils/augment_policy.py new file mode 100644 index 00000000..4c19ddca --- /dev/null +++ b/grl/utils/augment_policy.py @@ -0,0 +1,70 @@ +from jax.nn import softmax +import jax.numpy as jnp + +from grl.utils import reverse_softmax + +def deconstruct_aug_policy(aug_policy_probs): + O, M, AM = aug_policy_probs.shape + A = AM // M + aug_policy_probs_omam = aug_policy_probs.reshape([O, M, A, M]) + action_policy_probs_oma1 = aug_policy_probs_omam.sum(-1, keepdims=1) # (O, M, A, 1) + # pr(^|*) + action_policy_probs = action_policy_probs_oma1.squeeze(-1) + # assert np.allclose(action_policy_probs.sum(-1), 1) + + aug_policy_logits_omam = reverse_softmax(aug_policy_probs_omam) + action_policy_logits_oma1 = reverse_softmax(action_policy_probs_oma1) + + # Is there an independence assumption here from the log-sum-exp trick? + mem_logits_omam = (aug_policy_logits_omam - action_policy_logits_oma1) # (O, M, A, M) + mem_probs_omam = softmax(mem_logits_omam, -1) # (O, M, A, M) + # pr(^|*) + mem_probs = jnp.moveaxis(mem_probs_omam, -2, 0) # (A, O, M, M) + # assert np.allclose(mem_probs.sum(-1), 1) + + mem_logits = reverse_softmax(mem_probs) + return mem_logits, action_policy_probs + +def construct_aug_policy(mem_probs: jnp.ndarray, policy_probs: jnp.ndarray): + A, O, M, _ = mem_probs.shape + mem_probs_omam = jnp.moveaxis(mem_probs, 0, -2) # (O, M, A, M) + inp_aug_pi = jnp.expand_dims(policy_probs, axis=1).repeat(M, axis=1) + + policy_probs_oma1 = inp_aug_pi[..., None] # (O, M, A, 1) + + aug_policy_probs_omam = (mem_probs_omam * policy_probs_oma1) + aug_policy_probs = aug_policy_probs_omam.reshape([O, M, A*M]) + # assert np.allclose(aug_policy_probs.sum(-1), 1) + + return aug_policy_probs + +if __name__ == "__main__": + import numpy as np + A = 3 + M = 2 + O = 4 + aug_policy_probs = softmax(np.random.normal(size=np.prod([A, O, M, M])).reshape([O, M, A * M]), axis=-1) + mem_logits, action_policy_probs = deconstruct_aug_policy(aug_policy_probs) + aug_policy_probs_reconstructed = construct_aug_policy(mem_logits, action_policy_probs) + assert np.allclose(aug_policy_probs_reconstructed, aug_policy_probs) + + + import numpy as np + + from grl.environment.spec import load_pomdp + from grl.memory import get_memory + + env, info = load_pomdp('tmaze_5_two_thirds_up', memory_id='18') + pi = info['Pi_phi'][0] + mem_params = get_memory('18', + n_obs=env.observation_space.n, + n_actions=env.action_space.n, + n_mem_states=2) + + inp_aug_pi = np.expand_dims(pi, axis=1).repeat(mem_params.shape[-1], axis=1) + + aug_policy = construct_aug_policy(mem_params, inp_aug_pi) + mem_logits_reconstr, deconstr_aug_pi = deconstruct_aug_policy(aug_policy) + + softmax(mem_logits_reconstr, -1).round(3) + softmax(mem_params, -1).round(3) \ No newline at end of file diff --git a/grl/utils/file_system.py b/grl/utils/file_system.py index 25bda7ac..3c2414ee 100644 --- a/grl/utils/file_system.py +++ b/grl/utils/file_system.py @@ -1,12 +1,14 @@ -import numpy as np +from argparse import Namespace import hashlib -import time +import importlib import jax.numpy as jnp from pathlib import Path -from argparse import Namespace -from definitions import ROOT_DIR +import numpy as np +import time from typing import Union +from definitions import ROOT_DIR + def results_path(args: Namespace): results_dir = Path(ROOT_DIR, 'results') results_dir.mkdir(exist_ok=True) @@ -56,3 +58,11 @@ def make_hashable(o): return tuple(sorted(make_hashable(e) for e in o)) return o + + +def import_module_to_var(fpath: Path, var_name: str) -> Union[dict, list]: + spec = importlib.util.spec_from_file_location(var_name, fpath) + var_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(var_module) + instantiated_var = getattr(var_module, var_name) + return instantiated_var diff --git a/grl/utils/loss.py b/grl/utils/loss.py index 698c757e..407d9f9a 100644 --- a/grl/utils/loss.py +++ b/grl/utils/loss.py @@ -2,6 +2,8 @@ from jax import nn, lax, jit from functools import partial +from grl.utils.augment_policy import deconstruct_aug_policy +from grl.utils.math import reverse_softmax from grl.utils.mdp import functional_get_occupancy, get_p_s_given_o, functional_create_td_model from grl.utils.policy_eval import analytical_pe, lstdq_lambda, functional_solve_mdp from grl.memory import memory_cross_product @@ -59,9 +61,9 @@ def weight_and_sum_discrep_loss(diff: jnp.ndarray, if flip_count_prob: count_o = nn.softmax(-count_o) - # count_mask = (1 - jnp.isclose(count_o, 0, atol=1e-12)).astype(float) - # uniform_o = (jnp.ones(pi.shape[0]) / count_mask.sum()) * count_mask - uniform_o = jnp.ones(pi.shape[0]) + count_mask = (1 - jnp.isclose(count_o, 0, atol=1e-12)).astype(float) + uniform_o = (jnp.ones(pi.shape[0]) / count_mask.sum()) * count_mask + # uniform_o = jnp.ones(pi.shape[0]) p_o = alpha * uniform_o + (1 - alpha) * count_o @@ -109,7 +111,7 @@ def discrep_loss( lambda_1_vals = {'v': lambda_1_v_vals, 'q': lambda_1_q_vals} diff = lambda_1_vals[value_type] - lambda_0_vals[value_type] - c_s = info['occupancy'].at[-2:].set(0) + c_s = info['occupancy'] * (1 - pomdp.terminal_mask) loss = weight_and_sum_discrep_loss(diff, c_s, pi, @@ -188,9 +190,8 @@ def obs_space_mem_discrep_loss( diff = lambda_1_vals[value_type] - lambda_0_vals[value_type] - c_s = info['occupancy'] # set terminal counts to 0 - c_s = c_s.at[-1:].set(0) + c_s = info['occupancy'] * (1 - pomdp.terminal_mask) loss = weight_and_sum_discrep_loss(diff, c_s, @@ -228,7 +229,9 @@ def pg_objective_func(pi_params: jnp.ndarray, pomdp: POMDP): """ pi_abs = nn.softmax(pi_params, axis=-1) pi_ground = pomdp.phi @ pi_abs - occupancy = functional_get_occupancy(pi_ground, pomdp) + + # Terminals have p(S) = 0. + occupancy = functional_get_occupancy(pi_ground, pomdp) * (1 - pomdp.terminal_mask) p_pi_of_s_given_o = get_p_s_given_o(pomdp.phi, occupancy) T_obs_obs, R_obs_obs = functional_create_td_model(p_pi_of_s_given_o, pomdp) @@ -237,81 +240,233 @@ def pg_objective_func(pi_params: jnp.ndarray, pomdp: POMDP): p_init_obs = pomdp.p0 @ pomdp.phi return jnp.dot(p_init_obs, td_v_vals), (td_v_vals, td_q_vals) -def mem_magnitude_td_loss( +def mem_pg_objective_func(augmented_pi_params: jnp.ndarray, pomdp: POMDP): + augmented_pi_probs = nn.softmax(augmented_pi_params, axis=-1) + mem_logits, action_policy_probs = deconstruct_aug_policy(augmented_pi_probs) + mem_aug_mdp = memory_cross_product(mem_logits, pomdp) + O, M, A = action_policy_probs.shape + return pg_objective_func(reverse_softmax(action_policy_probs).reshape(O * M, A), mem_aug_mdp) + +def unrolled_mem_pg_objective_func(augmented_pi_params: jnp.ndarray, pomdp: POMDP):# O, M, AM + augmented_pi_probs_unflat = nn.softmax(augmented_pi_params, axis=-1) + mem_logits, action_policy_probs = deconstruct_aug_policy(augmented_pi_probs_unflat)# A,O,M->M ; O,M->A + O, M, A = action_policy_probs.shape + mem_aug_mdp = memory_cross_product(mem_logits, pomdp) + + pi_probs = action_policy_probs.reshape(O * M, A) + aug_pi_probs = augmented_pi_probs_unflat.reshape(O * M, A * M) + + pi_ground = mem_aug_mdp.phi @ pi_probs # pi: (S * M, A) + occupancy = functional_get_occupancy(pi_ground, mem_aug_mdp) # eta: S * M + om_occupancy = occupancy @ mem_aug_mdp.phi # om_eta: O * M + + # Calculate our Q vals over A x O * M + p_pi_of_s_given_o = get_p_s_given_o(mem_aug_mdp.phi, occupancy) # P(SM|OM) + T_obs_obs, R_obs_obs = functional_create_td_model(p_pi_of_s_given_o, mem_aug_mdp) # T(O'M'|O,M,A); R(O,M,A) + td_model = MDP(T_obs_obs, R_obs_obs, mem_aug_mdp.p0 @ mem_aug_mdp.phi, gamma=mem_aug_mdp.gamma) + td_v_vals, td_q_vals = functional_solve_mdp(pi_probs, td_model) # q: (A, O * M) + + # expand over A * M + mem_probs = nn.softmax(mem_logits, axis=-1) # (A, O, M, M) + mem_probs_omam = jnp.moveaxis(mem_probs, 0, -2) # (O, M, A, M) + mem_probs_omam = mem_probs_omam.reshape(O * M, A, M) # (OM, A, M) + #(OM,A,M) #(A, OM)^T => (OM, A) => (OM, A, 1) + am_q_vals = mem_probs_omam * jnp.expand_dims(td_q_vals.T, -1) # (OM, A, M) + am_q_vals = am_q_vals.reshape(O * M, A * M) # (OM, AM) + + # Don't take gradients over eta or Q + weighted_am_q_vals = jnp.expand_dims(om_occupancy, -1) * am_q_vals + weighted_am_q_vals = lax.stop_gradient(weighted_am_q_vals) + return (weighted_am_q_vals * aug_pi_probs).sum(), (td_v_vals, td_q_vals) + +def mem_bellman_loss( mem_params: jnp.ndarray, pi: jnp.ndarray, pomdp: POMDP, # input non-static arrays value_type: str = 'q', error_type: str = 'l2', + lambda_0: float = 0, + lambda_1: float = 1., # NOT CURRENTLY USED! + residual: bool = False, alpha: float = 1., flip_count_prob: bool = False): mem_aug_pomdp = memory_cross_product(mem_params, pomdp) - loss, _, _ = magnitude_td_loss(pi, - mem_aug_pomdp, - value_type, - error_type, - alpha, - flip_count_prob=flip_count_prob) + loss, _, _ = bellman_loss(pi, + mem_aug_pomdp, + value_type, + error_type, + alpha, + lambda_=lambda_0, + residual=residual, + flip_count_prob=flip_count_prob) return loss -@partial(jit, static_argnames=['value_type', 'error_type', 'alpha', 'flip_count_prob']) -def magnitude_td_loss( +@partial(jit, static_argnames=['value_type', 'error_type', 'alpha', 'residual', 'flip_count_prob']) +def bellman_loss( pi: jnp.ndarray, pomdp: POMDP, # non-state args value_type: str = 'q', error_type: str = 'l2', alpha: float = 1., + lambda_: float = 0., + residual: bool = False, flip_count_prob: bool = False): # initialize static args - n_states = pomdp.state_space.n - # TODO: this is wrong? - _, mc_vals, td_vals, info = analytical_pe(pi, pomdp) + + # First, calculate our TD(0) Q-values + v_vals, q_vals, info = lstdq_lambda(pi, pomdp, lambda_=lambda_) + vals = {'v': v_vals, 'q': q_vals} assert value_type == 'q' - R_s_o = pomdp.R @ pomdp.phi # A x S x O - expanded_R_s_o = jnp.expand_dims(R_s_o, -1).repeat(pomdp.action_space.n, axis=-1) + + c_s = info['occupancy'] + # Make TD(0) model + p_pi_of_s_given_o = get_p_s_given_o(pomdp.phi, c_s) + T_aoo, R_aoo = functional_create_td_model(p_pi_of_s_given_o, pomdp) + + # Tensor for AxOxOxA (obs action to obs action) + T_pi_aooa = jnp.einsum('ijk,kl->ijkl', T_aoo, pi) + R_ao = R_aoo.sum(axis=-1) + + # Calculate the expected next value for each (a, o) pair + expected_next_V_given_ao = jnp.einsum('ijkl,kl->ij', T_pi_aooa, q_vals.T) # A x O + + # Our Bellman error + target = R_ao + pomdp.gamma * expected_next_V_given_ao + if not residual: + target = lax.stop_gradient(target) + diff = target - q_vals + + # R_s_o = pomdp.R @ pomdp.phi # A x S x O + + # expanded_R_s_o = R_s_o[..., None].repeat(pomdp.action_space.n, axis=-1) # A x S x O x A # repeat the Q-function over A x O # Multiply that with p(O', A' | s, a) and sum over O' and A' dimensions. # P(O' | s, a) = T @ phi, P(A', O' | s, a) = P(O' | s, a) * pi (over new dimension) - pr_o = pomdp.T @ pomdp.phi - pr_o_a = jnp.einsum('ijk,kl->ijkl', pr_o, pi) - expected_next_Q = jnp.einsum('ijkl,kl->ijkl', pr_o_a, td_vals['q'].T) - expanded_Q = jnp.expand_dims( - jnp.expand_dims(td_vals['q'].T, 0).repeat(pr_o_a.shape[1], axis=0), - 0).repeat(pr_o_a.shape[0], axis=0) - diff = expanded_R_s_o + pomdp.gamma * expected_next_Q - expanded_Q + # pr_o = pomdp.T @ pomdp.phi + # pr_o_a = jnp.einsum('ijk,kl->ijkl', pr_o, pi) + # expected_next_Q = jnp.einsum('ijkl,kl->ijkl', pr_o_a, q_vals.T) + # expanded_Q = jnp.expand_dims( + # jnp.expand_dims(q_vals.T, 0).repeat(pr_o_a.shape[1], axis=0), + # 0).repeat(pr_o_a.shape[0], axis=0) # Repeat over OxA -> O x A x O x A + # diff = expanded_R_s_o + pomdp.gamma * expected_next_Q - expanded_Q - c_s = info['occupancy'] - # set terminal counts to 0 - c_s = c_s.at[-2:].set(0) - count_s = c_s / c_s.sum() - if flip_count_prob: - count_s = nn.softmax(-count_s) + # set terminal counts to 0 + c_s = info['occupancy'] * (1 - pomdp.terminal_mask) + loss = weight_and_sum_discrep_loss(diff, c_s, pi, pomdp, + value_type=value_type, + error_type=error_type, + alpha=alpha, + flip_count_prob=flip_count_prob) - uniform_s = jnp.ones(n_states) / n_states + return loss, vals, vals - p_s = alpha * uniform_s + (1 - alpha) * count_s +def mem_tde_loss( + mem_params: jnp.ndarray, + pi: jnp.ndarray, + pomdp: POMDP, # input non-static arrays + value_type: str = 'q', + error_type: str = 'l2', + lambda_0: float = 0, + lambda_1: float = 1., # NOT CURRENTLY USED! + residual: bool = False, + alpha: float = 1., + flip_count_prob: bool = False): + mem_aug_pomdp = memory_cross_product(mem_params, pomdp) + loss, _, _ = mstd_err(pi, + mem_aug_pomdp, + # value_type, + error_type, + # alpha, + lambda_=lambda_0, + residual=residual, + # flip_count_prob=flip_count_prob + ) + return loss - pi_states = pomdp.phi @ pi - weight = (pi_states * p_s[:, None]).T - if value_type == 'v': - weight = weight.sum(axis=0) - weight = lax.stop_gradient(weight) +@partial(jit, static_argnames=['error_type', 'residual']) +def mstd_err( + pi: jnp.ndarray, + pomdp: POMDP, # non-state args + error_type: str = 'l2', + lambda_: float = 0., + residual: bool = False): # initialize static args + # First, calculate our TD(0) Q-values + v_vals, q_vals, info = lstdq_lambda(pi, pomdp, lambda_=lambda_) + vals = {'v': v_vals, 'q': q_vals} + # assert value_type == 'q' + + n_states = pomdp.base_mdp.state_space.n + n_obs = pomdp.observation_space.n + n_actions = pomdp.action_space.n + + # Project Q-values from observations to states + q_soa = q_vals.T[None, :, :].repeat(n_states, axis=0) + # q_sa = pomdp.phi @ q_vals.T + + # Calculate all "potential" next q-values by repeating at the front. + qp_soasoa = ( + q_soa[None, None, None, ...] + .repeat(n_states, axis=0) + .repeat(n_obs, axis=1) + .repeat(n_actions, axis=2) + ) + + # Calculate all current q-values, repeating at the back. + q_soasoa = ( + q_soa[..., None, None, None] + .repeat(n_states, axis=-3) + .repeat(n_obs, axis=-2) + .repeat(n_actions, axis=-1) + ) + + # Expanded and repeated reward tensor + R_sas = jnp.swapaxes(pomdp.base_mdp.R, 0, 1) + R_soasoa = ( + R_sas[:, None, :, :, None, None] + .repeat(n_obs, axis=1) + .repeat(n_obs, axis=-2) + .repeat(n_actions, axis=-1) + ) + + # Calculate targets (R + gamma * Q') and stop_grad it. + targets = R_soasoa + pomdp.gamma * qp_soasoa + if not residual: + targets = lax.stop_gradient(targets) + + # Compute errors + tde_soasoa = (targets - q_soasoa) if error_type == 'l2': - unweighted_err = (diff**2) + mag_tde_soasoa = (tde_soasoa**2) elif error_type == 'abs': - unweighted_err = jnp.abs(diff) + mag_tde_soasoa = jnp.abs(tde_soasoa) else: raise NotImplementedError(f"Error {error_type} not implemented yet in mem_loss fn.") - expanded_weight = jnp.expand_dims( - jnp.expand_dims(weight, -1).repeat(pr_o_a.shape[2], axis=-1), -1).repeat(pr_o_a.shape[-1], - axis=-1) - weighted_err = expanded_weight * unweighted_err - if value_type == 'q': - weighted_err = weighted_err.sum(axis=0) - - loss = weighted_err.sum() - - return loss, mc_vals, td_vals + # set terminal count to 0 and compute Pr(s) + c_s = info['occupancy'] * (1 - pomdp.terminal_mask) + pr_s = c_s / c_s.sum() + + # Retrieve Pr(o|s), Pr(s'|s,a) + phi_so = pomdp.phi + T_sas = jnp.swapaxes(pomdp.base_mdp.T, 0, 1) + + # Compute Pr(s,o,a,s',o',a') + pr_s_soasoa = pr_s[ :, None, None, None, None, None] # Pr(s) + phi_soasoa = phi_so[ :, :, None, None, None, None] # Pr(o|s) + pi_soasoa = pi[None, :, :, None, None, None] # Pr(a|o) + T_soasoa = T_sas[ :, None, :, :, None, None] # Pr(s'|s,a) + next_phi_soasoa = phi_so[None, None, None, :, :, None] # Pr(o'|s') + next_pi_soasoa = pi[None, None, None, None, :, :] # Pr(a'|o') + # Pr(s,o,a,s',o',a') = Pr(s) * Pr(o|s) * Pr(a|o) * Pr(s'|s,a) * Pr(o'|s') * Pr(a'|o') + pr_soasoa = ( + pr_s_soasoa * phi_soasoa * pi_soasoa * T_soasoa * next_phi_soasoa * next_pi_soasoa + ) + + # Reweight squared errors according to Pr(s,o,a,s',o',a') + weighted_sq_tde_soasoa = pr_soasoa * mag_tde_soasoa + + # Sum over all dimensions + loss = weighted_sq_tde_soasoa.sum() + return loss, vals, vals diff --git a/grl/utils/replaymemory.py b/grl/utils/replaymemory.py index a6cd8707..9fa548ed 100644 --- a/grl/utils/replaymemory.py +++ b/grl/utils/replaymemory.py @@ -1,10 +1,12 @@ from collections import defaultdict +from functools import partial import json import marshal import os import pickle import types from typing import Tuple, Union, List +import warnings import numpy as np from jax import random, jit @@ -340,9 +342,19 @@ def __init__(self, unpack_state=unpack_state) self.jitted_sampled_seq_idxes = jit(sample_seq_idxes, static_argnums=(0, 1, 2)) self.end = np.zeros_like(self.d, dtype=bool) + self.done_idxes = [] def push(self, batch: Batch): self.end[self._cursor] = batch.end + # if batch.done: + # new_done_idxes = [] + # for d in self.done_idxes: + # new_d = d - 1 + # if new_d > 0: + # new_done_idxes.append(new_d) + # + # self.done_idxes.append(self._cursor) + super(EpisodeBuffer, self).push(batch) def sample_eligible_idxes(self, batch_size: int, seq_len: int) -> np.ndarray: @@ -406,9 +418,47 @@ def sample_idx(self, sample_idx: np.ndarray, as_dict: bool = False): return Batch(**batch) + def sample_full_episodes(self, batch_size: int): + """ + Samples batch_size full episodes. + """ + @partial(jit, static_argnames='n') + def fixed_size_choice(rand_key: random.PRNGKey, n: int, elements: np.ndarray): + rand_key, choice_key = random.split(rand_key) + return random.choice(choice_key, elements, shape=(n, )), rand_key + + all_start_idxes = [] + if len(self) < self.capacity: + all_start_idxes.append(0) + # don't include last done + all_potential_starts = np.array(self.done_idxes[:-1], dtype=int) + 1 + + all_start_idxes = np.concatenate((all_start_idxes, all_potential_starts)) + + # figure out maximum episode length + episode_lengths = self.done_idxes - all_start_idxes + wrapped_mask = episode_lengths < 0 + episode_lengths[wrapped_mask] += 2 * all_start_idxes[wrapped_mask] + + if batch_size > len(all_start_idxes): + warnings.warn(f"batch_size {batch_size} < number of start indices. " + f"Creating a batch of {len(all_start_idxes)} episodes.") + start_indices = all_start_idxes + else: + sampled_start_idxes, self.rand_key = \ + fixed_size_choice(self.rand_key, batch_size, jnp.arange(len(all_start_idxes))) + start_indices = all_start_idxes[sampled_start_idxes] + + max_episode_length = episode_lengths.max() + + seq_range = jnp.arange(max_episode_length, dtype=int)[:, None] + sample_seq_idx = (start_indices + seq_range).T % self.capacity + + return self.sample_idx(sample_seq_idx) + def sample_k(self, batch_size: int, seq_len: int = 1, k: int = 1): batch = self.sample(batch_size * k, seq_len=seq_len, as_dict=True) for key, arr in batch.items(): - batch[key] = np.stack(np.split(arr, k, axis=0)) + batch[key] = np.stack(np.split(arr, k, axis=0), dtype=int) return Batch(**batch) diff --git a/grl/vi.py b/grl/vi.py index cde78ce4..588d1766 100644 --- a/grl/vi.py +++ b/grl/vi.py @@ -97,7 +97,7 @@ def policy_iteration_step(pi_params: jnp.ndarray, pomdp: POMDP, eps: float = 0.1 def po_policy_iteration(pomdp: POMDP, eps: float = 0.1) -> jnp.ndarray: """ Value iteration over observations. - :param pomdp: AbstractMDP + :param pomdp: POMDP :param tol: tolerance for error :return Value function for optimal policy. """ diff --git a/requirements.txt b/requirements.txt index 30d5455f..09ca691e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -numpy gymnasium gmpy2 jax[cpu] @@ -10,3 +9,6 @@ matplotlib tqdm git+https://github.com/camall3n/onager.git optuna==3.1.0 +mazelib-alt +popgym +numpy>=1.23.4 diff --git a/scripts/approx_lambda_discrep.py b/scripts/approx_lambda_discrep.py new file mode 100644 index 00000000..d27a76b7 --- /dev/null +++ b/scripts/approx_lambda_discrep.py @@ -0,0 +1,66 @@ + +import jax.numpy as jnp + +from grl.environment import load_pomdp +from grl.mdp import MDP + + +def B(mdp: MDP, pi: jnp.ndarray, vp: jnp.ndarray): + """ + Bellman operator. + """ + # we repeat values over S x A + repeated_vp = vp[None, ...] + repeated_vp = repeated_vp.repeat(mdp.R.shape[0] * mdp.R.shape[1], axis=0) + repeated_vp_reshaped = repeated_vp.reshape(mdp.R.shape[0], mdp.R.shape[1], -1) + + # g = r + gamma * v(s') + g = mdp.R + mdp.gamma * repeated_vp_reshaped + + # g * p(s' | s, a) + new_q = (mdp.T * g).sum(axis=-1) + + # sum_a pi * Q + new_v = (new_q * pi.T).sum(axis=0) + return new_v + + +if __name__ == "__main__": + spec_name = 'tmaze_5_two_thirds_up' + + pomdp, pi_dict = load_pomdp(spec_name, + corridor_length=5, + discount=0.9, + epsilon=0.1) + phi_t_phi = pomdp.phi.T @ pomdp.phi + phi_t_phi_inv_phi = jnp.linalg.inv(phi_t_phi) @ pomdp.phi.T + proj = pomdp.phi @ phi_t_phi_inv_phi + + pi_phi = pi_dict['Pi_phi'][0] + pi_s = pomdp.phi @ pi_phi + + def mc_projection(v: jnp.ndarray, steps: int): + o = v + for i in range(steps): + o = B(pomdp, pi_s, o) + return proj @ o + + def td_projection(v: jnp.ndarray, steps: int): + o = v + for i in range(steps): + o = proj @ B(pomdp, pi_s, o) + return o + + val_func = jnp.zeros(pomdp.state_space.n) + + for k in [0, 1, 2, 3, 4, 5, 10, 20]: + mc_val = mc_projection(val_func, k) + td_val = td_projection(val_func, k) + + diff = mc_val - td_val + mse = (diff ** 2).mean() + + print(f"Difference for {k} steps: {diff}") + print(f"MSE for {k} steps: {mse}") + + diff --git a/scripts/hyperparams/final_analytical_discrep.py b/scripts/hyperparams/final_analytical_discrep.py new file mode 100644 index 00000000..cef6e779 --- /dev/null +++ b/scripts/hyperparams/final_analytical_discrep.py @@ -0,0 +1,30 @@ +from pathlib import Path + +exp_name = Path(__file__).stem + +hparams = { + 'file_name': + f'runs_{exp_name}.txt', + 'args': [{ + 'spec': [ + 'tiger-alt-start', 'tmaze_5_two_thirds_up', 'example_7', '4x3.95', + 'cheese.95', 'network', 'shuttle.95', 'paint.95' + # 'hallway' + # 'bridge-repair', + ], + 'policy_optim_alg': 'policy_iter', + 'value_type': 'q', + 'error_type': 'l2', + 'alpha': 1., + 'objective': 'discrep', + # 'objective': ['discrep', 'obs_space'], + 'mi_steps': 20000, + 'pi_steps': 10000, + 'optimizer': 'adam', + 'lr': 0.01, + 'use_memory': 0, + 'n_mem_states': [2, 4, 8], + 'mi_iterations': 1, + 'seed': [2020 + i for i in range(30)], + }] +} diff --git a/scripts/hyperparams/final_analytical_kitchen_sinks.py b/scripts/hyperparams/final_analytical_kitchen_sinks.py new file mode 100644 index 00000000..1cc7e34b --- /dev/null +++ b/scripts/hyperparams/final_analytical_kitchen_sinks.py @@ -0,0 +1,31 @@ +from pathlib import Path + +exp_name = Path(__file__).stem + +hparams = { + 'file_name': + f'runs_{exp_name}.txt', + 'args': [{ + 'spec': [ + 'tiger-alt-start', 'tmaze_5_two_thirds_up', 'example_7', '4x3.95', + 'cheese.95', 'network', 'shuttle.95', 'paint.95' + # 'hallway' + # 'bridge-repair', + ], + 'policy_optim_alg': ['policy_iter', 'policy_grad'], + 'value_type': 'q', + 'error_type': 'l2', + 'kitchen_sink_policies': 400, + 'alpha': 1., + 'objective': 'obs_space', + # 'objective': ['discrep', 'obs_space'], + 'mi_steps': 20000, + 'pi_steps': 10000, + 'optimizer': 'adam', + 'lr': 0.01, + 'use_memory': 0, + 'n_mem_states': [2, 4, 8], + 'mi_iterations': 1, + 'seed': [2020 + i for i in range(30)], + }] +} diff --git a/scripts/hyperparams/final_analytical_pg.py b/scripts/hyperparams/final_analytical_pg.py new file mode 100644 index 00000000..766b9c7f --- /dev/null +++ b/scripts/hyperparams/final_analytical_pg.py @@ -0,0 +1,30 @@ +from pathlib import Path + +exp_name = Path(__file__).stem + +hparams = { + 'file_name': + f'runs_{exp_name}.txt', + 'args': [{ + 'spec': [ + 'tiger-alt-start', 'tmaze_5_two_thirds_up', 'example_7', '4x3.95', + 'cheese.95', 'network', 'shuttle.95', 'paint.95' + # 'hallway' + # 'bridge-repair', + ], + 'policy_optim_alg': 'policy_grad', + 'value_type': 'q', + 'error_type': 'l2', + 'alpha': 1., + 'objective': 'obs_space', + # 'objective': ['discrep', 'obs_space'], + 'mi_steps': 20000, + 'pi_steps': 10000, + 'optimizer': 'adam', + 'lr': 0.01, + 'use_memory': 0, + 'n_mem_states': [2, 4, 8], + 'mi_iterations': 1, + 'seed': [2020 + i for i in range(30)], + }] +} diff --git a/scripts/hyperparams/final_analytical_pg_mem.py b/scripts/hyperparams/final_analytical_pg_mem.py new file mode 100644 index 00000000..566c40f0 --- /dev/null +++ b/scripts/hyperparams/final_analytical_pg_mem.py @@ -0,0 +1,28 @@ +from pathlib import Path + +exp_name = Path(__file__).stem + +hparams = { + 'file_name': + f'runs_{exp_name}.txt', + 'args': [{ + 'spec': [ + 'tiger-alt-start', 'tmaze_5_two_thirds_up', 'example_7', '4x3.95', + 'cheese.95', 'network', 'shuttle.95', 'paint.95' + # 'hallway' + # 'bridge-repair', + ], + 'policy_optim_alg': 'policy_mem_grad', + 'value_type': 'q', + 'error_type': 'l2', + 'alpha': 1., + 'mi_steps': 0, + 'pi_steps': 20000, + 'optimizer': 'adam', + 'lr': 0.01, + 'use_memory': 0, + 'n_mem_states': [2, 4, 8], + 'mi_iterations': 1, + 'seed': [2020 + i for i in range(30)], + }] +} diff --git a/scripts/hyperparams/final_analytical_pg_mem_unrolled.py b/scripts/hyperparams/final_analytical_pg_mem_unrolled.py new file mode 100644 index 00000000..da1d521f --- /dev/null +++ b/scripts/hyperparams/final_analytical_pg_mem_unrolled.py @@ -0,0 +1,28 @@ +from pathlib import Path + +exp_name = Path(__file__).stem + +hparams = { + 'file_name': + f'runs_{exp_name}.txt', + 'args': [{ + 'spec': [ + 'tiger-alt-start', 'tmaze_5_two_thirds_up', 'example_7', '4x3.95', + 'cheese.95', 'network', 'shuttle.95', 'paint.95' + # 'hallway' + # 'bridge-repair', + ], + 'policy_optim_alg': 'policy_mem_grad_unrolled', + 'value_type': 'q', + 'error_type': 'l2', + 'alpha': 1., + 'mi_steps': 0, + 'pi_steps': 20000, + 'optimizer': 'adam', + 'lr': 0.01, + 'use_memory': 0, + 'n_mem_states': [2, 4, 8], + 'mi_iterations': 1, + 'seed': [2020 + i for i in range(30)], + }] +} diff --git a/scripts/hyperparams/final_discrep_kitchen_sinks_pg.py b/scripts/hyperparams/final_discrep_kitchen_sinks_pg.py new file mode 100644 index 00000000..fb508a52 --- /dev/null +++ b/scripts/hyperparams/final_discrep_kitchen_sinks_pg.py @@ -0,0 +1,31 @@ +from pathlib import Path + +exp_name = Path(__file__).stem + +hparams = { + 'file_name': + f'runs_{exp_name}.txt', + 'args': [{ + 'spec': [ + 'tiger-alt-start', 'tmaze_5_two_thirds_up', 'example_7', '4x3.95', + 'cheese.95', 'network', 'shuttle.95', 'paint.95' + # 'hallway' + # 'bridge-repair', + ], + 'policy_optim_alg': 'policy_grad', + 'value_type': 'q', + 'error_type': 'l2', + 'kitchen_sink_policies': 400, + 'alpha': 1., + # 'objective': 'obs_space', + 'objective': 'discrep', + 'mi_steps': 20000, + 'pi_steps': 10000, + 'optimizer': 'adam', + 'lr': 0.01, + 'use_memory': 0, + 'n_mem_states': [2, 4, 8], + 'mi_iterations': 1, + 'seed': [2020 + i for i in range(30)], + }] +} diff --git a/scripts/hyperparams/final_hallway_analytical_pg.py b/scripts/hyperparams/final_hallway_analytical_pg.py new file mode 100644 index 00000000..7a408ea7 --- /dev/null +++ b/scripts/hyperparams/final_hallway_analytical_pg.py @@ -0,0 +1,28 @@ +from pathlib import Path + +exp_name = Path(__file__).stem + +hparams = { + 'file_name': + f'runs_{exp_name}.txt', + 'args': [{ + 'spec': 'hallway', + 'policy_optim_alg': 'policy_grad', + 'value_type': 'q', + 'error_type': 'l2', + 'alpha': 1., + 'objective': 'obs_space', + # 'objective': ['discrep', 'obs_space'], + # 'mi_steps': 20000, + 'mi_steps': 5000, + # 'pi_steps': 10000, + 'pi_steps': 5000, + 'optimizer': 'adam', + 'lr': 0.01, + 'use_memory': 0, + 'n_mem_states': [2, 4, 8], + 'mi_iterations': 1, + 'platform': 'gpu', + 'seed': [2020 + i for i in range(30)], + }] +} diff --git a/scripts/hyperparams/hallway_kitchen_sinks_analytical.py b/scripts/hyperparams/hallway_kitchen_sinks_analytical.py new file mode 100644 index 00000000..fe54cd71 --- /dev/null +++ b/scripts/hyperparams/hallway_kitchen_sinks_analytical.py @@ -0,0 +1,26 @@ +from pathlib import Path + +exp_name = Path(__file__).stem + +hparams = { + 'file_name': + f'runs_{exp_name}.txt', + 'args': [{ + 'spec': 'hallway', + 'policy_optim_alg': 'policy_grad', + 'value_type': 'q', + 'error_type': 'l2', + 'kitchen_sink_policies': 400, + 'alpha': 1., + 'objective': 'obs_space', + # 'objective': ['discrep', 'obs_space'], + 'mi_steps': 5000, + 'pi_steps': 5000, + 'optimizer': 'adam', + 'lr': 0.01, + 'use_memory': 0, + 'n_mem_states': [4, 8], + 'mi_iterations': 1, + 'seed': [2020 + i for i in range(30)], + }] +} diff --git a/scripts/hyperparams/mem_bellman_pg.py b/scripts/hyperparams/mem_bellman_pg.py new file mode 100644 index 00000000..b7719cef --- /dev/null +++ b/scripts/hyperparams/mem_bellman_pg.py @@ -0,0 +1,32 @@ +from pathlib import Path + +exp_name = Path(__file__).stem + +hparams = { + 'file_name': + f'runs_{exp_name}.txt', + 'args': [{ + 'spec': [ + 'tiger-alt-start', 'tmaze_5_two_thirds_up', 'example_7', '4x3.95', + 'cheese.95', 'network', 'shuttle.95', 'paint.95' + # 'hallway' + # 'bridge-repair', + ], + 'policy_optim_alg': 'policy_grad', + 'value_type': 'q', + 'error_type': 'l2', + # 'alpha': [0., 1.], + 'alpha': 0., + # 'objective': 'obs_space', + 'objective': 'bellman', + 'residual': [True, False], + 'mi_steps': 20000, + 'pi_steps': 10000, + 'optimizer': 'adam', + 'lr': 0.01, + 'use_memory': 0, + 'n_mem_states': [2, 4, 8], + 'mi_iterations': 1, + 'seed': [2020 + i for i in range(30)], + }] +} diff --git a/scripts/hyperparams/mem_lambda_tde_pg.py b/scripts/hyperparams/mem_lambda_tde_pg.py new file mode 100644 index 00000000..043bc9ad --- /dev/null +++ b/scripts/hyperparams/mem_lambda_tde_pg.py @@ -0,0 +1,33 @@ +from pathlib import Path + +exp_name = Path(__file__).stem + +hparams = { + 'file_name': + f'runs_{exp_name}.txt', + 'args': [{ + 'spec': [ + 'tiger-alt-start', 'tmaze_5_two_thirds_up', 'example_7', '4x3.95', + 'cheese.95', 'network', 'shuttle.95', 'paint.95' + ], + 'policy_optim_alg': 'policy_grad', + 'value_type': 'q', + 'error_type': 'l2', + 'alpha': 0., + # 'alpha': 0., + 'objective': ['tde', 'discrep'], + 'residual': [True, False], + 'mi_steps': 20000, + 'pi_steps': 10000, + 'optimizer': 'adam', + 'lr': 0.01, + 'use_memory': 0, + 'n_mem_states': [2, 4, 8], + 'mi_iterations': 1, + 'seed': [2020 + i for i in range(30)], + }], + 'exclude': { + 'objective': ['discrep'], + 'residual': [True] + } +} diff --git a/scripts/hyperparams/mem_tde_kitchen_sinks_pg.py b/scripts/hyperparams/mem_tde_kitchen_sinks_pg.py new file mode 100644 index 00000000..5f022e18 --- /dev/null +++ b/scripts/hyperparams/mem_tde_kitchen_sinks_pg.py @@ -0,0 +1,32 @@ +from pathlib import Path + +exp_name = Path(__file__).stem + +hparams = { + 'file_name': + f'runs_{exp_name}.txt', + 'args': [{ + 'spec': [ + 'tiger-alt-start', 'tmaze_5_two_thirds_up', 'example_7', '4x3.95', + 'cheese.95', 'network', 'shuttle.95', 'paint.95' + # 'hallway' + # 'bridge-repair', + ], + 'policy_optim_alg': 'policy_grad', + 'value_type': 'q', + 'error_type': 'l2', + 'kitchen_sink_policies': 400, + # 'alpha': [0., 1.], + # 'alpha': 0., + 'objective': 'tde', + 'residual': [True, False], + 'mi_steps': 20000, + 'pi_steps': 10000, + 'optimizer': 'adam', + 'lr': 0.01, + 'use_memory': 0, + 'n_mem_states': [2, 4, 8], + 'mi_iterations': 1, + 'seed': [2020 + i for i in range(30)], + }] +} diff --git a/scripts/hyperparams/mem_tde_pg.py b/scripts/hyperparams/mem_tde_pg.py new file mode 100644 index 00000000..f3726fbc --- /dev/null +++ b/scripts/hyperparams/mem_tde_pg.py @@ -0,0 +1,31 @@ +from pathlib import Path + +exp_name = Path(__file__).stem + +hparams = { + 'file_name': + f'runs_{exp_name}.txt', + 'args': [{ + 'spec': [ + 'tiger-alt-start', 'tmaze_5_two_thirds_up', 'example_7', '4x3.95', + 'cheese.95', 'network', 'shuttle.95', 'paint.95' + # 'hallway' + # 'bridge-repair', + ], + 'policy_optim_alg': 'policy_grad', + 'value_type': 'q', + 'error_type': 'l2', + 'alpha': [0., 1.], + # 'alpha': 0., + 'objective': 'tde', + 'residual': [True, False], + 'mi_steps': 20000, + 'pi_steps': 10000, + 'optimizer': 'adam', + 'lr': 0.01, + 'use_memory': 0, + 'n_mem_states': [2, 4, 8], + 'mi_iterations': 1, + 'seed': [2020 + i for i in range(30)], + }] +} diff --git a/scripts/hyperparams/memoryless_analytical.py b/scripts/hyperparams/memoryless_analytical.py new file mode 100644 index 00000000..06d5706f --- /dev/null +++ b/scripts/hyperparams/memoryless_analytical.py @@ -0,0 +1,32 @@ +from pathlib import Path + +exp_name = Path(__file__).stem + +hparams = { + 'entry': '-m grl.run', + 'file_name': + f'runs_{exp_name}.txt', + 'args': [{ + 'spec': [ + 'tiger-alt-start', 'tmaze_5_two_thirds_up', 'example_7', '4x3.95', + 'cheese.95', 'network', 'shuttle.95', 'paint.95' + # 'hallway' + # 'bridge-repair', + ], + 'policy_optim_alg': 'policy_iter', + 'value_type': 'q', + 'error_type': 'l2', + 'alpha': 1., + 'objective': 'obs_space', + # 'objective': ['discrep', 'obs_space'], + # 'mi_steps': 20000, + 'mi_steps': 1, + 'pi_steps': 10000, + 'optimizer': 'adam', + 'lr': 0.0, + 'use_memory': 0, + 'n_mem_states': [2, 4, 8], + 'mi_iterations': 1, + 'seed': [2020 + i for i in range(30)], + }] +} diff --git a/scripts/hyperparams/memoryless_hallway_analytical.py b/scripts/hyperparams/memoryless_hallway_analytical.py new file mode 100644 index 00000000..4b9de34c --- /dev/null +++ b/scripts/hyperparams/memoryless_hallway_analytical.py @@ -0,0 +1,29 @@ +from pathlib import Path + +exp_name = Path(__file__).stem + +hparams = { + 'entry': '-m grl.run', + 'file_name': + f'runs_{exp_name}.txt', + 'args': [{ + 'spec': 'hallway', + 'policy_optim_alg': 'policy_iter', + 'value_type': 'q', + 'error_type': 'l2', + 'alpha': 1., + 'objective': 'obs_space', + # 'objective': ['discrep', 'obs_space'], + # 'mi_steps': 20000, + 'mi_steps': 1, + # 'pi_steps': 10000, + 'pi_steps': 5000, + 'optimizer': 'adam', + 'lr': 0.0, + 'use_memory': 0, + 'n_mem_states': [2, 4, 8], + 'mi_iterations': 1, + 'platform': 'gpu', + 'seed': [2020 + i for i in range(30)], + }] +} diff --git a/scripts/hyperparams/memoryless_kitchen_sinks.py b/scripts/hyperparams/memoryless_kitchen_sinks.py new file mode 100644 index 00000000..8d4383fe --- /dev/null +++ b/scripts/hyperparams/memoryless_kitchen_sinks.py @@ -0,0 +1,31 @@ +from pathlib import Path + +exp_name = Path(__file__).stem + +hparams = { + 'file_name': + f'runs_{exp_name}.txt', + 'args': [{ + 'spec': [ + 'tiger-alt-start', 'tmaze_5_two_thirds_up', 'example_7', '4x3.95', + 'cheese.95', 'network', 'shuttle.95', 'paint.95' + # 'hallway' + # 'bridge-repair', + ], + 'policy_optim_alg': ['policy_iter', 'policy_grad'], + 'value_type': 'q', + 'error_type': 'l2', + 'kitchen_sink_policies': 400, + 'alpha': 1., + 'objective': 'obs_space', + # 'objective': ['discrep', 'obs_space'], + 'mi_steps': 0, + 'pi_steps': 10000, + 'optimizer': 'adam', + 'lr': 0.01, + 'use_memory': ['random_discrete', 'random_uniform'], + 'n_mem_states': [2, 4, 8], + 'mi_iterations': 1, + 'seed': [2020 + i for i in range(30)], + }] +} diff --git a/scripts/hyperparams/popgym_minesweeper_mc.py b/scripts/hyperparams/popgym_minesweeper_mc.py new file mode 100644 index 00000000..002b85e9 --- /dev/null +++ b/scripts/hyperparams/popgym_minesweeper_mc.py @@ -0,0 +1,63 @@ +from pathlib import Path +from popgym.envs import ALL_EASY + +exp_name = Path(__file__).stem + +hparams = { + 'file_name': f'runs_{exp_name}.txt', + 'entry': '-m grl.run_sample_based', + 'args': [ + { + # Env + 'spec': "popgym-MineSweeperEasy-v0", + 'no_gamma_terminal': True, + 'max_episode_steps': 1000, + 'gamma': 0.99, + + # Agent + 'algo': 'multihead_rnn', + 'epsilon': 0.1, + 'arch': 'gru', + 'lr': [10**-i for i in range(3, 7)], + # 'lr': 1e-4, + 'optimizer': 'adam', + 'feature_encoding': 'none', + + # RNN + 'hidden_size': 256, + 'value_head_layers': 0, + 'trunc': -1, + 'action_cond': 'none', + + # Multihead RNN/Lambda discrep + 'multihead_action_mode': ['mc'], + 'multihead_loss_mode': ['both', 'mc'], + # 'multihead_loss_mode': 'both', + 'multihead_lambda_coeff': [0., 1.], + # 'multihead_lambda_coeff': 0., + # 'normalize_rewards': True, + + # Replay + 'replay_size': -1, + # 'batch_size': 16, + + # Logging and Checkpointing + 'offline_eval_freq': int(1e4), + 'offline_eval_episodes': 10, + 'offline_eval_epsilon': None, # Defaults to epsilon + 'checkpoint_freq': -1, # only save last agent + + # Experiment + 'total_steps': int(15e6), + 'seed': [2020 + i for i in range(3)], + # 'seed': 2020, + 'study_name': exp_name + }, + ], + # exclusion criteria. If any of the runs match any of the + # cross-product of all the values in the dictionary, skip + 'exclude': { + 'multihead_loss_mode': ['mc'], + 'multihead_lambda_coeff': [-1, 1] + } +} diff --git a/scripts/hyperparams/popgym_minesweeper_td.py b/scripts/hyperparams/popgym_minesweeper_td.py new file mode 100644 index 00000000..71d9edee --- /dev/null +++ b/scripts/hyperparams/popgym_minesweeper_td.py @@ -0,0 +1,63 @@ +from pathlib import Path +from popgym.envs import ALL_EASY + +exp_name = Path(__file__).stem + +hparams = { + 'file_name': f'runs_{exp_name}.txt', + 'entry': '-m grl.run_sample_based', + 'args': [ + { + # Env + 'spec': "popgym-MineSweeperEasy-v0", + 'no_gamma_terminal': True, + 'max_episode_steps': 1000, + 'gamma': 0.99, + + # Agent + 'algo': 'multihead_rnn', + 'epsilon': 0.1, + 'arch': 'gru', + 'lr': [10**-i for i in range(3, 7)], + # 'lr': 1e-4, + 'optimizer': 'adam', + 'feature_encoding': 'none', + + # RNN + 'hidden_size': 256, + 'value_head_layers': 0, + 'trunc': -1, + 'action_cond': 'none', + + # Multihead RNN/Lambda discrep + 'multihead_action_mode': ['td'], + 'multihead_loss_mode': ['both', 'td'], + # 'multihead_loss_mode': 'both', + 'multihead_lambda_coeff': [0., 1.], + # 'multihead_lambda_coeff': 0., + # 'normalize_rewards': True, + + # Replay + 'replay_size': -1, + # 'batch_size': 16, + + # Logging and Checkpointing + 'offline_eval_freq': int(1e4), + 'offline_eval_episodes': 10, + 'offline_eval_epsilon': None, # Defaults to epsilon + 'checkpoint_freq': -1, # only save last agent + + # Experiment + 'total_steps': int(15e6), + 'seed': [2020 + i for i in range(3)], + # 'seed': 2020, + 'study_name': exp_name + }, + ], + # exclusion criteria. If any of the runs match any of the + # cross-product of all the values in the dictionary, skip + 'exclude': { + 'multihead_loss_mode': ['td'], + 'multihead_lambda_coeff': [-1, 1] + } +} diff --git a/scripts/hyperparams/popgym_sweep_mc.py b/scripts/hyperparams/popgym_sweep_mc.py new file mode 100644 index 00000000..7a291b1c --- /dev/null +++ b/scripts/hyperparams/popgym_sweep_mc.py @@ -0,0 +1,63 @@ +from pathlib import Path +from popgym.envs import ALL_EASY + +exp_name = Path(__file__).stem + +hparams = { + 'file_name': f'runs_{exp_name}.txt', + 'entry': '-m grl.run_sample_based', + 'args': [ + { + # Env + 'spec': [v['id'] for k, v in ALL_EASY.items()], + 'no_gamma_terminal': True, + 'max_episode_steps': 1000, + 'gamma': 0.99, + + # Agent + 'algo': 'multihead_rnn', + 'epsilon': 0.1, + 'arch': 'gru', + 'lr': [10**-i for i in range(3, 7)], + # 'lr': 1e-4, + 'optimizer': 'adam', + 'feature_encoding': 'none', + + # RNN + 'hidden_size': 256, + 'value_head_layers': 0, + 'trunc': -1, + 'action_cond': 'none', + + # Multihead RNN/Lambda discrep + 'multihead_action_mode': ['mc'], + 'multihead_loss_mode': ['both', 'mc'], + # 'multihead_loss_mode': 'both', + 'multihead_lambda_coeff': [0., 1.], + # 'multihead_lambda_coeff': 0., + # 'normalize_rewards': True, + + # Replay + 'replay_size': -1, + # 'batch_size': 16, + + # Logging and Checkpointing + 'offline_eval_freq': int(5e4), + 'offline_eval_episodes': 10, + 'offline_eval_epsilon': None, # Defaults to epsilon + 'checkpoint_freq': -1, # only save last agent + + # Experiment + 'total_steps': int(15e6), + 'seed': [2020 + i for i in range(3)], + # 'seed': 2020, + 'study_name': exp_name + }, + ], + # exclusion criteria. If any of the runs match any of the + # cross-product of all the values in the dictionary, skip + 'exclude': { + 'multihead_loss_mode': ['mc'], + 'multihead_lambda_coeff': [-1, 1] + } +} diff --git a/scripts/hyperparams/popgym_sweep_mc_test.py b/scripts/hyperparams/popgym_sweep_mc_test.py new file mode 100644 index 00000000..fc69ff85 --- /dev/null +++ b/scripts/hyperparams/popgym_sweep_mc_test.py @@ -0,0 +1,63 @@ +from pathlib import Path +from popgym.envs import ALL_EASY + +exp_name = Path(__file__).stem + +hparams = { + 'file_name': f'runs_{exp_name}.txt', + 'entry': '-m grl.run_sample_based', + 'args': [ + { + # Env + 'spec': [v['id'] for k, v in ALL_EASY.items()], + 'no_gamma_terminal': True, + 'max_episode_steps': 1000, + 'gamma': 0.99, + + # Agent + 'algo': 'multihead_rnn', + 'epsilon': 0.1, + 'arch': 'gru', + # 'lr': [10**-i for i in range(3, 7)], + 'lr': 1e-4, + 'optimizer': 'adam', + 'feature_encoding': 'none', + + # RNN + 'hidden_size': 256, + 'value_head_layers': 0, + 'trunc': -1, + 'action_cond': 'none', + + # Multihead RNN/Lambda discrep + 'multihead_action_mode': ['mc'], + # 'multihead_loss_mode': ['both', 'mc'], + 'multihead_loss_mode': 'both', + # 'multihead_lambda_coeff': [0., 1.], + 'multihead_lambda_coeff': 0., + # 'normalize_rewards': True, + + # Replay + 'replay_size': -1, + # 'batch_size': 16, + + # Logging and Checkpointing + 'offline_eval_freq': int(1e4), + 'offline_eval_episodes': 10, + 'offline_eval_epsilon': None, # Defaults to epsilon + 'checkpoint_freq': -1, # only save last agent + + # Experiment + 'total_steps': int(15e6), + # 'seed': [2020 + i for i in range(3)], + 'seed': 2020, + 'study_name': exp_name + }, + ], + # exclusion criteria. If any of the runs match any of the + # cross-product of all the values in the dictionary, skip + 'exclude': { + 'multihead_loss_mode': ['mc'], + 'multihead_lambda_coeff': [-1, 1] + } +} diff --git a/scripts/hyperparams/popgym_sweep_td.py b/scripts/hyperparams/popgym_sweep_td.py new file mode 100644 index 00000000..905b1537 --- /dev/null +++ b/scripts/hyperparams/popgym_sweep_td.py @@ -0,0 +1,63 @@ +from pathlib import Path +from popgym.envs import ALL_EASY + +exp_name = Path(__file__).stem + +hparams = { + 'file_name': f'runs_{exp_name}.txt', + 'entry': '-m grl.run_sample_based', + 'args': [ + { + # Env + 'spec': [v['id'] for k, v in ALL_EASY.items()], + 'no_gamma_terminal': True, + 'max_episode_steps': 1000, + 'gamma': 0.99, + + # Agent + 'algo': 'multihead_rnn', + 'epsilon': 0.1, + 'arch': 'gru', + 'lr': [10**-i for i in range(3, 7)], + # 'lr': 1e-4, + 'optimizer': 'adam', + 'feature_encoding': 'none', + + # RNN + 'hidden_size': 256, + 'value_head_layers': 0, + 'trunc': -1, + 'action_cond': 'none', + + # Multihead RNN/Lambda discrep + 'multihead_action_mode': ['td'], + 'multihead_loss_mode': ['both', 'td'], + # 'multihead_loss_mode': 'both', + 'multihead_lambda_coeff': [0., 1.], + # 'multihead_lambda_coeff': 0., + # 'normalize_rewards': True, + + # Replay + 'replay_size': -1, + # 'batch_size': 16, + + # Logging and Checkpointing + 'offline_eval_freq': int(5e4), + 'offline_eval_episodes': 10, + 'offline_eval_epsilon': None, # Defaults to epsilon + 'checkpoint_freq': -1, # only save last agent + + # Experiment + 'total_steps': int(15e6), + 'seed': [2020 + i for i in range(3)], + # 'seed': 2020, + 'study_name': exp_name + }, + ], + # exclusion criteria. If any of the runs match any of the + # cross-product of all the values in the dictionary, skip + 'exclude': { + 'multihead_loss_mode': ['td'], + 'multihead_lambda_coeff': [-1, 1] + } +} diff --git a/scripts/hyperparams/random_discrete_analytical.py b/scripts/hyperparams/random_discrete_analytical.py new file mode 100644 index 00000000..06857a69 --- /dev/null +++ b/scripts/hyperparams/random_discrete_analytical.py @@ -0,0 +1,32 @@ +from pathlib import Path + +exp_name = Path(__file__).stem + +hparams = { + 'entry': '-m grl.run', + 'file_name': + f'runs_{exp_name}.txt', + 'args': [{ + 'spec': [ + 'tiger-alt-start', 'network', 'tmaze_5_two_thirds_up', 'example_7', '4x3.95', + 'cheese.95', 'network', 'shuttle.95', 'paint.95' + # 'hallway' + # 'bridge-repair', + ], + 'policy_optim_alg': 'policy_iter', + 'value_type': 'q', + 'error_type': 'l2', + 'alpha': 1., + 'objective': 'obs_space', + # 'objective': ['discrep', 'obs_space'], + # 'mi_steps': 20000, + 'mi_steps': 1, + 'pi_steps': 10000, + 'optimizer': 'adam', + 'lr': 0.0, + 'use_memory': 'random_discrete', + 'n_mem_states': [2, 4, 8], + 'mi_iterations': 1, + 'seed': [2020 + i for i in range(30)], + }] +} diff --git a/scripts/hyperparams/random_uniform_analytical.py b/scripts/hyperparams/random_uniform_analytical.py new file mode 100644 index 00000000..2c6bd31d --- /dev/null +++ b/scripts/hyperparams/random_uniform_analytical.py @@ -0,0 +1,32 @@ +from pathlib import Path + +exp_name = Path(__file__).stem + +hparams = { + 'entry': '-m grl.run', + 'file_name': + f'runs_{exp_name}.txt', + 'args': [{ + 'spec': [ + 'tiger-alt-start', 'network', 'tmaze_5_two_thirds_up', 'example_7', '4x3.95', + 'cheese.95', 'network', 'shuttle.95', 'paint.95' + # 'hallway' + # 'bridge-repair', + ], + 'policy_optim_alg': 'policy_iter', + 'value_type': 'q', + 'error_type': 'l2', + 'alpha': 1., + 'objective': 'obs_space', + # 'objective': ['discrep', 'obs_space'], + # 'mi_steps': 20000, + 'mi_steps': 1, + 'pi_steps': 10000, + 'optimizer': 'adam', + 'lr': 0.0, + 'use_memory': 'random_uniform', + 'n_mem_states': [2, 4, 8], + 'mi_iterations': 1, + 'seed': [2020 + i for i in range(30)], + }] +} diff --git a/scripts/hyperparams/rnn_reruns_mc.py b/scripts/hyperparams/rnn_reruns_mc.py index 04a6464c..4a780526 100644 --- a/scripts/hyperparams/rnn_reruns_mc.py +++ b/scripts/hyperparams/rnn_reruns_mc.py @@ -16,6 +16,7 @@ ], 'no_gamma_terminal': False, 'max_episode_steps': 1000, + 'feature_encoding': 'one_hot', # Agent 'algo': 'multihead_rnn', diff --git a/scripts/hyperparams/rnn_reruns_sweep_mc.py b/scripts/hyperparams/rnn_reruns_sweep_mc.py index c8b2aeff..3360024a 100644 --- a/scripts/hyperparams/rnn_reruns_sweep_mc.py +++ b/scripts/hyperparams/rnn_reruns_sweep_mc.py @@ -14,6 +14,7 @@ ], 'no_gamma_terminal': False, 'max_episode_steps': 1000, + 'feature_encoding': 'one_hot', # Agent 'algo': 'multihead_rnn', diff --git a/scripts/hyperparams/rnn_reruns_sweep_td.py b/scripts/hyperparams/rnn_reruns_sweep_td.py index 8f8405af..5a7b1382 100644 --- a/scripts/hyperparams/rnn_reruns_sweep_td.py +++ b/scripts/hyperparams/rnn_reruns_sweep_td.py @@ -14,6 +14,7 @@ ], 'no_gamma_terminal': False, 'max_episode_steps': 1000, + 'feature_encoding': 'one_hot', # Agent 'algo': 'multihead_rnn', diff --git a/scripts/hyperparams/rnn_reruns_td.py b/scripts/hyperparams/rnn_reruns_td.py index 41259d56..46ecac91 100644 --- a/scripts/hyperparams/rnn_reruns_td.py +++ b/scripts/hyperparams/rnn_reruns_td.py @@ -16,6 +16,7 @@ ], 'no_gamma_terminal': False, 'max_episode_steps': 1000, + 'feature_encoding': 'one_hot', # Agent 'algo': 'multihead_rnn', diff --git a/scripts/hyperparams/rnn_split.py b/scripts/hyperparams/rnn_split.py index 92f1b334..147015c6 100644 --- a/scripts/hyperparams/rnn_split.py +++ b/scripts/hyperparams/rnn_split.py @@ -16,6 +16,7 @@ ], 'no_gamma_terminal': False, 'max_episode_steps': 1000, + 'feature_encoding': 'one_hot', # Agent 'algo': 'multihead_rnn', diff --git a/scripts/launching/onager_gpu_slurm_launch.sh b/scripts/launching/onager_gpu_slurm_launch.sh index aa2041cd..787859b8 100755 --- a/scripts/launching/onager_gpu_slurm_launch.sh +++ b/scripts/launching/onager_gpu_slurm_launch.sh @@ -8,6 +8,6 @@ onager launch \ --duration 0-06:00:00 \ --venv venv \ --gpus 1 \ - --account 3090-gcondo + --partition 3090-gcondo # --tasks-per-node 4 \ diff --git a/scripts/launching/run_cpu_locally.sh b/scripts/launching/run_cpu_locally.sh index 1754aa2e..ea944cfa 100755 --- a/scripts/launching/run_cpu_locally.sh +++ b/scripts/launching/run_cpu_locally.sh @@ -8,5 +8,5 @@ source venv/bin/activate #TO_RUN=$(sed -n "101,800p" scripts/runs/runs_rnn_reruns_sweep_mc.txt) -parallel --eta -u --jobs $N_JOBS < 'scripts/runs/runs_rnn_reruns_sweep_mc.txt' +parallel --eta -u --jobs $N_JOBS < 'scripts/runs/runs_final_analytical_discrep.txt' #parallel --eta -u < "$TO_RUN" diff --git a/scripts/launching/run_gpu_locally.sh b/scripts/launching/run_gpu_locally.sh index 51e8d147..31b13db6 100755 --- a/scripts/launching/run_gpu_locally.sh +++ b/scripts/launching/run_gpu_locally.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -export XLA_PYTHON_CLIENT_MEM_FRACTION=0.8 +export XLA_PYTHON_CLIENT_MEM_FRACTION=0.22 cd ../../ source venv/bin/activate -parallel --eta --jobs 1 -u < 'scripts/runs/runs_tiger_grid_mi_pi_obs_space.txt' +parallel --eta --jobs 4 -u < 'scripts/runs/runs_memoryless_hallway_analytical.txt' diff --git a/scripts/mem_traj_logging.py b/scripts/mem_traj_logging.py new file mode 100644 index 00000000..593e5a1a --- /dev/null +++ b/scripts/mem_traj_logging.py @@ -0,0 +1,107 @@ +from collections import namedtuple +from functools import partial +from pathlib import Path + +import numpy as np +from tqdm import trange + +from grl.memory import memory_cross_product +from grl.utils.file_system import load_info +from grl.environment import load_pomdp +from definitions import ROOT_DIR + +Step = namedtuple('Step', ['state', 'mem', 'obs', 'action', 'reward']) + +def act(policy: np.ndarray, obs: int, mem: int, + n_mem: int = 4): + idx = (obs * n_mem) + mem + return np.random.choice(np.arange(policy.shape[-1]), p=policy[idx]) + +def memory_step(memory: np.ndarray, obs: int, mem: int, action: int): + return np.random.choice(np.arange(memory.shape[-1]), p=memory[action, obs, mem]) + +def get_mappings(env_name: str = 'tiger-alt-start'): + if env_name == 'tiger-alt-start': + action_mapping = ['listen', 'open-left', 'open-right'] + obs_mapping = ['init', 'tiger-left', 'tiger-right', 'terminal'] + else: + raise NotImplementedError + return obs_mapping, action_mapping + +if __name__ == "__main__": + seed = 2021 + np.random.seed(seed) + rand_state = np.random.RandomState(seed) + episodes = 10000 + + env_name = 'tiger-alt-start' + env, _ = load_pomdp(env_name, rand_key=rand_state) + + agent_fpath = Path(ROOT_DIR, 'results', 'agent', + 'tiger-alt-start_seed(2020)_time(20231018-134915)_4a3ffb1323a55af44906b2082a787957.pkl.npy') + agent = load_info(agent_fpath) + mem_env = memory_cross_product(agent.mem_params, env) + + n_mem = agent.memory.shape[-1] + def convert_m_obs(m_obs: int): + obs = m_obs // n_mem + mem = m_obs % n_mem + return obs, mem + + def convert_to_m_state(state: int, mem: int): + return (state * n_mem) + mem + + # okay this is a super weird bug. when converting from jax.numpy.array to np.array, + # we get a roundoff error. + + # we greedify here for testing. + mem_env.phi = np.array(mem_env.phi) + mem_env.T = np.array(mem_env.T) + mem_env.p0 = np.array(mem_env.p0) + + act = partial(act, n_mem=agent.memory.shape[-1]) + + obs_map, action_map = get_mappings(env_name) + + memory = np.array(agent.memory) + pi = np.array(agent.policy) + + discounted_episode_returns = [] + + for ep in trange(episodes): + episode_traj = [] + episode_rewards = [] + + done = False + mem = 0 + + obs, _ = env.reset() + state = env.current_state + + while not done: + action = act(pi, obs, mem) + + next_obs, reward, done, _, info = env.step(action) + + episode_rewards.append(reward) + + episode_traj.append(Step(obs=obs_map[obs], + mem=mem, + action=action_map[action], + reward=reward, + state=state)) + + + state = env.current_state + + next_mem = memory_step(memory, obs, mem, action) + obs = next_obs + mem = next_mem + + episode_rewards = np.array(episode_rewards) + episode_discounts = env.gamma ** np.arange(len(episode_rewards)) + discounted_return = np.dot(episode_rewards, episode_discounts) + discounted_episode_returns.append(discounted_return) + + + print(f"average discounted returns: {np.mean(discounted_episode_returns)}") diff --git a/scripts/onager_write_jobs.py b/scripts/onager_write_jobs.py index 00236599..be7c4d12 100644 --- a/scripts/onager_write_jobs.py +++ b/scripts/onager_write_jobs.py @@ -5,18 +5,12 @@ import os import numpy as np import argparse -import importlib.util from typing import List from pathlib import Path -from definitions import ROOT_DIR +from grl.utils.file_system import import_module_to_var -def import_module_to_hparam(hparam_path: Path) -> dict: - spec = importlib.util.spec_from_file_location("hparam", hparam_path) - hparam_module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(hparam_module) - hparams = hparam_module.hparams - return hparams +from definitions import ROOT_DIR def generate_onager_runs(run_dicts: List[dict], experiment_name: str, @@ -55,6 +49,8 @@ def generate_onager_runs(run_dicts: List[dict], if all([isinstance(el, bool) for el in v]): arg_string = f"+flag --{k}" else: + assert not isinstance(v[0], list), "No functionality for passing in a list of lists." \ + "Please pass in a list of space-separated strings instead." arg_string = f"+arg --{k} {' '.join(map(str, v))}" arg_list.append(arg_string) @@ -75,15 +71,16 @@ def generate_onager_runs(run_dicts: List[dict], os.chdir(ROOT_DIR) os.system(prelaunch_string) + if __name__ == "__main__": parser = argparse.ArgumentParser() + parser.add_argument('hyperparam_file', type=str) parser.add_argument('--study_name', default=None, type=str) - parser.add_argument('--hparam', default='', type=str) parser.add_argument('--local', action='store_true') args = parser.parse_args() - hparam_path = Path(ROOT_DIR, 'scripts', 'hyperparams', args.hparam + ".py") - hparams = import_module_to_hparam(hparam_path) + hparam_path = Path(args.hyperparam_file).resolve() + hparams = import_module_to_var(hparam_path, 'hparams') main_fname = '-m grl.run' if 'entry' in hparams: @@ -93,7 +90,8 @@ def generate_onager_runs(run_dicts: List[dict], if 'exclude' in hparams: exclude = hparams['exclude'] - exp_name = args.hparam + exp_name = hparam_path.stem if args.study_name is not None: exp_name = args.study_name + generate_onager_runs(hparams['args'], exp_name, main_fname=main_fname, exclude=exclude) diff --git a/scripts/parse_alpha.py b/scripts/parse_alpha.py index 1455db44..d4013b2c 100644 --- a/scripts/parse_alpha.py +++ b/scripts/parse_alpha.py @@ -7,7 +7,21 @@ import numpy as np from pathlib import Path -from grl.environment.pomdp_file import POMDPFile +from grl.environment import load_pomdp + + +def get_max_val(b0: np.ndarray, coeffs: np.ndarray): + max_start_val = -float('inf') + max_start_idx = 0 + for i, coeff in enumerate(coeffs): + piece_start_val = np.dot(coeff, b0) + if piece_start_val > max_start_val: + max_start_val = piece_start_val + max_start_idx = i + + return max_start_val, max_start_idx + + if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('filename', type=str, help='name of .alpha file to parse') @@ -36,22 +50,33 @@ coeffs = np.array(coeffs) pomdp_name = '-'.join(filename.stem.split('-')[:-1]) - pomdp = POMDPFile(filename.parent / f"{pomdp_name}.POMDP") - p0 = pomdp.start - max_start_val = -float('inf') - max_start_idx = 0 - for i, coeff in enumerate(coeffs): - piece_start_val = np.dot(coeff, p0) - if piece_start_val > max_start_val: - max_start_val = piece_start_val - max_start_idx = i + pomdp, _ = load_pomdp(pomdp_name) + all_start_beliefs = [] + b0 = pomdp.p0 + + start_val = 0 + for s, p0_s in enumerate(pomdp.p0): + if p0_s > 0: + obs_p = pomdp.phi[s] + for o, p0_o in enumerate(obs_p): + w = np.zeros_like(b0) + for new_s in range(pomdp.state_space.n): + w[new_s] = pomdp.phi[new_s, o] * b0[new_s] + + if np.all(w == 0): + continue + initial_belief = w / np.sum(w) + all_start_beliefs.append((initial_belief, s, o)) + max_start_val, max_start_idx = get_max_val(initial_belief, coeffs) + start_val += p0_s * p0_o * max_start_val + - res = {'p0': p0, 'actions': actions, 'coeffs': coeffs, 'max_start_idx': max_start_idx} + res = {'all_start_beliefs': all_start_beliefs, 'start_val': start_val, 'coeffs': coeffs} res_file = filename.parent / f"{pomdp_name}-pomdp-solver-results.npy" - print(f"Maximum start val for pomdp soln.: {max_start_val}") + print(f"Maximum start val for pomdp soln.: {start_val}") print(f"Saving results to {res_file}") np.save(res_file, res) diff --git a/scripts/plotting/__init__.py b/scripts/plotting/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scripts/plotting/cheese_memory.ipynb b/scripts/plotting/cheese_memory.ipynb deleted file mode 100644 index b12cc2d6..00000000 --- a/scripts/plotting/cheese_memory.ipynb +++ /dev/null @@ -1,566 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 10, - "id": "94525b55", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import jax.numpy as jnp\n", - "import matplotlib.pyplot as plt\n", - "import matplotlib.cm as cm\n", - "from jax.nn import softmax\n", - "from jax.config import config\n", - "from pathlib import Path\n", - "from collections import namedtuple\n", - "\n", - "config.update('jax_platform_name', 'cpu')\n", - "np.set_printoptions(precision=4)\n", - "plt.rcParams['axes.facecolor'] = 'white'\n", - "plt.rcParams.update({'font.size': 22})\n", - "\n", - "from grl.utils import load_info\n", - "from grl.utils.mdp import get_perf\n", - "from definitions import ROOT_DIR\n", - "np.set_printoptions(precision=3)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "f7b379dc", - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "results_dir = Path(ROOT_DIR, 'results')\n" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "89877f07", - "metadata": {}, - "outputs": [], - "source": [ - "all_results = {}\n", - "\n", - "# for results_path in results_dir.iterdir():\n", - "# if 'tmaze' not in results_path.stem:\n", - "# continue\n", - "# info = load_info(results_path)\n", - "# results_paths = [res_path for res_path in results_dir.iterdir() if 'cheese' in res_path.stem]\n", - "# results_path = results_paths[0]\n", - "results_path = results_dir / 'cheese.95_mi_pi(pi)_miit(1)_s(2020)_Thu Feb 2 20:50:49 2023.npy'" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "07b7093b", - "metadata": {}, - "outputs": [], - "source": [ - "agent_path = results_path.parent / 'agents' / f\"{results_path.stem}.pkl.npy\"" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "35bfc81c", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "NORTH\n", - "[[[[8.947e-01 1.053e-01]\n", - " [9.436e-02 9.056e-01]]\n", - "\n", - " [[9.946e-01 5.388e-03]\n", - " [5.859e-02 9.414e-01]]\n", - "\n", - " [[9.340e-01 6.596e-02]\n", - " [1.060e-01 8.940e-01]]\n", - "\n", - " [[9.020e-01 9.800e-02]\n", - " [9.971e-02 9.003e-01]]\n", - "\n", - " [[9.420e-01 5.805e-02]\n", - " [6.018e-02 9.398e-01]]\n", - "\n", - " [[7.470e-04 9.993e-01]\n", - " [3.918e-05 1.000e+00]]\n", - "\n", - " [[9.996e-01 4.149e-04]\n", - " [9.978e-01 2.189e-03]]]\n", - "\n", - "\n", - " [[[9.549e-04 9.990e-01]\n", - " [7.301e-04 9.993e-01]]\n", - "\n", - " [[9.946e-01 5.388e-03]\n", - " [5.859e-02 9.414e-01]]\n", - "\n", - " [[9.996e-01 4.246e-04]\n", - " [9.992e-01 7.841e-04]]\n", - "\n", - " [[5.079e-03 9.949e-01]\n", - " [5.079e-03 9.949e-01]]\n", - "\n", - " [[9.934e-01 6.623e-03]\n", - " [6.244e-03 9.938e-01]]\n", - "\n", - " [[8.963e-01 1.037e-01]\n", - " [7.467e-02 9.253e-01]]\n", - "\n", - " [[9.859e-01 1.409e-02]\n", - " [4.830e-01 5.170e-01]]]\n", - "\n", - "\n", - " [[[2.506e-01 7.494e-01]\n", - " [4.062e-02 9.594e-01]]\n", - "\n", - " [[9.235e-01 7.648e-02]\n", - " [1.070e-01 8.930e-01]]\n", - "\n", - " [[9.972e-01 2.828e-03]\n", - " [7.670e-01 2.330e-01]]\n", - "\n", - " [[9.020e-01 9.800e-02]\n", - " [9.971e-02 9.003e-01]]\n", - "\n", - " [[9.900e-01 9.966e-03]\n", - " [1.294e-03 9.987e-01]]\n", - "\n", - " [[8.963e-01 1.037e-01]\n", - " [7.467e-02 9.253e-01]]\n", - "\n", - " [[9.859e-01 1.409e-02]\n", - " [4.830e-01 5.170e-01]]]\n", - "\n", - "\n", - " [[[8.947e-01 1.053e-01]\n", - " [9.436e-02 9.056e-01]]\n", - "\n", - " [[9.919e-01 8.072e-03]\n", - " [8.154e-01 1.846e-01]]\n", - "\n", - " [[1.394e-01 8.606e-01]\n", - " [4.872e-02 9.513e-01]]\n", - "\n", - " [[9.949e-01 5.146e-03]\n", - " [9.897e-01 1.031e-02]]\n", - "\n", - " [[9.900e-01 9.966e-03]\n", - " [1.294e-03 9.987e-01]]\n", - "\n", - " [[8.963e-01 1.037e-01]\n", - " [7.467e-02 9.253e-01]]\n", - "\n", - " [[9.859e-01 1.409e-02]\n", - " [4.830e-01 5.170e-01]]]]\n", - "\n", - "SOUTH\n", - "[[[[8.947e-01 1.053e-01]\n", - " [9.436e-02 9.056e-01]]\n", - "\n", - " [[9.946e-01 5.388e-03]\n", - " [5.859e-02 9.414e-01]]\n", - "\n", - " [[9.340e-01 6.596e-02]\n", - " [1.060e-01 8.940e-01]]\n", - "\n", - " [[9.020e-01 9.800e-02]\n", - " [9.971e-02 9.003e-01]]\n", - "\n", - " [[9.420e-01 5.805e-02]\n", - " [6.018e-02 9.398e-01]]\n", - "\n", - " [[7.470e-04 9.993e-01]\n", - " [3.918e-05 1.000e+00]]\n", - "\n", - " [[9.996e-01 4.149e-04]\n", - " [9.978e-01 2.189e-03]]]\n", - "\n", - "\n", - " [[[9.549e-04 9.990e-01]\n", - " [7.301e-04 9.993e-01]]\n", - "\n", - " [[9.946e-01 5.388e-03]\n", - " [5.859e-02 9.414e-01]]\n", - "\n", - " [[9.996e-01 4.246e-04]\n", - " [9.992e-01 7.841e-04]]\n", - "\n", - " [[5.079e-03 9.949e-01]\n", - " [5.079e-03 9.949e-01]]\n", - "\n", - " [[9.934e-01 6.623e-03]\n", - " [6.244e-03 9.938e-01]]\n", - "\n", - " [[8.963e-01 1.037e-01]\n", - " [7.467e-02 9.253e-01]]\n", - "\n", - " [[9.859e-01 1.409e-02]\n", - " [4.830e-01 5.170e-01]]]\n", - "\n", - "\n", - " [[[2.506e-01 7.494e-01]\n", - " [4.062e-02 9.594e-01]]\n", - "\n", - " [[9.235e-01 7.648e-02]\n", - " [1.070e-01 8.930e-01]]\n", - "\n", - " [[9.972e-01 2.828e-03]\n", - " [7.670e-01 2.330e-01]]\n", - "\n", - " [[9.020e-01 9.800e-02]\n", - " [9.971e-02 9.003e-01]]\n", - "\n", - " [[9.900e-01 9.966e-03]\n", - " [1.294e-03 9.987e-01]]\n", - "\n", - " [[8.963e-01 1.037e-01]\n", - " [7.467e-02 9.253e-01]]\n", - "\n", - " [[9.859e-01 1.409e-02]\n", - " [4.830e-01 5.170e-01]]]\n", - "\n", - "\n", - " [[[8.947e-01 1.053e-01]\n", - " [9.436e-02 9.056e-01]]\n", - "\n", - " [[9.919e-01 8.072e-03]\n", - " [8.154e-01 1.846e-01]]\n", - "\n", - " [[1.394e-01 8.606e-01]\n", - " [4.872e-02 9.513e-01]]\n", - "\n", - " [[9.949e-01 5.146e-03]\n", - " [9.897e-01 1.031e-02]]\n", - "\n", - " [[9.900e-01 9.966e-03]\n", - " [1.294e-03 9.987e-01]]\n", - "\n", - " [[8.963e-01 1.037e-01]\n", - " [7.467e-02 9.253e-01]]\n", - "\n", - " [[9.859e-01 1.409e-02]\n", - " [4.830e-01 5.170e-01]]]]\n", - "\n", - "EAST\n", - "[[[[8.947e-01 1.053e-01]\n", - " [9.436e-02 9.056e-01]]\n", - "\n", - " [[9.946e-01 5.388e-03]\n", - " [5.859e-02 9.414e-01]]\n", - "\n", - " [[9.340e-01 6.596e-02]\n", - " [1.060e-01 8.940e-01]]\n", - "\n", - " [[9.020e-01 9.800e-02]\n", - " [9.971e-02 9.003e-01]]\n", - "\n", - " [[9.420e-01 5.805e-02]\n", - " [6.018e-02 9.398e-01]]\n", - "\n", - " [[7.470e-04 9.993e-01]\n", - " [3.918e-05 1.000e+00]]\n", - "\n", - " [[9.996e-01 4.149e-04]\n", - " [9.978e-01 2.189e-03]]]\n", - "\n", - "\n", - " [[[9.549e-04 9.990e-01]\n", - " [7.301e-04 9.993e-01]]\n", - "\n", - " [[9.946e-01 5.388e-03]\n", - " [5.859e-02 9.414e-01]]\n", - "\n", - " [[9.996e-01 4.246e-04]\n", - " [9.992e-01 7.841e-04]]\n", - "\n", - " [[5.079e-03 9.949e-01]\n", - " [5.079e-03 9.949e-01]]\n", - "\n", - " [[9.934e-01 6.623e-03]\n", - " [6.244e-03 9.938e-01]]\n", - "\n", - " [[8.963e-01 1.037e-01]\n", - " [7.467e-02 9.253e-01]]\n", - "\n", - " [[9.859e-01 1.409e-02]\n", - " [4.830e-01 5.170e-01]]]\n", - "\n", - "\n", - " [[[2.506e-01 7.494e-01]\n", - " [4.062e-02 9.594e-01]]\n", - "\n", - " [[9.235e-01 7.648e-02]\n", - " [1.070e-01 8.930e-01]]\n", - "\n", - " [[9.972e-01 2.828e-03]\n", - " [7.670e-01 2.330e-01]]\n", - "\n", - " [[9.020e-01 9.800e-02]\n", - " [9.971e-02 9.003e-01]]\n", - "\n", - " [[9.900e-01 9.966e-03]\n", - " [1.294e-03 9.987e-01]]\n", - "\n", - " [[8.963e-01 1.037e-01]\n", - " [7.467e-02 9.253e-01]]\n", - "\n", - " [[9.859e-01 1.409e-02]\n", - " [4.830e-01 5.170e-01]]]\n", - "\n", - "\n", - " [[[8.947e-01 1.053e-01]\n", - " [9.436e-02 9.056e-01]]\n", - "\n", - " [[9.919e-01 8.072e-03]\n", - " [8.154e-01 1.846e-01]]\n", - "\n", - " [[1.394e-01 8.606e-01]\n", - " [4.872e-02 9.513e-01]]\n", - "\n", - " [[9.949e-01 5.146e-03]\n", - " [9.897e-01 1.031e-02]]\n", - "\n", - " [[9.900e-01 9.966e-03]\n", - " [1.294e-03 9.987e-01]]\n", - "\n", - " [[8.963e-01 1.037e-01]\n", - " [7.467e-02 9.253e-01]]\n", - "\n", - " [[9.859e-01 1.409e-02]\n", - " [4.830e-01 5.170e-01]]]]\n", - "\n", - "WEST\n", - "[[[[8.947e-01 1.053e-01]\n", - " [9.436e-02 9.056e-01]]\n", - "\n", - " [[9.946e-01 5.388e-03]\n", - " [5.859e-02 9.414e-01]]\n", - "\n", - " [[9.340e-01 6.596e-02]\n", - " [1.060e-01 8.940e-01]]\n", - "\n", - " [[9.020e-01 9.800e-02]\n", - " [9.971e-02 9.003e-01]]\n", - "\n", - " [[9.420e-01 5.805e-02]\n", - " [6.018e-02 9.398e-01]]\n", - "\n", - " [[7.470e-04 9.993e-01]\n", - " [3.918e-05 1.000e+00]]\n", - "\n", - " [[9.996e-01 4.149e-04]\n", - " [9.978e-01 2.189e-03]]]\n", - "\n", - "\n", - " [[[9.549e-04 9.990e-01]\n", - " [7.301e-04 9.993e-01]]\n", - "\n", - " [[9.946e-01 5.388e-03]\n", - " [5.859e-02 9.414e-01]]\n", - "\n", - " [[9.996e-01 4.246e-04]\n", - " [9.992e-01 7.841e-04]]\n", - "\n", - " [[5.079e-03 9.949e-01]\n", - " [5.079e-03 9.949e-01]]\n", - "\n", - " [[9.934e-01 6.623e-03]\n", - " [6.244e-03 9.938e-01]]\n", - "\n", - " [[8.963e-01 1.037e-01]\n", - " [7.467e-02 9.253e-01]]\n", - "\n", - " [[9.859e-01 1.409e-02]\n", - " [4.830e-01 5.170e-01]]]\n", - "\n", - "\n", - " [[[2.506e-01 7.494e-01]\n", - " [4.062e-02 9.594e-01]]\n", - "\n", - " [[9.235e-01 7.648e-02]\n", - " [1.070e-01 8.930e-01]]\n", - "\n", - " [[9.972e-01 2.828e-03]\n", - " [7.670e-01 2.330e-01]]\n", - "\n", - " [[9.020e-01 9.800e-02]\n", - " [9.971e-02 9.003e-01]]\n", - "\n", - " [[9.900e-01 9.966e-03]\n", - " [1.294e-03 9.987e-01]]\n", - "\n", - " [[8.963e-01 1.037e-01]\n", - " [7.467e-02 9.253e-01]]\n", - "\n", - " [[9.859e-01 1.409e-02]\n", - " [4.830e-01 5.170e-01]]]\n", - "\n", - "\n", - " [[[8.947e-01 1.053e-01]\n", - " [9.436e-02 9.056e-01]]\n", - "\n", - " [[9.919e-01 8.072e-03]\n", - " [8.154e-01 1.846e-01]]\n", - "\n", - " [[1.394e-01 8.606e-01]\n", - " [4.872e-02 9.513e-01]]\n", - "\n", - " [[9.949e-01 5.146e-03]\n", - " [9.897e-01 1.031e-02]]\n", - "\n", - " [[9.900e-01 9.966e-03]\n", - " [1.294e-03 9.987e-01]]\n", - "\n", - " [[8.963e-01 1.037e-01]\n", - " [7.467e-02 9.253e-01]]\n", - "\n", - " [[9.859e-01 1.409e-02]\n", - " [4.830e-01 5.170e-01]]]]\n", - "\n" - ] - } - ], - "source": [ - "agent = load_info(agent_path)\n", - "mem = agent.memory\n", - "action_map = ['NORTH', 'SOUTH', 'EAST', 'WEST']\n", - "# info['logs']['initial_improvement_policy'], agent.memory, agent.policy\n", - "for a, m in enumerate(mem):\n", - " print(action_map[a])\n", - " print(mem)\n", - " print()" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "cb709cda", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(obs: 1, action: NORTH) -> HOLD\n", - "(obs: 3, action: NORTH) -> HOLD\n", - "(obs: 4, action: NORTH) -> HOLD\n", - "(obs: 5, action: NORTH) -> SET\n", - "(obs: 0, action: SOUTH) -> SET\n", - "(obs: 1, action: SOUTH) -> HOLD\n", - "(obs: 2, action: SOUTH) -> RESET\n", - "(obs: 3, action: SOUTH) -> SET\n", - "(obs: 4, action: SOUTH) -> HOLD\n", - "(obs: 3, action: EAST) -> HOLD\n", - "(obs: 4, action: EAST) -> HOLD\n", - "(obs: 3, action: WEST) -> RESET\n", - "(obs: 4, action: WEST) -> HOLD\n" - ] - } - ], - "source": [ - "tol = 0.1\n", - "\n", - "SET = np.array([\n", - " [0, 1], \n", - " [0, 1]\n", - "])\n", - "\n", - "RESET = np.array([\n", - " [1, 0], \n", - " [1, 0]\n", - "])\n", - "\n", - "HOLD = np.array([\n", - " [1, 0], \n", - " [0, 1]\n", - "])\n", - "\n", - "FLIP = np.array([\n", - " [0, 1],\n", - " [1, 0]\n", - "])\n", - "\n", - "for a, act_mem in enumerate(agent.memory):\n", - " for obs_idx, obs_act_mem in enumerate(act_mem):\n", - " if obs_idx == act_mem.shape[0] - 1:\n", - " continue\n", - " \n", - " if np.allclose(obs_act_mem, SET, atol=tol):\n", - " print(f\"(obs: {obs_idx}, action: {action_map[a]}) -> SET\")\n", - " elif np.allclose(obs_act_mem, RESET, atol=tol):\n", - " print(f\"(obs: {obs_idx}, action: {action_map[a]}) -> RESET\")\n", - " elif np.allclose(obs_act_mem, HOLD, atol=tol):\n", - " print(f\"(obs: {obs_idx}, action: {action_map[a]}) -> HOLD\")\n", - " elif np.allclose(obs_act_mem, FLIP, atol=tol):\n", - " print(f\"(obs: {obs_idx}, action: {action_map[a]}) -> FLIP\")\n", - " else:\n", - " continue\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cf708a97", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "da36b97c", - "metadata": {}, - "source": [ - "(obs: 1, action: NORTH) -> HOLD\n", - "(obs: 5, action: NORTH) -> SET\n", - "(obs: 0, action: SOUTH) -> SET\n", - "(obs: 1, action: SOUTH) -> HOLD\n", - "(obs: 2, action: SOUTH) -> RESET\n", - "(obs: 3, action: SOUTH) -> SET\n", - "(obs: 0, action: EAST) -> SET\n", - "(obs: 4, action: EAST) -> HOLD\n", - "(obs: 3, action: WEST) -> RESET\n", - "(obs: 4, action: WEST) -> HOLD" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2908580b", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.6" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/plotting/cheese_memory.py b/scripts/plotting/cheese_memory.py new file mode 100644 index 00000000..567ed819 --- /dev/null +++ b/scripts/plotting/cheese_memory.py @@ -0,0 +1,97 @@ +# %% codecell +import numpy as np +import jax.numpy as jnp +import matplotlib.pyplot as plt +import matplotlib.cm as cm +from jax.nn import softmax +from jax.config import config +from pathlib import Path +from collections import namedtuple + +config.update('jax_platform_name', 'cpu') +np.set_printoptions(precision=4) +plt.rcParams['axes.facecolor'] = 'white' +plt.rcParams.update({'font.size': 22}) + +from grl.utils import load_info +from grl.utils.mdp import get_perf +from definitions import ROOT_DIR +np.set_printoptions(precision=3) +# %% codecell + +results_dir = Path(ROOT_DIR, 'results') + +# %% codecell +all_results = {} + +# for results_path in results_dir.iterdir(): +# if 'tmaze' not in results_path.stem: +# continue +# info = load_info(results_path) +# results_paths = [res_path for res_path in results_dir.iterdir() if 'cheese' in res_path.stem] +# results_path = results_paths[0] +results_path = results_dir / 'cheese.95_mi_pi(pi)_miit(1)_s(2020)_Thu Feb 2 20:50:49 2023.npy' +# %% codecell +agent_path = results_path.parent / 'agents' / f"{results_path.stem}.pkl.npy" +# %% codecell +agent = load_info(agent_path) +mem = agent.memory +action_map = ['NORTH', 'SOUTH', 'EAST', 'WEST'] +# info['logs']['initial_improvement_policy'], agent.memory, agent.policy +for a, m in enumerate(mem): + print(action_map[a]) + print(mem) + print() +# %% codecell +tol = 0.1 + +SET = np.array([ + [0, 1], + [0, 1] +]) + +RESET = np.array([ + [1, 0], + [1, 0] +]) + +HOLD = np.array([ + [1, 0], + [0, 1] +]) + +FLIP = np.array([ + [0, 1], + [1, 0] +]) + +for a, act_mem in enumerate(agent.memory): + for obs_idx, obs_act_mem in enumerate(act_mem): + if obs_idx == act_mem.shape[0] - 1: + continue + + if np.allclose(obs_act_mem, SET, atol=tol): + print(f"(obs: {obs_idx}, action: {action_map[a]}) -> SET") + elif np.allclose(obs_act_mem, RESET, atol=tol): + print(f"(obs: {obs_idx}, action: {action_map[a]}) -> RESET") + elif np.allclose(obs_act_mem, HOLD, atol=tol): + print(f"(obs: {obs_idx}, action: {action_map[a]}) -> HOLD") + elif np.allclose(obs_act_mem, FLIP, atol=tol): + print(f"(obs: {obs_idx}, action: {action_map[a]}) -> FLIP") + else: + continue + +# %% codecell + +# %% markdown +# (obs: 1, action: NORTH) -> HOLD +# (obs: 5, action: NORTH) -> SET +# (obs: 0, action: SOUTH) -> SET +# (obs: 1, action: SOUTH) -> HOLD +# (obs: 2, action: SOUTH) -> RESET +# (obs: 3, action: SOUTH) -> SET +# (obs: 0, action: EAST) -> SET +# (obs: 4, action: EAST) -> HOLD +# (obs: 3, action: WEST) -> RESET +# (obs: 4, action: WEST) -> HOLD +# %% codecell diff --git a/scripts/plotting/mi_performance.ipynb b/scripts/plotting/mi_performance.ipynb deleted file mode 100644 index 863d2ad4..00000000 --- a/scripts/plotting/mi_performance.ipynb +++ /dev/null @@ -1,518 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 23, - "id": "8b4f5fc4", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import jax.numpy as jnp\n", - "import matplotlib.pyplot as plt\n", - "import matplotlib.cm as cm\n", - "import pandas as pd\n", - "\n", - "from jax.nn import softmax\n", - "from jax.config import config\n", - "from pathlib import Path\n", - "from collections import namedtuple\n", - "\n", - "config.update('jax_platform_name', 'cpu')\n", - "np.set_printoptions(precision=4)\n", - "plt.rcParams['axes.facecolor'] = 'white'\n", - "plt.rcParams.update({'font.size': 18})\n", - "\n", - "from grl.utils import load_info\n", - "from definitions import ROOT_DIR\n" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "id": "a50e200e", - "metadata": {}, - "outputs": [], - "source": [ - "# results_dir = Path(ROOT_DIR, 'results', 'final_analytical')\n", - "results_dir = Path(ROOT_DIR, 'results', 'tiger_grid_mi_pi_obs_space')\n", - "\n", - "# results_dir = Path(ROOT_DIR, 'results', 'prisoners_dilemma')\n", - "# results_dir = Path(ROOT_DIR, 'results', 'pomdps_mi_dm')\n", - "vi_results_dir = Path(ROOT_DIR, 'results', 'vi')\n", - "pomdp_files_dir = Path(ROOT_DIR, 'grl', 'environment', 'pomdp_files')\n", - "\n", - "args_to_keep = ['spec', 'n_mem_states', 'seed']\n", - "split_by = [arg for arg in args_to_keep if arg != 'seed']\n", - "\n", - "# this option allows us to compare to either the optimal belief state soln\n", - "# or optimal state soln. ('belief' | 'state')\n", - "# compare_to = 'belief'\n", - "compare_to = 'state'\n", - "\n", - "\n", - "# spec_plot_order = [\n", - "# 'example_7', 'tmaze_5_two_thirds_up', 'tiger-alt-start', 'paint.95', 'cheese.95', 'network',\n", - "# 'shuttle.95', '4x3.95', 'hallway'\n", - "# ]\n", - "# spec_plot_order = [\n", - "# 'hallway', 'network', 'paint.95', '4x3.95', 'tiger-alt-start', 'shuttle.95', 'cheese.95', 'tmaze_5_two_thirds_up'\n", - "# ]\n", - "spec_plot_order = ['tiger-alt-start','tiger-grid']\n", - "\n", - "# game_name = 'prisoners_dilemma'\n", - "# # leader_policies = ['all_d', 'extort', 'tit_for_tat', 'treasure_hunt', 'sugar', 'all_c', 'grudger2', 'alternator', 'majority3']\n", - "\n", - "# leader_policies = ['all_d', 'extort', 'tit_for_tat', 'treasure_hunt', 'sugar', 'grudger2', 'alternator', 'majority3']\n", - "# leader_policy_labels = {\n", - "# 'all_d': 'all d', \n", - "# 'extort': 'extort', \n", - "# 'tit_for_tat': 'tit for\\ntat',\n", - "# 'treasure_hunt': 'treasure\\nhunt',\n", - "# 'sugar':'sugar', \n", - "# 'all_c': 'all c', \n", - "# 'grudger2': 'grudger 2', \n", - "# 'alternator':'alternator', \n", - "# 'majority3':'majority 3'\n", - "# }\n", - "\n", - "# spec_plot_order = []\n", - "# prisoners_spec_map = {}\n", - "# for leader in leader_policies:\n", - "# spec_id = f'{game_name}_{leader}'\n", - "# prisoners_spec_map[spec_id] = leader_policy_labels[leader]\n", - "# spec_plot_order.append(spec_id)\n", - "\n", - "spec_to_belief_state = {'tmaze_5_two_thirds_up': 'tmaze5'}\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "id": "739315d8", - "metadata": {}, - "outputs": [], - "source": [ - "compare_to_dict = {}\n", - "\n", - "for spec in spec_plot_order:\n", - " if compare_to == 'belief':\n", - "\n", - " for fname in pomdp_files_dir.iterdir():\n", - " if 'pomdp-solver-results' in fname.stem:\n", - " if (fname.stem ==\n", - " f\"{spec_to_belief_state.get(spec, spec)}-pomdp-solver-results\"\n", - " ):\n", - " belief_info = load_info(fname)\n", - " coeffs = belief_info['coeffs']\n", - " max_start_vals = coeffs[belief_info['max_start_idx']]\n", - " compare_to_dict[spec] = np.dot(max_start_vals, belief_info['p0'])\n", - " break\n", - " # print(f\"loaded results for {hparams.spec} from {fname}\")\n", - " else:\n", - " for vi_path in vi_results_dir.iterdir():\n", - " for spec in spec_plot_order:\n", - " if spec_to_belief_state.get(spec, spec) in vi_path.name:\n", - " vi_info = load_info(vi_path)\n", - " max_start_vals = vi_info['optimal_vs']\n", - " compare_to_dict[spec] = np.dot(max_start_vals, vi_info['p0'])\n", - " \n", - " elif compare_to == 'state':\n", - " for vi_path in vi_results_dir.iterdir():\n", - " if spec_to_belief_state.get(spec, spec) in vi_path.name:\n", - " vi_info = load_info(vi_path)\n", - " max_start_vals = vi_info['optimal_vs']\n", - " compare_to_dict[spec] = np.dot(max_start_vals, vi_info['p0'])\n", - " \n", - "# compare_to_list.append(spec_compare_indv)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 58, - "id": "82aa13fd", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'tiger-alt-start': 10.0, 'tiger-grid': 2.9818344116210938}" - ] - }, - "execution_count": 58, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "compare_to_dict\n" - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "id": "dee0c91f", - "metadata": {}, - "outputs": [], - "source": [ - "all_results = []\n", - "\n", - "for results_path in results_dir.iterdir():\n", - " if results_path.is_dir() or results_path.suffix != '.npy':\n", - " continue\n", - "\n", - " info = load_info(results_path)\n", - "\n", - " args = info['args']\n", - " if args['spec'] not in spec_plot_order:\n", - " continue\n", - "\n", - " # agent = info['agent']\n", - " init_policy_info = info['logs']['initial_policy_stats']\n", - " init_improvement_info = info['logs']['greedy_initial_improvement_stats']\n", - " final_mem_info = info['logs']['greedy_final_mem_stats']\n", - "\n", - " def get_perf(info: dict):\n", - " return (info['state_vals_v'] * info['p0']).sum()\n", - " single_res = {k: args[k] for k in args_to_keep}\n", - "\n", - " final_mem_perf = get_perf(final_mem_info)\n", - " compare_to_perf = compare_to_dict[args['spec']]\n", - " init_policy_perf = get_perf(init_policy_info)\n", - " init_improvement_perf = get_perf(init_improvement_info)\n", - "# if (final_mem_perf > compare_to_perf):\n", - "# if np.isclose(final_mem_perf, compare_to_perf):\n", - "# final_mem_perf = compare_to_perf\n", - "# else:\n", - "# raise Exception(f\"{args['spec']}, compare_to_perf: {compare_to_perf:.3f}, final_mem_perf: {final_mem_perf:.3f}\")\n", - " \n", - " if init_policy_perf > init_improvement_perf:\n", - " init_policy_perf = init_improvement_perf\n", - " \n", - " single_res.update({\n", - " 'init_policy_perf': init_policy_perf,\n", - " 'init_improvement_perf': init_improvement_perf,\n", - " 'final_mem_perf': final_mem_perf,\n", - " 'compare_to_perf': compare_to_perf,\n", - " # 'init_policy': info['logs']['initial_policy'],\n", - " # 'init_improvement_policy': info['logs']['initial_improvement_policy'],\n", - " # 'final_mem': np.array(agent.memory),\n", - " # 'final_policy': np.array(agent.policy)\n", - " })\n", - " all_results.append(single_res)\n", - "\n", - "\n", - "all_res_df = pd.DataFrame(all_results)" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "id": "925822f8", - "metadata": {}, - "outputs": [], - "source": [ - "all_res_groups = all_res_df.groupby(split_by, as_index=False)\n", - "all_res_means = all_res_groups.mean()\n", - "del all_res_means['seed']\n", - "all_res_means.to_csv(Path(ROOT_DIR, 'results', 'all_pomdps_means.csv'))" - ] - }, - { - "cell_type": "code", - "execution_count": 69, - "id": "0ec56846", - "metadata": {}, - "outputs": [], - "source": [ - "cols_to_normalize = ['init_improvement_perf', 'final_mem_perf']\n", - "# merged_df = all_res_df.merge(compare_to_df, on='spec')\n", - "merged_df = all_res_df\n", - "\n", - "# for col_name in cols_to_normalize:\n", - "\n", - "normalized_df = merged_df.copy()\n", - "normalized_df['init_improvement_perf'] = (normalized_df['init_improvement_perf'] - merged_df['init_policy_perf']) / (merged_df['compare_to_perf'] - merged_df['init_policy_perf'])\n", - "normalized_df['final_mem_perf'] = (normalized_df['final_mem_perf'] - merged_df['init_policy_perf']) / (merged_df['compare_to_perf'] - merged_df['init_policy_perf'])\n", - "del normalized_df['init_policy_perf']\n", - "del normalized_df['compare_to_perf']" - ] - }, - { - "cell_type": "code", - "execution_count": 70, - "id": "dc39ebe7", - "metadata": {}, - "outputs": [], - "source": [ - "normalized_df.loc[(normalized_df['spec'] == 'hallway') & (normalized_df['n_mem_states'] == 8), 'final_mem_perf'] = 0\n" - ] - }, - { - "cell_type": "code", - "execution_count": 71, - "id": "8236ec56", - "metadata": {}, - "outputs": [], - "source": [ - "# normalized_df[normalized_df['spec'] == 'prisoners_dilemma_all_c']\n", - "seeds = normalized_df[normalized_df['spec'] == normalized_df['spec'][0]]['seed'].unique()" - ] - }, - { - "cell_type": "code", - "execution_count": 72, - "id": "d677a428", - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuoAAAIDCAYAAABSLDvsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAACaXklEQVR4nOzdd1hUR/s38O9Zel2xIAIKlthNNLbYAHtJ7L2BYo8xxZYYY0TNE0usiS02kMSIqIldUQkKdrEbjS00QWzA0uue9w9ezg+kLcvCsvr9XNdeD5wzZ+ZeND73DjP3CKIoiiAiIiIiogpFpu0AiIiIiIgoPybqREREREQVEBN1IiIiIqIKiIk6EREREVEFxESdiIiIiKgCYqJORERERFQBMVEnIiIiIqqA9LUdAGmWUqlEVFQULCwsIAiCtsMhIiIiojeIooiEhATY2tpCJit83pyJ+lsmKioKNWvW1HYYRERERFSMiIgI2NvbF3qfifpbxsLCAkD2H7ylpaWWoyEiIiKiN8XHx6NmzZpS3lYYJupvmZzlLpaWlkzUiYiIiCqw4pYpczMpEREREVEFxESdiIiIiKgCYqJORERERFQBMVEnIiIiIqqAmKgTEREREVVATNSJiIiIiCogJupERERERBUQE3UiIiIiogpIJw88yszMxMuXL/Hy5UukpqaiSpUqqFatGg/4ISIiIqK3hs4k6mfPnsWJEydw9uxZXLt2DZmZmfna1KhRA05OTnB2dsaQIUNQpUoVLURKRERERFR6giiKoraDKEx8fDy2bduGLVu24NGjRwAAVcIVBAGGhoYYNGgQpk2bho4dO5Z1qBVGfHw85HI5FAoFf8NAREREVAGpmq9VyEQ9MzMTGzZswA8//ICYmBiIoggLCwu0adMGbdu2RYsWLVC1alVUrlwZJiYmiImJQUxMDEJCQnD58mVcvnwZDx48AJCdtPfq1Qs//fQTGjdurHIMycnJ0uz99evXce3aNYSHhwMAFi5cCA8Pj1K/z+fPn2PFihU4cuQIwsPDYWJigiZNmsDNzQ0TJkyAIAgl7pOJOhEREVHFpmq+ViGXvjRu3BhPnjyBvr4++vXrhzFjxqBv374wNDQs9tlp06YBAB49eoTff/8df/zxB44fP46TJ09ix44dGDt2rEoxXLlyBX369CnV+yjKtWvX0LNnT7x+/RoAYG5ujoSEBJw7dw7nzp3Dvn37cOjQIZXeMxERERG9fSpk1ZewsDBMnjwZjx8/xl9//YXBgweXOGF97733sGjRIjx69Ai7d+9GgwYNEBISUqI+rKys0LVrV8yZMwe7d++GjY1NiZ4vjEKhwCeffILXr1+jYcOGuHr1KhISEpCUlIT169fDwMAAfn5++PLLLzUyHhERERHpngq59CUiIgI1a9bUaJ+iKCIqKgp2dnYqtc/KyoKenl6ea46OjggLCyv10pcFCxbghx9+gImJCf755x/Url07z/2lS5fi22+/hZ6eHu7du4f69eur3DeXvhARERFVbKrmaxVyRl3TSTqQvVZd1SQdQL4kXZO8vb0BACNGjMiXpAPAjBkzYG5ujqysLOzatavM4iAiIiKiiqtCJupvswcPHkibUnv37l1gG3Nzc3Tq1AkAcPLkyXKLjYiIiIgqDp1J1Lt06YI9e/YgIyND26GUyt27d6WvmzZtWmi7nHv37t0r85iIiIiIqOLRmUT9zJkzGDVqFOzs7DB37lyprrquiYqKkr4uailOzr34+HgkJiYW2i4tLQ3x8fF5XkRERESk+3QmUR89ejSMjIzw6tUrrFq1Cg0bNkTXrl3h6+urU7PsCQkJ0tempqaFtst9L/czb1q6dCnkcrn0Kov1/URERERU/nQmUf/tt98QFRWFdevWoWnTphBFEQEBARg5ciTs7Ozw9ddf4/Hjx9oOs9zNmzcPCoVCekVERGg7JCIiIiLSAJ1J1AGgUqVKmDFjBm7duoWLFy9i/PjxMDU1xatXr7By5Uo0aNCgws+yW1hYSF8nJycX2i73vdzPvMnIyAiWlpZ5XkRERESk+3QqUc+tbdu22L59O6KiorBx40Y0b95cJ2bZbW1tpa8jIyMLbZdzz9LSEubm5mUeFxERERFVLDqbqOewsLDA1KlTce3aNVy7dg1OTk4QRRGvX7+WZtl79OhRYcoc5q70krsCzJty7jVu3LjMYyIiIiKiikfnE3UASElJgaenJz799FMEBQUByD6J1MrKCqIo4vTp0+jduzcGDBhQ5HKT8lC/fn3UqlULAHDixIkC2yQlJUnvo0ePHuUWGxERERFVHDqdqN+8eROffvopatSogYkTJ+Ly5cvQ09PD4MGDcfr0abx69QrBwcFwdXWFTCbD4cOH8f3332s1ZkEQ4OrqCgDw8fFBaGhovjYbNmxAYmIi9PT0MHr06HKOkIiIiIgqAp1L1JOSkrB161a0bt0aLVu2xK+//or4+HjY29tjyZIlCA8Px969e9GlSxcAwIcffggvLy8cPHgQoihi7969Ko8VGxuLV69eSS+lUgkge6Nn7utv1jn38PCAIAgQBKHARHz27NmwsbFBcnIyPv74Y1y7dg0AkJ6ejk2bNmHBggUAgMmTJ6N+/frq/JiIiIiISMcJoiiK2g5CFcHBwdiyZQt8fHyQlJQEURQhk8nQs2dPTJs2DX369IFMVvTnjurVq+P169fIzMxUaUxHR0eEhYUV287NzQ1eXl7S9x4eHli0aBEAICQkBI6OjvmeuXbtGnr27InXr18DyF5rn5qaKlWr6dGjBw4dOgQjIyOVYs0RHx8PuVwOhULBCjBEREREFZCq+Zp+OcZUKm3atIEgCBBFEdbW1pgwYQImT54MBwcHlfswMTFBRflc0rJlS/zzzz9Yvnw5jhw5goiICJiZmaFp06Zwc3ODu7t7sR88iIiIiOjtpTMz6jKZDM7Ozpg2bRoGDRoEff2Sf8aIjIxEZmZmiZJ7XcMZdSIiIqKK7a2bUb9//z4aNGhQqj7s7Ow0FA0RERERUdnSmbUVpU3SiYiIiIh0ic4k6gkJCTh06BACAgKKbfv333/j0KFD+aqxEBERERHpCp1J1Hft2oWBAwfi+PHjxbbdu3cvBg4ciD179pRDZEREREREmqczifpff/0FABg5cmSxbd3d3SGKIvbv31/WYRERERERlQmdSdQfPnwIAwMDNG/evNi2LVu2hIGBAR48eFD2gRERERERlQGdSdSjo6NhaWkJQRCKbSuTyWBpaYno6OhyiIyIiIiISPN0JlE3NTWFQqFAVlZWsW0zMzMRHx8PQ0PDcoiMiIiIiEjzdCZRf++995CZmQl/f/9i2/r7+yMjIwN169Yth8iIiIiIiDRPZxL1Pn36QBRFzJkzBwkJCYW2S0xMxJw5cyAIAvr06VOOERIRERERaY7OJOqfffYZrKyscPfuXbRu3Rp//fUXUlJSpPspKSn4888/0apVK9y9exdyuRxffPGFFiMmIiIiIlKfvrYDUFXlypWxe/duDBgwAA8fPsSQIUOgp6eHqlWrAgBevXqFrKwsiKIIY2Nj+Pr6okqVKlqOmoiIiIhIPTqTqANAjx49cP78eXz++ec4f/48MjMz81V2cXJywrp16/DBBx9oKUoiIqpIRFFEenq6SsUIiOjdoqenB0NDQ5WqCmqDTiXqANCiRQsEBQXh8ePHuHDhAqKjoyEIAmxsbNC+fXtuICUiIgBAVlYWoqOjERcXh/T0dG2HQ0QVlKGhISpVqgQbGxvo6elpO5w8dC5Rz1GvXj3Uq1dP22EQEVEFlJWVhSdPniA1NRVWVlawtLSEgYGBtsMiogomIyMD8fHxeP36NZKSklC3bt0KlazrbKJORERUmOjoaKSmpqJevXowNTXVdjhEVIHJ5XJUqVIFjx8/RnR0NOzs7LQdkkRnqr4QERGpQhRFxMXFwcrKikk6EanE1NQUVlZWiIuLgyiK2g5HUiFn1Lt06QIAcHBwgKenZ55rJSEIgkoHJBER0dsjPT0d6enpsLS01HYoRKRDLC0t8erVK2RkZFSY0+0rZKJ+5swZAEDDhg3zXSuJirqDl4iIyk5OdReuSSeiksj5NyMzM5OJelEWLlwIAFKN9NzXiIiIiIjeBRU6US/uGhERERHR24qbSYmIiIiIKiAm6kREREREFRATdSIiIiKiCqhCrlFXpxRjQViekYiIiN42jo6OCAsLg6enJ8aNG6ftcKgMVchEXZ1SjAVheUYiIiqK4zdHtR2CRoQu+7jM+vbw8MCiRYuk73fv3o0RI0YU+czHH3+MY8eOSd+HhITA0dGxrEIkemtVyESdFV6IiIgqJk9PzyIT9aioKPj5+ZVjRERvLybqREREVKyqVasiJSUFp0+fxtOnT2Fvb19gO29vb2RlZcHR0RGhoaHlGyTRW4abSYmIiKhYZmZmGDJkCJRKJby8vApt5+npCQBcO02kAUzUiYiISCXjx48HgEIT9XPnzuHhw4eoU6cOnJyciu3v6NGjGDx4MOzs7GBkZAQrKys4OTlh06ZNSE9PL/AZFxcXCIIADw8PZGZmYs2aNWjRogXMzc1hbW2NAQMG4NatW1L75ORk/PDDD2jatCnMzMxQpUoVDB8+HE+ePCkytujoaMyZMwdNmjSBmZkZzMzM0KRJE8ydOxfPnz8v8JnQ0FAIggBBEBAaGoonT55g8uTJqF27NoyMjODo6IjY2FiYmppCEAT4+voWGcOCBQsgCALq1KkDURSL+Wnmdf78eYwZMwYODg4wNjaGXC5HmzZtsHz5ciQmJhb6nJ+fHwYNGgR7e3sYGhrC0tISderUQY8ePbBy5UrExMTke+by5csYPXo0ateuDWNjY5iZmcHBwQHOzs5YsmQJnj59WqLY6f9UyKUvxXn+/Dn27duH4OBgvHjxAgBgbW2N1q1bY/DgwahevbqWIyQiInr7ODk5oW7dunjy5AkCAwPzJeO5Z9OLKuiQkpICV1dX7Nu3T7pmaWkJhUKBoKAgBAUFwdvbG8eOHYOVlVWBfWRkZKBXr17w9/eHoaEhDAwM8PLlSxw8eBD+/v4ICAhA7dq10b17d9y4cQPGxsYQBAExMTHw9fXFmTNncPXqVdSqVStf32fPnsWAAQMQFxcHIPu3CQBw79493Lt3D9u2bcOhQ4fQsWPHQt/jhQsXMGXKFCQmJsLU1BQGBgYAACsrKwwbNgw7d+7Eli1bMGzYsAKfz8rKkn6eEydOVLlAhlKpxFdffYWff/5ZumZubo6kpCRcvXoVV69ehaenJ/z8/ODg4JDn2cWLF+dZfmxqagpRFBESEoKQkBCcOnUKrVq1gouLi9Rm586dGD9+vPRBwsjICPr6+ggPD0d4eDgCAwNRs2ZN/oZFTTo1o56VlYVvv/0WDg4O+Pzzz+Ht7Y3jx4/j+PHj8Pb2xowZM+Dg4ID58+cjKytL2+ESERG9VQRBkBKuHTt25LmXlJQEX19fyGSyYpOyyZMnY9++fahTpw527doFhUIBhUKB5ORkHDx4EHXq1MGlS5fg7u5eaB8bN27EzZs3sXfvXiQmJiIhIQFXrlxBnTp1kJiYiC+++AKTJk1CbGws/Pz8kJSUhMTERJw+fRrVqlXDixcv8O233+brNyIiQkrSGzdujHPnziExMRGJiYkIDAxEgwYNEBsbi/79+yMyMrLQ+KZMmYImTZrg6tWr0tgnT54EAEybNg0A8Pfff+O///4r8Pljx44hMjIS+vr6Rf4c3rRw4UL8/PPPsLa2xoYNG/D69WskJCQgJSUFAQEBaNGiBR48eIBBgwZBqVRKz4WFhUnVfWbOnInIyEgkJSUhISEBcXFxCAoKwqeffgoLCwvpmeTkZMyYMQOiKGLMmDF4/PgxUlNToVAokJiYiODgYMyZMwfW1tYqx0956VSi7urqiuXLlyM9PR2GhoZo3749hg8fjuHDh6N9+/YwNDREeno6li1bxk9uREREZcDNzQ0ymQz79u3Ls4TC19cXiYmJ6Nq1K2rWrFno80FBQfj9999hbW2NM2fOYNSoUbC0tAQAGBsbo1+/fjh79izMzMxw4MAB3Lx5s8B+4uLicODAAQwZMgQGBgYQBAGtW7fG1q1bAWTPaJ84cQKnTp1Cjx49IJPJIJPJ0LVrVyxbtgwA8OeffyIjIyNPvz/++CPi4uJgZWUFf39/dOjQQbrXqVMnnD59GpaWloiJicHSpUsLfZ9VqlTB6dOn0apVK+la/fr1AQBt27ZF8+bNIYqiFO+btmzZAgDo168fbGxsCh0nt9DQUCxduhQmJiY4efIkPv30U1SuXBkAYGBgABcXF5w9exb29va4fv06Dh06JD17+fJlKJVK1K9fH6tWrYKtra10Ty6Xo2PHjtiwYQNatmwpXb979y4SEhJgZmYGT09P1K1bV7pnZmaGli1bYsWKFejTp49K8VN+OpOoHzhwALt374Yoipg5cyaePXuGoKAg7N69G7t370ZQUBCio6Mxe/ZsiKKIP/74I89fQCIiIiq9mjVrolu3btIMeo6cZRrFzf5u374dADB69OhCE3p7e3t07twZAAot9dixY8cCl544OzvDyMgIADBkyBDUq1cvX5uePXsCyF6C8+jRI+m6KIrSe5o6dWqBCbK9vT2mTp0KAPDx8Sn4TQL47LPPYG5uXuj9nFl1Ly+vfB8WIiMjcfz4cQDZM/Oq8vLyQlZWFnr16oUPPvigwDYWFhYYMGAAgLw/20qVKgEAEhISkJSUpNJ4Oc+kp6fj9evXKsdJqtOZRH379u0QBAHz58/HypUrpb8cucnlcqxYsQLz588v8lMqERERqS9nU2nO8pfHjx8jKCgIVlZWUhJYmPPnzwPI/v91GxubQl+nT58GkL0koyBt2rQp8Lqenh6qVq0KAGjdunWBbXLvZYuNjZW+DgkJkTZLduvWrdD30L17dwDA69evERISUmCb3DPxBRk1ahQsLCwQHR2Nw4cP57m3Y8cOZGVlSWvsVZXzsz158mSRP9ucD1W5f7Zt2rRB1apV8ezZM7Rt2xbr16/Hv//+W+Qm1rp166Jhw4bIyMhA27ZtsXz5cty8eZPLjzVIZxL1q1evQiaTYfbs2cW2nT17NmQyGa5evVoOkREREb1bBg4cCCsrK5w/fx6PHj2SEr+RI0fC2Ni4yGejoqIAAPHx8Xj+/Hmhr9TUVADZ66ALknut9Jv09fWLbJNzH0Ce2eycAhUAYGdnV2j/uWvI534mt+LWZZubm2PMmDEA/m+ZC5C9GTTntw6TJk0q0SnrOT/bpKSkIn+2OTPmuX+2lSpVwu7du1GtWjX8888/mDFjBho1agQrKyv069cPv//+e76Zfz09Pfj4+KB27doICwvDN998gxYtWsDS0hLdu3fHpk2bCv3zI9XoTKIeGxsLuVwOuVxebNucdrk/JRMREZFmGBkZYeTIkQCAbdu2wdvbG8D/zbQXJWe2ddOmTRBFsdhXUTXbKzI9Pb1i2+Qsfzl16pR0ONTJkycRFhYGfX19lX6eueX8bL/++muVfrZnzpzJ83y3bt0QEhICb29vuLm54b333oNCocDhw4cxduxYtGjRIt8G2g8++AD//vsv9u/fj8mTJ6Np06bSwViffvopGjZsiDt37pTofdD/0ZlE3crKCgqFAvHx8cW2zdk9XlhJJyIiIiqdnCRy7dq1ePr0KZo2bZpn42RhctZ9F7akRZtyz4IXVfs7973SVDRp1qwZ2rdvn2cWPWfZbv/+/VXeRJpDEz9bMzMzjB07Fl5eXnj48CGePn2K5cuXw9jYWJppf5OhoSEGDRqEX3/9FXfu3MHLly+xefNmVK5cGREREXBzc1M7nnedziTqrVu3hlKpxJo1a4ptu2bNGiiVSpX+wSAiIqKSa9WqFZo1ayYdTKRqCcGctdtHjhwps9jUVbt2balKir+/f6HtctbPV6lSBbVr1y7VmDmz6jt27EBkZKS0Xn3y5Mkl7ivnZ3v69Glp6VBp2dnZYe7cuZg1axaA7Nn/4lSpUgVTpkzB8uXLAQA3btzgZlM16UyinlNMf8mSJViwYEGBp2olJCTgu+++w5IlSyAIAiZMmKCFSImIiN4Ny5cvx6xZszBr1ixpvXVxchLQu3fvYtOmTUW2TUpKKvSE0rIgCAKGDx8OAPj1118RHR2dr01UVBR+/fVXAJCW/5TG0KFDUaVKFURFRWHUqFHIyMgo8SbSHO7u7tDX18erV6/yHFxUkPT09Dy5VFpaWpHtTUxMAAAy2f+ljqo+8+ZzpDqd+akNGjQIw4YNg1KpxI8//ojq1avDxcUFo0ePxujRo+Hs7AwbGxssXboUoihi2LBhGDhwoLbDJiIiemv17t0bK1euxMqVK1GtWjWVnnF2dpaWzUyfPh1fffVVnkN/0tLScOnSJcydOxcODg6FbtYsK99++y0qVaqEmJgYdOvWDRcuXJDunT9/Ht26dUNcXBwqV66Mb775ptTjGRkZSWe/BAYGAij5JtIcdevWxYIFCwAAK1asgKurK+7evSvdz8zMxM2bN7F48WLUq1cvT4365cuXo3fv3vjtt9/yLO1JS0uDr68vfvrpJwDAxx9/LN3z8fFBhw4d8Ouvv+b5M8zKyoKfn5/082nXrh2XI6tJv/gmFcdvv/0Ge3t7/Pzzz0hJSUFgYKD0FzmnfJC+vj6++OIL/Pjjj9oMlYiIiAqxefNm6OnpYdu2bVi7di3Wrl0Lc3NzGBgYQKFQ5DkxU52EtTTs7e1x4MAB9O/fH//88w86dOgAMzMzAJCqpVSqVAkHDhwosjJMSUydOhWrV6+GKIpqbSLNbcGCBcjMzMQPP/yA3377Db/99htMTExgamqKuLi4PKUTc/9slUolTpw4gRMnTgDIng03MTFBbGyslGM1atQIq1evlp4RRREXLlyQPswYGRnB3NwcsbGx0p+hra1tvlNsSXU6lagbGBhg5cqVmDlzJvbv34/g4GDpk7a1tTVatWqFwYMH5zlNi4iIqDChyz4uvhFpnKGhIbZu3Qp3d3ds2bIFQUFBiIqKQlpaGqytrdGwYUM4OTlhyJAhGkuGS8LZ2Rn379/HqlWrcOzYMYSGhkIQBDRq1Agff/wxZs2aVeKNnkWpV68emjdvjhs3bqi1iTQ3QRCwePFiDBs2DJs2bUJAQAAiIiKkIhv169dHhw4dMHDgQLRr1056bvLkybCzs0NAQADu3LmDZ8+eSc80adIEgwcPxpQpU/KU3+zXrx+8vb0REBCA69ev49mzZ4iJiYGFhQUaNGiAvn374rPPPivw7BtSjSAWVcmedE58fDzkcjkUCoV0JDMR0bskOTkZDx48QIMGDWBqaqrtcIiKFR0djZo1ayIzMxN+fn7o0aOHtkN6J5Xnvx2q5ms6s0adiIiI6G20efNmZGZmol69emptIqW3l04tfcntn3/+KXDpS5MmTbQcGREREZFqgoODsWrVKgDAzJkzy31NPlVsOpeoHzlyBN9++y3++eefAu83adIEP/zwA/r161fOkRERERGpxtHREWlpaVIJyBYtWmDixIlajooqGp1a+rJ48WL0798fd+/ehSiK0NPTg7W1NaytraGnpwdRFHH37l0MHDgQHh4e2g6XiIiIqEBhYWGIjo6GjY0Nxo0bh+PHj8PAwEDbYVEFozOJ+okTJ+Dh4QFRFOHk5ISTJ08iISEBz549w7Nnz5CYmIiTJ0/CxcVFOhjJz89P22ETERER5SOKIkRRxLNnz+Dp6Ynq1atrOySqgHQmUc+p2zl06FAEBASgW7duMDIyku4bGhqiW7du8Pf3x9ChQyGKYp5an0REREREukRnEvXg4GAIgoDVq1cXudFCEARpU8bVq1fLKzwiIiIiIo3SmUQ9PT0dlSpVUungA3t7e1hZWSEjI6McIiMiIiIi0jydSdTr1KmDxMREpKenF9s2LS0NiYmJqFOnTjlERkRERESkeTqTqI8aNQoZGRnw9vYutu1vv/2GjIwMjBo1qhwiIyIiIiLSPJ1J1GfNmoWOHTvi888/x86dOwtt5+3tjc8//xydOnXCrFmzyjFCIiIiIiLN0ZkDj5YuXQonJyfcuXMH7u7uWLhwIVxcXKQ165GRkTh79izCw8Mhl8vRqVMn/PjjjwX29f3335dn6EREREREJSaIoihqOwhVyGQyqdpLTshvVn8p7PqbsrKyyiDCiiE+Ph5yuRwKhQKWlpbaDoeIqNwlJyfjwYMHaNCgAUxNTbUdDhHpiPL8t0PVfE1nZtSdnJyKTcCJiIiIiN4WOpOonzlzRtshEBERERGVG53ZTEpERERE9C5hok5EREREVAHpzNIXIiIijfOQazsCzfBQaDsCohJxcXHB2bNnsXDhQnh4eGg7nAqLM+pERERUoOTkZBw/fhw//PADBg0aBAcHBwiCAEEQNJJcjRs3TupPEARcunSp2GeaNm2a5xkqf46Ojnn+DHJeenp6sLKyQps2bbBw4UK8evVKrf7Xrl0LDw8P3Lx5U7OB6yDOqBMREVGBrly5gj59+pTbeJ6envjoo48KvX/58mX8888/5RYPFc3Y2Bhy+f/9Vio1NRVxcXG4evUqrl69ig0bNsDPzw8tW7bM92ytWrXQoEEDVK1aNd+9tWvXIiwsDI6OjmjevHlZvoUKjzPqREREVCgrKyt07doVc+bMwe7du2FjY6PxMWrVqgVBELBnzx6kpKQU2s7T0xNA9owuad/w4cMRHR0tveLi4hAfH49ff/0VxsbGeP36NUaPHl3gs97e3vj333/x2WeflXPUuoWJOhERERWoU6dOiImJwenTp7FixQqMGDECRkZGGh+ndu3acHJygkKhwP79+wtsk5KSAh8fHwiCAFdXV43HQJphYWGByZMnY/78+QCABw8e4P79+1qOSncxUSciIqIC6enpldtY48ePB/B/s+Zv2r9/PxQKBVxcXFC7du1i+1Mqldi1axf69OmD6tWrw9DQENWqVUOPHj2we/duFHYwe876ay8vLyQnJ8PDwwONGjWCqakpbG1tMXbsWISEhEjtX716ha+//hr169eHiYkJbGxsMHHiRDx//rzI+J48eYJp06bhvffeg4mJCSwtLfHhhx9i8eLFiI+PL/CZM2fO5Fmbf+PGDYwePRr29vYwMDCAi4sL/v33X6nNlStXioxh7NixEAQBLi4uRbZTR+4lK4mJifnuu7i45Nvr4OHhAUEQEBYWBiD778Sb6+Bzi42Nxffff48PP/wQlpaWMDQ0hI2NDd5//31MnToV/v7+Gn9f5Y1r1ImIiEjrhgwZghkzZiAgIAChoaH5lrfkJPDjx49HVlZWkX3FxMRg4MCBCAwMlK7J5XK8evUKp06dwqlTp+Dj44O9e/fC0NCwwD4UCgU++ugj3LlzB8bGxhAEAc+ePcPvv/8Of39/XLhwAUqlEt26dUNISAhMTU2hVCrx/PlzbN++HYGBgQgODi7weHhfX1+4uroiLS0NQPYsdHp6Om7cuIEbN25g27Zt8PPzQ6NGjQp9j/v378fIkSORkZEBS0tL6Otnp3QNGzaEs7Mzzp49iy1btqBNmzYFPh8bG4t9+/YBACZPnlzkz1Mdt27dApD9Ya9u3boqPWNubo7q1avj5cuXUCqVsLS0hImJSYFtnz59ig4dOiA8PBwAIJPJpD/j58+f486dO/j333/RtWtXzbwhLSn1jPrTp08xc+ZMNGnSBObm5tJflByxsbH48ccfsXTpUmRmZpZ2OCIiInoLmZmZYdiwYRBFEV5eXnnuhYaGIiAgAJaWlhg8eHCR/WRlZWHQoEEIDAxE8+bNcfjwYSQlJSEuLg6JiYnYuXMnrK2tcejQIXz99deF9uPh4YGEhAScPHkSSUlJ0tfVqlXDs2fP8PXXX2PkyJGQy+W4ePEikpKSkJiYiD179sDU1BSPHj3CihUr8vV7/fp1jBkzBmlpaejQoQNu376N+Ph4JCcn49ChQ6hRowYiIiLQt2/fAmeic4wbNw7du3fH/fv3oVAokJKSgq1btwIApk2bBgDw8fFBQkJCgc///vvvSE1NRZUqVYr9mZZEYmIitm/fjv/9738AgAkTJqBy5coqPTt79mxER0ejZs2aAIB169blWQMfHR0ttfXw8EB4eDgcHR1x+vRppKenIyYmBmlpaQgNDcWmTZuK3JisK0qVqJ86dQrNmjXDunXrcP/+fSQnJ+f7VZKVlRUOHDiA7777DseOHStVsNqQkJAADw8PNGvWDObm5pDL5WjdujVWrVqF9PT0UvW9b98+9O3bF7a2tjA0NISZmRkaNGiASZMmsSQRERG9c3KWv+zcuTNPPuHp6QlRFDF8+HCYmpoW2ccff/yBs2fPomHDhjhz5gw++eQT6RkzMzO4urri2LFjEAQBGzduxIsXLwrsJy0tDadOnUL37t0hk8mgp6eH7t27Y9myZQCyZ8XDwsJw+vRpKSE0MDDAsGHDMGvWLADZifKb5s+fj4yMDNSrVw8nT55Es2bNAGTPCPft2xdHjx6Fvr4+njx5gs2bNxf6Phs3boxDhw6hYcOG0rX33nsPADBo0CBYW1sjKSkJf/zxR4HP5yT1bm5uau872LNnD2xsbKSXlZUVLCwsMHHiRNSqVQs//vgjNm7cqFbfxblw4QIA4Mcff0TXrl2lZVp6enpwcHDA1KlTpT8rXab20peIiAgMGTIECQkJ6NevH1xdXTFp0iTExcXla+vu7o7g4GAcPXoU/fr1K7Zvd3d3dcPKQxAEbN++Xe3nw8LC4OLigtDQUACAqakp0tLSEBwcjODgYOzatQv+/v6wsrIqUb9paWkYOnQoDh8+LF0zNzdHeno6Hj58iIcPH2LHjh1YuXIlvvrqK7XjJyIi0iUdOnRA/fr18fDhQwQEBKBLly4QRRE7d+4EoFp+kPP/+9OmTctTOjC3li1bokmTJrh79y4CAgIwfPjwfG0GDx6MevXq5bves2dP6evJkyejSpUqBbZZsmQJnjx5gqSkJJiZmQEA4uLi4OfnBwCYM2dOgR86WrRogUGDBsHX1xe7d+/G7NmzC3wPc+bMKXQPgYGBASZMmIClS5diy5YtmDJlSp77ly5dwp07d6T3oK7U1FSkpqYWeC8mJgYvX75EcnIyLCws1B6jMJUqVQIAPHv2TON9VyRqJ+qrVq1CQkIChg0bJn1inD59eoFtc/5SX716VaW+vby8IAhCoRs9ipPzbGkS9czMTPTt2xehoaGoUaMGvL290a1bNyiVSuzduxeTJk3CjRs3MGbMGBw9erREff/4449Skv7pp5/i22+/hZ2dHZRKJW7cuIEvv/wS586dw6xZs+Dk5FRg/VEiIqK30fjx4zFv3jzs2LEDXbp0gb+/P8LCwtCoUaNilzJkZWVJhyZ5eHjgxx9/LLRtTEwMAEgbF99U2Nru6tWrS1+3bt262DZxcXFSon79+nUpt+nWrVuhsXXv3h2+vr64ffs2MjIyYGBgkK9Nhw4dCn0eyE7Aly9fjuvXr+P69ev48MMPpXs5s+nOzs5o0KBBkf0Uxc3NLc8yJVEUERsbi/Pnz2PevHlYs2YNTp8+DX9/f1SrVk3tcQryySef4OLFi/jmm2/w77//YtCgQWjfvn2BewJ0mdqJup+fHwRBwJIlS4ptW7t2bRgZGeXZJV0UV1dXrZ82tnPnTunT5v79+9GuXTsA2b+aGj58OJRKJUaNGoVjx47B39+/RJsVvL29AWT/B7JhwwbpukwmQ8uWLXHkyBHY29sjMTER+/btY6JORETvDFdXV3z33Xf4888/ER8fn2cTaXFy1igD2XvkVJGcnFzg9cJmgXPvxVOlTUZGhvR17mU2dnZ2hcZkb28PIHvSMCYmJk/in8Pa2rrQ54Hs6jU9e/bE8ePHsWXLFmkZTXx8PPbs2QMA+WbaS0sQBFSuXBl9+/ZFixYt0KRJE9y5cwdff/01duzYodGx5syZg1u3bsHX1xdbt27F1q1bIQgCmjRpgl69emHixIml+hBSUaidqIeHh8PExERaD1Ucc3NzKBQKldq+uYlEG3J+zda5c2cpSc9txIgRmD9/PkJCQuDt7V2iRD3n1zStWrUq8L5cLkf9+vVx/fr1IjeSEBERvW1sbW3Rs2dPHDt2DJs3b8Zff/0FPT09jB07tthnc1eDOX78OHr16lWWoWqVKqUzp02bhuPHj+OPP/7AqlWrYGZmhj/++ANJSUmoUqUKBg0aVGbx2dvbo1evXvD19YWvry+2bNmSr+BIaRgYGGDPnj349ttv8eeff+LcuXO4fPky7t69i7t372LNmjVYvny5tF9AV6m9mVQmk0GpVKrUNjMzE/Hx8Trz64jk5GScP38eANC7d+8C2wiCIP0DcPLkyRL1X6dOHQDAtWvXCryvUCjw8OFDAIUn80RERG+rnNnzBQsWICUlBb1791bpRNQqVapIyWBhS1q0Kfcs+NOnTwttl3NPX19f5YopBenTpw9q1qyJhIQEaZlyzrKXcePGlcnhVbk5ODgAAJKSkvDy5csyGeODDz7AokWL4O/vj7i4OJw+fRpOTk7IysqSZt11mdqJuoODA9LS0qT6lUUJDAxERkaGyrPv2nb//n3pQ0jTpk0LbZdzLzo6WlrrpoqcsklnzpzB9OnTERkZCSB7bdf169fxySefIDExEe3atcOYMWPUfRtEREQ6qV+/fqhSpYpUXU3VIhMGBgbS2vLcBRsqig8//BAyWXbqVdRhPKdPnwaQnYQWtD5dVXp6etJm0S1btkjr1YGyqZ3+ptwfRnLW6asi52dU0r2K+vr66Nq1K44ePQojIyOIoij9LHWV2ol6ziaIokoHAdlrs+bPnw9BEAqdna5ooqKipK+LWkOW+17uZ4ozffp0zJ07FzKZDBs3boS9vT0sLCxgbGyMli1b4vHjx/jmm2/g7+9f7K+20tLSEB8fn+dFRESkywwNDbFu3TrMmjULc+fOxSeffKLyszkJ6LFjx4otC12SSTZNqFSpklRg46effipwffytW7ewf/9+AMDIkSNLPeaECROgr6+PK1euSJXknJ2dUb9+/VL3XZScw6WA7LKRJVlVkdO2oEqCOXL2IhTEyMhIyp9ykn5dpXb0X331FQwNDbFq1apCK6tcv34d3bp1w+XLl2FhYYFPP/1U7UBzpKWl4Y8//sCnn36Kfv36oWvXrujSpUuBL3VPo8p9OEBR9Vpz3yvsQIGCyGQyLF26FDt27IC5uTmA7AMCcmYOUlNToVAokJSUVGxfS5cuhVwul145hwQQERFpQmxsLF69eiW9cn7jnJycnOe6pvdUjR49GitXrsTy5ctLNKs8ZswYdOvWDaIoYuDAgfjhhx/yTKYlJSUhICAA06dPl5ailqcffvgBBgYGePz4MXr27CkVrlAqlTh27Bj69OmDzMxM1K1bVyObPWvUqIH+/fsDgHRSq6Y3keamVCpx9epVDBgwAK9evQIAzJw5s0R95KxY2LdvX6Gbgh0cHDBv3jxcunQpT9L++PFjjB49GsnJyZDJZHnKaeoitVf1Ozg4YNu2bXBzc8PkyZPx7bffSptF27dvj7CwMERHR0MURejr68Pb2xtVq1YtVbAXLlzA8OHDERUVJZVfBP7vVyO5K8Xkvl/RvHr1CkOHDsWZM2fQvXt3LFy4EE2bNkVKSgouXryIr7/+Gps2bYKfnx8CAwOLnNWfN29env8A4uPjmawTEZHGtGjRosD13j/99BN++ukn6fs3S/Vpi56eHvbv34/Ro0fjyJEjWLBgARYsWABLS0vIZDIoFAopb9Dk5kZVffjhh/jtt9/g6uqKc+fO4f3334elpSXS09OlmuQ1a9bE4cOHpcm80po2bZo0S6/JTaR79uzBiRMnpO9FUUR8fHye2uozZ87E1KlTS9Tv5MmT8ccff+DChQuoVq0arK2tYWhoCADS2TbPnz/HsmXLsGzZMshkMsjlcqSkpEhjC4KAVatWoXHjxqV8l9pVqr+ho0ePhrW1NaZPn47Hjx9L13NqmAJAvXr1sHnzZnTp0qU0QyEiIgIff/wxFAoF3n//ffTq1QsrVqyAubk5vvzyS0RHR+Pvv//Gf//9h6pVq2Lq1Kkq7YguSO5yS4WVbXrzXkmK+bu5ueHMmTNwdnaWylwC2dVeBg4ciA4dOqBJkyb477//8M033+C3334rtC8jI6My3wxCRPTW8lCtGhnpFktLSxw+fBjHjx/Hzp07cfHiRTx//hyiKMLOzg6NGzdG586dMWzYMK3EN3z4cLRs2RIrV67E6dOn8fTpUxgYGKB58+YYOHAgvvzyS40W4OjSpQsqV66MmJgYjW4iLejAI1NTU9SvXx8dO3bExIkTC6ycVxwnJyccPXoUq1evxo0bN/D8+fN8BUxOnjyJgIAAnDt3DuHh4Xj+/DmA7LyzU6dOmD59+ltR3loQ1T1VKBdRFBEYGIjz588jKioKWVlZsLGxQYcOHdC5c2e1E+bcvvrqK6xbtw69e/fGkSNHIAgCZDIZbGxs8vxKa9OmTfj888/Rp08fHDx4UK2xrl27JlVbOXbsWKFr6zdu3Cgd8vT69WuVdmbfv39f+nR36NAh9O3bt8B2s2bNwurVq2FqaorExESVfzsQHx8PuVwOhUKhM1V2iIg0KTk5GQ8ePECDBg2KPW6e6F2QO6958OBBma9P11Xl+W+HqvmaRn7nIwgCnJ2d4ezsrInuCnTy5EkIgoBFixYVmbROmzYNsbGxWLBgAbZu3YpJkyaVeKxGjRpJ5Sfv3r1baKJ+9+5dAICNjY3K5ZPu3bsnfV23bt1C2+VUyElOTsaLFy8KPOyAiIiIqDi//PILgOyZdSbpukVntsKGh4dDJpPlOQIXgLQBM7fp06dDEAS118qZmppKR/PmXnuVmyiK8PPzAwD06NFD5b5z7z4uqsZrzq9wAGhsjRoRERG9W44dO4bff/8dADB79mwtR0MlpXai/uLFC/z888/YvXt3sW137dqFn3/+Wdr9qw5RFGFlZZUn0TUzM0N8fHy+Ops5FVD+/fdftcdzc3MDAAQEBODy5cv57u/duxf//fcfgOzjjlWV+4PGpk2bCmyTlJQEb29vAMD7779fotqjRERE9G57+vQpHB0dYW1tjY8//hhZWVn45JNPdKZMNv0ftRP133//HV999VWeTaSFuXXrFr766iv88ccf6g4HOzu7fCUQ7e3tkZWVhfv37+e5npycjLi4uCI3ghbHzc0NzZo1gyiKGDx4sHQwgVKpxN69e6UlNb17985XBtLDwwOCIEAQBGl3cg4HBwdpXfrhw4cxduxYPHnyBKIoIiMjAxcuXICLi4v0IUDXj74lIiKi8pWZmYmwsDC8evUK9vb2+PLLL1WaWKWKR+1E/dChQwCAoUOHFtvW1dUVoiiqvbkTAOrUqYP09HQ8efJEuta2bVsA+Q9dWr16NURRhKOjo9rj6evr49ChQ3B0dERkZCS6desGMzMzmJmZYdiwYYiPj0eLFi2wa9euEve9Y8cOaSfy77//jnr16sHc3FxachMcHAwAmDNnTolm64mIiIgcHR0hiiKUSiUiIiKwZs0aLqPVUWon6k+ePIGRkREaNmxYbNumTZvC2Ng4T5JdUi4uLhBFUTrlCgAmTpwIURSxYcMG9OnTB/Pnz8cnn3yChQsXQhCEUp/o5ejoiNu3b+P7779H06ZNIQgCDAwMpJJKly5dgpWVVYn7rVq1Ki5duoRt27ahZ8+eqF69OjIyMqCvr486depgzJgxCAoKwooVK0oVPxERERHpLrXLMxoZGcHS0hIvX75UqX21atWQmJiIlJQUdYZDaGgoxo8fjxYtWmD16tXS9a+//lo6cEEQBGm9upOTE06ePCkVyH9XsDwjEb3rWJ6RiNTxVpVntLS0RFxcHFJTU2FsbFxk29TUVMTFxZUqcXR0dERAQEC+68uXL0f37t3h4+ODiIgIyOVy9OrVC66urlo5cYyIiIiISBPUzmSbNGmCoKAgHDlyBEOGDCmy7eHDh5GVlaXSMhl1dOvWDd26dSuTvomIiIiItEHtNer9+vWDKIqYPXt2npNB3xQZGYnZs2dDEAQMGDBA3eGIiIiIiN4paifqU6dOhb29PSIiItC8eXOsWbMGjx49Qnp6OtLT0/Ho0SOsXr0aLVq0QEREBOzs7PDpp59qMnYiIiIioreW2ktfTE1NceDAAfTq1QuvXr3C7NmzCzzxShRFVK1aFYcOHVL54J6cw37kcjn69++f51pJsbwhEREREekitau+5Hj69CnmzZsHX19fZGRk5LlnaGiIESNG4H//+x/s7OxU7lMmk0EQBDRo0AD37t3Lc60kBEFAZmZmiZ7Rdaz6QkTvOlZ9ISJ1vFVVX3LY29vjt99+w6+//org4GBER0dDEATY2NigVatWMDExKXGftWrVgiAIsLW1zXeNiIiIiOhdoLH6haampnByctJIX6GhoSpdIyIiIiJ6W6m9mZSIiIiIiMqOxmbUU1NTERsbm2+d+ptq1aqlVv/h4eHQ09NTea17VFQUMjMz1R6PiIiIiMrGuHHjsHPnTri5ucHLy0vb4VRYpUrUk5OTsWLFCuzevRuPHz8utn1pNnc6OjqiRo0aiIyMVKl9hw4dEBER8c5tJiUiItU129lM2yFoxB23O2XS7+vXr3Ho0CH4+/vj+vXrCAsLQ2ZmJqpVq4ZWrVrBzc0NAwcOVLt/Dw8PLFq0SPp+9+7dGDFiRJHPfPzxxzh27Jj0fUhICBwdHdWOgUrOxcUFZ8+ezXddEASYm5ujTp066NatG2bMmAEHB4cS9+/l5YXQ0FC4uLjAxcVFAxHrLrUT9bi4ODg5OeGff/6BqoVjSllgpsTPl3Y8IiKid5mNjU2eCS9jY2MYGBggMjISkZGROHjwIHr37o19+/ZppEqGp6dnkYl6VFQU/Pz8Sj0OaYaBgQEqV64sfZ+RkYGYmBjcunULt27dwsaNG7F//3707t0737M1atRAgwYNUKNGjXz3vLy8pA8C73qirvYa9SVLluDu3bvQ19fHzJkzcfbsWTx69AghISFFvspLamoq9PU1trKHiIjonZOZmYk2bdpg48aNePLkCVJSUpCYmIiQkBBMmDABAHD8+HFMmTKlVONUrVoVZmZmOH36NJ4+fVpoO29vb2RlZXEGvYJo3749oqOjpdfr16+RkpICHx8fVKlSBSkpKRgzZgySkpLyPbt06VL8+++/WLp0qRYi1x1qJ+oHDhyAIAhYu3YtVq5ciU6dOqFu3bpwcHAo8lUeoqKi8PLlS1SpUqVcxiMiInob/f3337h8+TKmTZuGOnXqSNcdHR2xbds2KUH//fffERERofY4ZmZmGDJkCJRKZZHrlT09PQFkr2+misnY2BjDhw/H2rVrAQAxMTEIDAzUblA6TO1EPTIyEjKZDOPHj9dkPJLAwEAsXrxYegFAYmJinmtvvhYtWoSvvvoKnTt3hiiKaNu2bZnERkRE9C7o3LlzkfdzZtUBIDg4uFRj5eQThSXq586dw8OHD1GnTh2Vy0EfPXoUgwcPhp2dHYyMjGBlZQUnJyds2rQJ6enpBT7j4uICQRDg4eGBzMxMrFmzBi1atIC5uTmsra0xYMAA3Lp1S2qfnJyMH374AU2bNoWZmRmqVKmC4cOH48mTJ0XGFh0djTlz5qBJkyYwMzODmZkZmjRpgrlz5+L58+cFPhMaGgpBECAIAkJDQ/HkyRNMnjwZtWvXhpGRERwdHREbGwtTU1MIggBfX98iY1iwYAEEQUCdOnU0vly4efPm0teJiYn57o8bNw6CIOT50OXl5QVBEKRlL4sWLZLeb+73nSMlJQUrV65Eu3btYGVlBQMDA1SrVg2NGzeGm5sb9u/fr9H3pA1qrw2pXLkyUlNTYWxsrMl4JAEBAdIfUI6kpKQ8m04KI4oijI2NMW/evDKJjYiIiJAnB8jKyipVX05OTqhbty6ePHmCwMDAfMl47tn04g5ATElJgaurK/bt2ydds7S0hEKhQFBQEIKCguDt7Y1jx47BysqqwD4yMjLQq1cv+Pv7w9DQEAYGBnj58iUOHjwIf39/BAQEoHbt2ujevTtu3LgBY2NjCIKAmJgY+Pr64syZM7h69WqB1efOnj2LAQMGIC4uDkD2bxQA4N69e7h37x62bduGQ4cOoWPHjoW+xwsXLmDKlClITEyEqakpDAwMAABWVlYYNmwYdu7ciS1btmDYsGEFPp+VlSX9TCdOnKjxQyVzf5hp0KCBSs+YmJigevXqiImJQUZGBszMzGBubp6njZ6eHgAgISEBnTp1ksYRBAFyuRxxcXF49eoV7t+/j7Nnz2Lw4MEaekfaofaMeseOHaFQKFSuwlJSjo6OcHZ2hpOTk/Qfq4GBgfR9QS8XFxf069cP8+fPx+3bt9GmTZsyiY2IiIiAM2fOSF83a1a6Cjq5Z1d37NiR515SUhJ8fX0hk8lUWvYyefJk7Nu3D3Xq1MGuXbugUCigUCiQnJyMgwcPok6dOrh06RLc3d0L7WPjxo24efMm9u7di8TERCQkJODKlSuoU6cOEhMT8cUXX2DSpEmIjY2Fn58fkpKSkJiYiNOnT6NatWp48eIFvv3223z9RkRESEl648aNce7cOSQmJiIxMRGBgYFo0KABYmNj0b9//yJzrClTpqBJkya4evWqNPbJkycBANOmTQOQvXTpv//+K/D5Y8eOITIyEvr6+kX+HEoqLS0N+/btw1dffQUA6NGjB95//32Vnh0+fDiio6PRvn17AMDs2bPzrIGPjo5GzZo1AQDr1q3DrVu3ULlyZezfvx8pKSmIjY1FWloaIiMj4e3tjR49emjsfWmL2jPqX3/9NQ4ePIglS5Zg8+bNmowJAODm5gY3Nzfpe5lMhsqVKyMgIEDjYxEREVHJxMXFSRsBO3XqpPKsaVHc3NywcOFC7Nu3D+vXr5dmU319fZGYmIju3bujZs2aRS4rCQoKwu+//w5ra2ucOXNGSuyA7N8A9OvXDx9++CEaNmyIAwcO4ObNm3mWaeR+f0FBQXlmtVu3bo2tW7eia9euuHDhAkxMTHD79m3Uq1dPatO1a1csW7YMEyZMwJ9//omMjAxpthsAfvzxR8TFxcHKygr+/v6wsbGR7nXq1AmnT59GkyZNEBMTg6VLl2L9+vUFvs8qVarg9OnTeWac69evDwBo27Ytmjdvjps3b2Lr1q0FbtjcsmULAKBfv355YiiJCxcu5Hk2p+oLANSsWRNz586Fh4eHWn2rMjaQncwPGjRIui6TyWBra4uxY8di7NixZTJ2eVJ7Rr1ly5bw8vLCzp07MWHChEI/sWmKp6entDGBiIiItEepVGLs2LF49uwZjI2NC00mS6pmzZro1q2bNIOeI2eJhiozv9u3bwcAjB49Ok+Snpu9vb20/r6wco8dO3YscOmJs7MzjIyMAABDhgzJk6Tn6NmzJ4DsJTiPHj2SrouiKL2vqVOnFpgg29vbY+rUqQAAHx+fgt8kgM8++yzfspDccmbVvby88h1GGRkZiePHjwNAqSr2ZGRk4Pnz59IrJ0kHAIVCgZiYGCgUCrX7L0qlSpUAAM+ePSuT/isKtRP1OnXqYP78+dDT04OXlxfee+89VKtWDXXq1Cn0VbduXbUDdXNzK3SdFREREZWfL774AkeOHAEAbNiwQeWlDarI2VSas/zl8ePHCAoKgpWVFQYMGFDs8+fPnweQnbDb2NgU+jp9+jQAICwsrMB+Cls+q6enh6pVqwLInmEvSPXq1aWvY2Njpa9DQkKkZLZbt26Fvofu3bsDyD5wqrDS1h06dCj0eQAYNWoULCwsEB0djcOHD+e5t2PHDmRlZUlr7NXl7OwMURTzvBQKBU6fPo33338f27ZtQ5s2bfJ8WNGUTz75BACwfv16jBw5EgcOHMCrV680Po62qb30Jfeu2xyvX7/G69evC31G0xsViIiIqHzNnj1bmkFfs2aNRtc3A8DAgQNhZWWF8+fP49GjR1IVmJEjR6pUwCIqKgoAEB8fj/j4+GLbJycnF3jdwsKi0GdyzmkprE3uc1xyz2a/ePFC+trOzq7Q/u3t7fM8U7t27XxtrK2tC30eAMzNzTFmzBhs2rQJW7ZskZaHKJVK6bcOkyZN0nhuZmlpia5du6JNmzZo1qwZwsLCMGXKFPz9998aHWfUqFG4cuUKfvnlF/j4+Ei/fahXrx569OgBd3d3tGzZUqNjaoPaiXrOr6HKQs5/9DVq1MD//ve/PNdKQhAE6S8jERERlc7cuXOxatUqAMDKlSvx5ZdfanwMIyMjjBw5Ehs3bsS2bdvwxx9/AIDK5aBzqs9s2rRJWkLyNsqpflKUadOmYdOmTTh16hRCQ0Ph6OiIkydPIiwsDPr6+mVWYhvI/hAzfPhwrFixAgEBAXj27FmBp5CWxtq1a/HZZ59h7969CAoKwsWLF/H48WM8fvwYGzduxBdffKHzy6bVTtRzb/TUtJw6mg0aNJAS9ZxrqtT5zGnHRJ2IiEgz5syZg5UrVwIAVqxYgVmzZpXZWOPHj8fGjRuxdu1apKeno2nTpmjVqpVKz9rY2CAsLKzQJS3alHsW/OnTp4VuwM19OmtxM+dFadasGdq3b48LFy5g+/btWLJkCbZu3QoA6N+/v9qbSFWV+6DL0NBQjSfqQPYM+rx58zBv3jwolUpcuXIFy5cvx4EDB7Bu3Tp06dIF/fr10/i45UXtRL0subq6QhCEPH+gOdeIiIiofM2ePVuaSV+xYgXmzJlTpuO1atUKzZo1w507dwCU7LfqHTp0QFhYGI4cOVLhjqevXbs2KleujJiYGPj7+6Nr164FtstZP1+lSpUCl72UxLRp03DhwgXs2LEDU6dOldarT548uVT9qiL3B46cWvGqkMmyt1CW9BAmmUyGjz76SCrNGR4ejlOnTjFR17SCTiUr6khhIiIiKhu5k/SVK1eW6Ux6bsuXL4e/vz8AYMyYMSo/N3nyZPzxxx+4e/cuNm3aJFU/KUhSUhIMDAxgaGhY6nhVIQgChg8fjk2bNuHXX3/F559/nm9WOyoqCr/++iuA7HX5pTV06FB8+eWXiIqKwqhRo5CRkVHqTaSqSE1NxZ9//gkge718Scp3WlpaAoB0IFRB0tLSpOo7b9LT05P+THOSfl2lsURdFEXExsYiKSmpyE9ABZ3QRVQWkpKSpNJViYmJJfo0T9l05WfYbGfpDlopK8o0Je5NuQcAaPxrY8iMKu7/Ydxxu6PtEKgCyr0mffXq1dIhNuWhd+/e6N27d4mfc3Z2xvjx4+Hp6Ynp06fj4cOHmDFjBurUqQMgO8G7ceMG/vzzT+zYsQM3b97Ms3mzrH377bfYvXs3YmJi0K1bN2zZskU64Of8+fOYNGkS4uLiULlyZXzzzTelHs/IyAjjxo3DqlWrEBgYCKBsNpHmdu/ePXz77bd48OABAGD69OmFJtUFadq0KQ4ePIhjx45h7ty5BW68bdu2LZycnDBo0CC0bt1a+v+nqKgoLF26FI8fPwYA9OnTRwPvSHtKnagfOXIEP//8My5evFjozukcgiAgMzOztENSBeP4zVFth1AgZXqq9HWjBScgMyy+WoC2hBqP0nYIBUvP9aH7fzUAwwq6/Kx2xZwAkBnJ0NSrqbbDIFJLeHg4fvrpJwDZs5LLly/H8uXLC20/e/ZszJ49u7zCK9LmzZuhp6eHbdu2Ye3atVi7di3Mzc1hYGAAhUIBpVIptS3vZbX29vY4cOAA+vfvj3/++QcdOnSQksykpCQA2TXCDxw4UGRlmJKYOnUqVq9eDVEUNbqJ9M0DjwAgISEhTz44fPhwLFmypET9urm5YdWqVXj8+DFq1aqFatWqSRV/zp07B3t7e8TFxeGXX37BL7/8AkEQIJfLkZGRIf0MAeCrr76SatrrqlIl6jmftFVdQ1TStUZEpSEzNIbD10e0HYZOMzMUIC601HYYRGWGv0koXO5kVqlU4vnz50W2T0xMLOuQVGZoaIitW7fC3d0dW7ZsQVBQEKKiopCWlgZra2s0bNgQTk5OGDJkiMaS4ZJwdnbG/fv3sWrVKhw7dgyhoaEQBAGNGjXCxx9/jFmzZml0o2e9evXQvHlz3LhxQ6ObSHMOPMrN2NgYtWvXRtu2beHm5oZevXqVuN/33nsPAQEBWLp0KS5fvozXr19LE705/+vj44OTJ08iMDAQISEhiI6ORmZmJhwcHPDRRx9h8uTJ6NKlS+nfpJYJoprZ84kTJ9CnTx8YGBhg6dKl6N27N5o0aYJq1arh4sWLiI6OxqlTp/DLL79AJpPB09MTTZs2zbMDWB1nzpzB7t27cfv2bcTExOQ7bSs3QRCKPGb4bRQfHw+5XA6FQiGt8SprFXVGXZdU2Bl1HdGsgs6o65K3KWFNTk7GgwcP0KBBA5iammo7HCKti46ORs2aNZGZmQk/Pz/06NFD2yFVSOX5b4eq+ZraM+q//vorBEHAggULMHPmTOm6np6edBJp+/btMWHCBHTu3BkTJkzAzZs31R0OoijC3d0d3t7e0vfFYZUYIiIietdt3rwZmZmZqFevXplvIiXNUjtRv3LlCoDsDQm5vZlA29vbY/369ejduzeWL1+ONWvWqDXeL7/8gp07dwIAWrZsiX79+sHW1jbP6V9ERERE9H+Cg4OlDcEzZ87kJKaOUTvLff36NUxNTVG9enXpmp6eXoEbSrt37w5jY2McPXpU7UTd09MTgiBg4sSJUtkiIiIiIsrP0dERaWlpiI6OBgC0aNECEydO1HJUVFJq1wqztLSEgYFBnmtyuRyJiYl5dtwC2bvF9fX1ERkZqe5wePjwIQBg2bJlavdBRERE9C4ICwtDdHQ0bGxsMG7cOBw/fjxf3kYVn9qJup2dHeLj45Ga+n8l8OrXrw8guw5obo8ePUJiYmKplqkYGxujUqVKsLKyUrsPIiIioneBKIoQRRHPnj2Dp6dnnhUQpDvUTtTff/99iKKIGzduSNe6d+8OURTx7bffSr9qefnypVRYv1WrVmoH2qxZM8THx1eo8k9ERERERGVF7US9V69eEEURBw4ckK5Nnz4dlSpVwo0bN1CrVi3Y2dmhRo0aCAoKAgDMmTNH7UA/++wzZGVlYceOHWr3QURERESkK9RO1AcMGABPT0906NBBumZtbY2jR49KtTqfPXsGpVIJU1NTbNy4Ua2i9zmGDBmC6dOn4+uvv8Zvv/2mdj9ERERERLpA7UXjJiYmcHNzy3e9Xbt2ePLkCS5evIiIiAjI5XJ07Nix1IfvuLu7AwBMTU0xbtw4LFiwAK1bt4aFhUWhzwiCgO3bt5dqXCIiIiIibSiTIuR6enro2LGjRvv08vKCIAhSnfbw8HCEh4cX2DanHRN1IqJ3j56eHgAUeXI1EdGbcv7NqEhn9FScSIrh6urKIv1ERFQsQ0NDGBoaSkd0ExGpIj4+HoaGhhWqjKXOJOpeXl7aDoGIiHSAIAioVKkSXr9+jSpVqsDU1FTbIRFRBZecnIzY2FhUqVKlQk0MlypRF0URnp6e8PHxwe3btxEbG4vMzMxC2wuCUOR9IiIiTbCxsUFSUhIeP34MKyurAg/pIyLKyMhAfHw8YmNjYWxsDBsbG22HlIfaiXpiYiL69OmD8+fPS+vGiYiIKgI9PT3UrVsX0dHRiIuLw6tXr7QdEhFVUIaGhqhSpQpsbGykPS4VhdqJuoeHB86dOwc9PT2MGjUKPXv2RPXq1SvUAnwiInp36enpwc7ODra2tsjIyOBvdIkoH319fRgYGFSo5S65qZ1V7927F4IgYN26dfj00081GRO6dOkCAHBwcICnp2eeayUhCAL8/f01GhsREekWQRCkDaZERLpE7UT9xYsX0NfXx8SJEzUZDwDgzJkzAICGDRvmu1YSFfXTERERERFRcdRO1GvUqIHY2NgymaFYuHAhAKBq1ar5rhERERERvQvUTtR79uyJLVu24N9//80z860JBSXlTNSJiIiI6F0iU/fBefPmoXLlyvj88895+hsRERERkYapPaNeq1YtHDt2DMOGDUPLli0xa9YstGrVChYWFsU+R0RERERERStVLcUGDRqgb9++WL9+Pdzd3YttzwOPiIiIiIhUo3ai/urVK7i4uOD+/fsAoNKhRzwYiYiIiIhINWon6osWLcK9e/dgamqKWbNm8cAjIiIiIiINUjurPnz4MARBwI4dOzBs2DBNxkRERERE9M5Tu+rLixcvYGhoiMGDB2syHiIiIiIiQikSdVtbWxgYGEBPT0+T8RAREREREUqRqPfr1w9JSUkIDg7WZDxERERERIRSJOrfffcdbG1tMXXqVMTFxWkwJM15+fKltkMgIiIiIlKL2ptJ7969ix9//BFffPEFGjdujEmTJqFNmzbFHnjk5OSk7pB5LF++HM2bN0fnzp1haGiY7/5///2HXr164eHDhxoZj4iIiIioPKmdqLu4uEAQBACAQqHADz/8UOwzmjzwaN68eRAEAebm5hg1ahTmzp2L2rVrAwBu3ryJ3r1748WLFxoZi4iIiIiovKm99AXIPsCoJC+lUqmpuNGqVSvY2NggISEBv/76Kxo1aoTFixfjxIkTcHFxwfPnz2FlZaWx8YiIiIiIypPaM+qaTLrVceXKFQDAkydPsH37dqxfvx6LFi0CkP0B4r333sORI0e0GSIRERERkdrUnlEPDw9HeHg4UlNTNRlPidWtWxc//vgj1q5dK83cy2QyHD16FO+9955WYyMiIiIiUpfaibqjoyPq1KmDmJgYTcYjiYuLg7e3N/76669i2wYGBmLGjBkQBAEWFhYQRRHz5s0rk7iIiIiIiMqD2om6ubk55HI5bG1tNRmPxNfXF+PHj0dAQECR7a5cuYJPPvkEKSkpcHZ2xqNHj1CnTh38+eefuHr1apnERkRERERU1ko1o56cnIysrCxNxiM5ePAgAGDEiBGFtrl9+zb69OmDxMREdOvWDceOHYO1tTUWLFgAURSxY8eOMomNiIiIiKisqZ2oDxgwAOnp6Th27Jgm45E8ePAA+vr6aNOmTYH3Hz58iJ49eyImJga9evXCoUOHYGxsDADo378/BEHAmTNnyiQ2IiIiIqKypnai/vXXX6NevXqYOnUqbt++rcmYAABRUVEwNzeHvn7+wjRhYWHo3r07nj9/jk8++QQHDhyQknQAkMvlkMvliIyM1HhcRERERETlQe3yjPv378eUKVPg4eGBVq1aoVevXujQoQOsra2hp6dX6HOurq4q9W9gYIC4uDjExcWhUqVK0vXIyEh07doVERERGDBgAPbs2QMDA4N8z6enp2u9hCQRERERkbrUTtTHjRsnnUwqiiKOHj2Ko0ePFvmMIAgqJ+pNmzbFpUuXsGTJEqxatQoA8OjRI/Tp0wchISEYOnQodu3aVeCM+507d5CcnCydVFoaCQkJWLVqFfbv34+QkBDo6emhfv36GDFiBGbMmAFDQ8NS9R8dHY0NGzbg2LFjCAkJQUpKCqytrdGoUSO4uLhg1qxZBX4QISIiIqK3m9qJeq1ataREvSyMGTMGFy9exNq1a3H+/HnY2trCz88PKSkpEAQBkydPLjBJB4AffvgBgiCgY8eOpYohLCwMLi4uCA0NBQCYmpoiLS0NwcHBCA4Oxq5du+Dv76/2Cah79uzB5MmTER8fDwAwNjaGoaGhVKPez88PU6dOzfMbBSIiIiJ6N6idqOckr2VlypQp+Ouvv3D69GnpFFIA6NKlC86fP4/+/ftj1apVmDx5svSBISMjA99++y327t0LQRAwadIktcfPzMxE3759ERoaiho1asDb2xvdunWDUqnE3r17MWnSJNy4cQNjxowp9jcJBdm7dy9GjRoFpVKJyZMn44svvkDjxo0BZM/i37x5E3/99Rdn04mIiIjeUYIoiqK2gyhMZmYmNm/ejFOnTkFfXx8jRoyQlrzkLKGpUaMGPvroIwiCgHPnzuHFixcQRRHjxo0rVXnG7du3Y+LEiQCACxcuoF27dnnu7969G6NGjQIAnD59Gl27dlW572fPnqFJkyaIjY3FqlWrMHPmTLXjfFN8fDzkcjkUCgUsLS011m9RHL8p+QcVyivUeJS2Q9BpzWrX0nYIOu+O2x1th0BE9M5QNV+r0Il6UQ4cOAA3NzckJCQAyF7/nvNWxowZgx07dhS6NEYVTk5OCAoKQufOnfH333/nuy+KIurWrYuQkBC4urpi586dKvc9b948LFu2DC1atMC1a9c0uoSIibpuYqJeOkzUS4+JOhFR+VE1X1M/k33DP//8g+DgYLx48QIAYG1tjdatW0vLOTRtwIABCA0NhaenJ65cuYL4+HjUrVsXw4YNQ6dOnUrVd3JyMs6fPw8A6N27d4FtBEFAr169sGnTJpw8ebJE/Xt7ewPI/kBRluv8iYiIiEh3lTpR9/Pzw9y5c3H37t0C7zdr1gwrVqxAjx49SjtUPlZWVhpdNpLj/v37UmnHpk2bFtou5150dDRiYmJQuXLlYvsOCQlBVFQUAKBly5a4c+cOli5dioCAAMTExKBatWro0KEDPv/8c3To0EED74aIiIiIdJHaBx4BwPr16/Hxxx/j7t27EEURMpkM1tbWUi11URRx+/Zt9O7dGxs2bNBUzGUuJ5EGADs7u0Lb5b6X+5miPHz4UPr6/PnzaNWqFXbv3g2FQgFjY2NERkbC19cXnTp1wpIlS4rtLy0tDfHx8XleRERERKT71E7Ub926hS+//BJKpRJt2rTBsWPHkJiYiGfPnuHZs2dISEjAsWPH0K5dO4iiiC+//LJMTjAtCznr3oHskoyFyX0v9zNFiY2Nlb5esGABbG1tcerUKSQmJkKhUOCff/6Bi4sLRFHE999/jz///LPI/pYuXSqdxCqXy1GzZk2V4iAiIiKiik3tRH316tVQKpXo27cvzp07h169esHIyEi6b2RkhF69eiEwMBB9+/ZFVlYW1qxZo5GgdVnu01JFUcT+/fvRrVs3yGTZfxSNGzfG4cOHYWNjAwBYtGhRkf3NmzcPCoVCekVERJRd8ERERERUbtRO1M+ePQtBELBu3Tro6ekV2k5PTw9r164FAAQEBKg7XLmysLCQvk5OTi60Xe57uZ9Rte+uXbviww8/zNfG3Nwc06dPBwDcvn0bz58/L7Q/IyMjWFpa5nkRERERke5TO1F//vw55HI5HB0di21bu3ZtVKpUqciEsyKxtbWVvo6MjCy0Xe57uZ8pSu517Y0aNSq0Xe5qOWFhYSr1TURERERvD7UTdRMTEyQnJyMzM7PYtpmZmUhOToaJiYm6w5WrRo0aSUtRCqtmk/uejY2NShVfgOwEvKjfQOTIXd6eJRyJiIiI3j1qJ+qNGjVCRkYG9u3bV2zbvXv3Ij09vcgZ5IrE1NRUKo144sSJAtuIogg/Pz8AKFHpSWNjYzg5OQHILgNZmHv37gHITtJV+a0FEREREb1d1E7Uhw4dClEU8emnn8Lf37/QdqdPn8ann34KQRAwbNgwdYcrd25ubgCy19Vfvnw53/29e/fiv//+AwC4urqWqO/x48cDAPz9/XH9+vV89xMTE7Fx40YAQNu2bVGtWrUS9U9EREREuk/tRH3atGlo0qQJ4uLi0KNHD3Ts2BEeHh7YunUrtm7dioULF6Jjx47o2bMnFAoFmjRpgmnTpmky9jLl5uaGZs2aQRRFDB48WPowolQqsXfvXkyaNAlA9smlXbt2zfOsh4cHBEGAIAgIDQ3N1/fo0aPRpk2bPH3nVIO5f/8++vXrh+joaMhkMvzvf/8r2zdKRERERBWS2ieTGhkZwc/PD4MGDcKVK1dw4cIFXLx4MU+bnHXWbdu2xf79+2FoaFi6aHN5+fIlwsLCkJycLC0l0SR9fX0cOnQInTt3RmhoKLp16wZTU1MolUqkpqYCAFq0aIFdu3aVuG+ZTIaDBw+ia9euuHfvntS3gYEBFAoFAMDAwAAbNmxAly5dNPq+iIiIiEg3lOpkUltbW1y4cAE+Pj4YOHAg7O3tYWhoCENDQ9jb22PgwIHYs2cPzp8/r3JVlOIcOnQIH374IWxsbNC2bdt8iWxsbCx69eqFXr16SUmvuhwdHXH79m18//33aNq0KQRBgIGBAVq2bImVK1fi0qVLsLKyUqtvGxsbXL9+HStXrkTr1q1hYGCAlJQUODo6wt3dHdevX5dm7YmIiIjo3SOIucuLFCI+Ph4ymQzm5ublEVOhli1bhvnz5+eriJKVlZWn3cCBA3Ho0CFs2bIFEyZMKO8wtSo+Ph5yuRwKhaLcaqo7fnO0XMZ5m4Uaj9J2CDqtWe1a2g5B591xu6PtEIiI3hmq5msqzahXqlQJDRo0yHNt8eLFWL16demiLIFLly5h/vz50NfXx5o1a/Dq1StUr169wLZjxoyBKIo4depUucVHRERERKRJKi99eXPi3cPDAytXrtR4QIVZt24dAGDevHn44osviqxb7uzsDAC4ceNGucRGRERERKRpKiXqhoaGSEpKKutYinT+/HkAwGeffVZs26pVq8LMzAxRUVFlHRYRERERUZlQKVG3t7dHYmJivqou5enFixewsLBA1apVVWpvZGSE9PT0Mo6KiIiIiKhsqFSesU+fPli/fj06d+6M999/X9pUGhMTU6LygYIgFHk4UlHMzMyQkJCArKws6OnpFdk2MTERcXFxPCiIiIiIiHSWSon6okWLEBQUhFu3biE4OFi6np6ejjNnzqg8mCAIJQ4wR4MGDXD58mXcvn0bLVq0KLLtgQMHoFQq0bx5c7XHIyIiIiLSJpUSdSsrK1y7dg2nTp3CnTt3kJycDA8PD5ibm2PWrFllHSMAoF+/frh06RKWLl0KX1/fQts9ffoU33zzDQRBwODBg8slNiIiIiIiTVOpjnpBZDIZbGxsym3DZmJiIho1aoSoqCiMHj0ac+fORffu3fHixQukpqYiNDQUhw8fxvLly/Hy5Us0aNAAt2/fhoGBQbnEV1GwjrpuYh310mEd9dJjHXUiovKjar6m0ox6Qdzc3FCpUiV1Hy8xc3NzHD58GD179sTvv/+OXbt2SfeMjY2lr0VRhK2tLQ4cOPDOJelERERE9PZQuY56QRQKBUJCQjQVS7GaN2+OW7duYfz48TAyMoIoinleBgYGGDduHIKDg/Md0EREREREpEvUXvqir68PfX19pKSklGqTqLrS0tJw7do1REVFISsrCzY2NmjdujVMTU3LPZaKhEtfdBOXvpQOl76UHpe+EBGVnzJf+mJtbY3U1FStJOlAdp309u3ba2VsIiIiIqKypvbSlzZt2kChUCAyMlKT8RTK3d0dM2fOVLn93LlzMWHChDKMiIiIiIio7KidqH/xxRcAgIULF2osmKJ4eXnBx8dH5fZ79+6Fl5dX2QVERERERFSG1E7UO3fujDVr1mDnzp0YNmwYrl+/rsm4Sk3NpfdERERERBWC2mvU69SpAwAwMDDA/v37sX//fpiYmKBKlSrQ09Mr8BlBEPDkyRN1hyyRV69evfMbS4mIiIhId6mdqIeGhua7lpycjOTk5EKfKY+NpwqFAtu2bUNycjLef//9Mh+PiIiIiKgsqJ2oe3p6ajKOfBYtWoTFixfnufb8+fNCZ+vfJAgCBg8eXBahERERERGVuVKdTFrWcq8zFwRB5XXnhoaGGDt2LL755puyCo2IiIiIqEypnaiXtXHjxsHFxQVAdsLepUsXVK5cGfv37y/0GZlMBktLS9SvXx8mJiblFCkRERERkeZV2ETdwcEBDg4O0ve1atVC9erV4ezsrMWoiIiIiIjKR6kT9adPn2L16tXw8/NDWFgYUlNTkZmZKd2PjY3Fpk2bIAgC5syZA3199YYsaPMqEREREdHbqlSJ+qlTpzBs2DDEx8dL68ffrOxiZWWFAwcO4Nq1a2jSpAn69etXmiGJiIiIiN4JaifqERERGDJkCBISEtCvXz+4urpi0qRJiIuLy9fW3d0dwcHBOHr0qMYS9RcvXuDp06dISkoqcpOpk5OTRsYjIiIiIipPaifqq1atQkJCAoYNGwYfHx8AwPTp0wts27NnTwDA1atX1R1Osn79evz8888qHZwkCEKeZThERERERLpC7UTdz88PgiBgyZIlxbatXbs2jIyMEBISou5wAIARI0Zg7969KpdpVLUdEREREVFFI1P3wfDwcJiYmOC9995Tqb25uTmSkpLUHQ4+Pj7w9fWFpaUl9u3bJ/VlY2ODzMxMPH36FJ6enqhXrx6qVq0Kf39/KJVKtccjIiIiItImtRN1mUymciKcmZmJ+Ph4WFpaqjscvLy8pBn8QYMG5amTLpPJYGtrCzc3N1y/fh01a9bEgAED8PjxY7XHIyIiIiLSJrUTdQcHB6SlpSE8PLzYtoGBgcjIyFB59r0gN27cAACMGTMmz/U3PyyYm5tj/fr1SEhIwPLly9Uej4iIiIhIm9RO1Lt16wYA2Lx5c5HtMjIyMH/+fAiCgN69e6s7HOLi4mBhYYFKlSpJ1wwMDApcTtOuXTuYmpri9OnTao9HRERERKRNaifqX331FQwNDbFq1Sps3769wDbXr19Ht27dcPnyZVhYWODTTz9VO9AqVarkq9FeqVIlJCcnF1gSEgCio6PVHo+IiIiISJtKtfRl27ZtyMrKwuTJk1G9enXExsYCANq3bw87Ozu0bt0aQUFB0NfXh7e3N6pWrap2oHZ2doiPj0diYqJ0rVGjRgCAgICAPG2vX7+O5ORkmJqaqj0eEREREZE2qZ2oA8Do0aNx/Phx1K1bFy9fvkR6ejpEUcSlS5fw7NkziKKIevXq4cSJE6U+6OjDDz8EkLcW+8cffwxRFDF79mxcvXoVGRkZCA4OhpubGwRBQIcOHUo1JhERERGRtqhdRz1H9+7d8eDBAwQGBuL8+fOIiopCVlYWbGxs0KFDB3Tu3Bl6enqlDvTjjz/G1q1bsXfvXnTu3BkAMG3aNPz8888ICQnBRx99JLUVRREGBgaYP39+qcclIiIiItKGUifqQPYJoM7OznB2dtZEdwXq06cPAgIC8ixnMTc3x99//41x48bh4sWL0vVatWphw4YNaNu2bZnFQ0RERERUlkqcqB8/fhzr16/H1atXER8fj8qVK+Ojjz7CV199hU6dOpVFjAAAfX39Aj8IvPfeezh//jyePn2KiIgIyOVyNGrUKN/GUyIiIiIiXVKiNeqLFy/GJ598ghMnTuDVq1dIT09HdHQ0Dh48iM6dO2PTpk1lFWex7O3t0a5dOzRu3JhJOhERERHpPJUT9cuXL2PRokXS+u927dph6NChaN26NQRBgFKpxFdffVVmp4G6u7tj5syZKrefO3cuJkyYUCaxEBERERGVNZUT9c2bN0MURbz33nu4ceMGzp8/jz179uDy5cu4cOECbGxskJGRgW3btpVJoF5eXvDx8VG5/d69e+Hl5VUmsRARERERlTWVE/ULFy5AEARs3LhRql+eo02bNli+fDlEUcSFCxc0HqQ6RFHUdghERERERGpTOVGPjIyEnp5eoZVdunXrBgCIiorSTGSl9OrVKx54REREREQ6S+VEPTk5GVWrVoW+fsGFYmrUqAEASElJ0UxkalIoFFi1ahWSk5NRt25drcZCRERERKQujdRRz01TS04WLVqExYsX57n2/PlzlQ9PEgQBgwcP1kgsRERERETlTeOJuiblTvoFQVD5Q4ChoSHGjh2Lb775pqxCIyIiIiIqUyVK1GNiYtClSxe12wiCAH9/f5XGGjduHFxcXABkJ+xdunRB5cqVsX///kKfkclksLS0RP369WFiYqLSOEREREREFVGJEvWMjAycOXOmyDbp6en52uTMhpfkICIHBwc4ODhI39eqVQvVq1cvdDMrEREREdHbROVE3cnJSasnfoaGhmptbCIiIiKi8qZyol7cTLq2ZWZm4s6dO5DJZHj//fe1+qGCiIiIiKi0VC7PqG0PHjzA4sWL4e3tne/emTNnUKtWLbRq1QoffvghateuXWEOXiIiIiIiUofOJOre3t5YtGgRwsPD81yPjY3F4MGDER0dDVEUIYoiwsPD8fHHHyM6OlpL0RIRERERlY7OJOp///03AOSrjb59+3bExsbCwcEBp06dwrlz59CsWTPEx8fj559/1kaoRERERESlpjOJemRkJADkO2304MGDEAQBS5cuRdeuXdG+fXts2rQJoijCz89PG6ESEREREZWaziTqL1++RKVKlWBoaChdy8jIwNWrV6Gvr4++fftK19u3bw99fX08fvxYG6ESEREREZWaziTqMpkMSUlJea7duHED6enp+OCDD2BmZpbnnlwuR1paWnmGSERERESkMTqTqNvb2yMjIwP379+Xrh09ehQA0KFDhzxtRVFEfHw8qlatWq4xEhERERFpis4k6s7OzhBFEbNmzcKLFy9w8+ZNbN68GYIgoE+fPnnaPnjwABkZGbC1tdVStEREREREpaMzifqsWbNgZGQEPz8/1KhRAy1btsTLly/xwQcfoHv37nnanjhxAgDQpk0bbYRKRERERFRqOpOoN2jQAIcOHULt2rUhiiIEQUD37t1x8ODBfG09PT0BAJ07dy7vMImIiIiINEJfE50olUpcu3YNYWFhSE5Ohqurqya6zad79+54/PgxXr58CQsLCxgbG+drk5GRIdVPb926dZnEQURERERU1kqdqP/yyy/44Ycf8OrVK+la7kQ9NjYWnTp1QmZmJs6ePYvq1auXdkhUq1at0HsGBgZwdnYu9RhERERERNpUqqUv06dPx5dffinNcAuCkK+NlZUVPvzwQzx69Ah79+4tzXBERERERO8MtRP1EydOYNOmTTA3N8dff/2FuLi4Qme6R40aBVEUcfr0abUDJSIiIiJ6l6idqOeURly8eDH69+9fZNt27doBAO7cuaPucERERERE7xS1E/XLly8DANzd3YttK5fLYWlpiejoaHWHIyIiIiIdlJSUBEEQIAhCvlPmqWhqJ+oxMTGQy+WwsLBQbSCZDEqlUt3hiIiIiIjeKWon6paWloiPj0dGRkaxbWNiYqBQKFC1alV1hyMiIiIieqeonag3a9YMoihKS2CKsnv3boiiiFatWqk7HBERERHRO0XtRH3IkCEQRREeHh5FLmm5desWvvvuOwiCgJEjR6o7HBERERHRO0XtRH3SpElo3LgxAgIC0L17dxw5cgRZWVkAgEePHuHUqVP4/PPP0b59eygUCnz00UcYOnSoxgInIiIiInqbqZ2oGxgY4OjRo6hfvz4CAgLQv39/vH79GgDQsGFD9OrVCxs2bEBKSgqaNWuG/fv3F3ggUkWXkJAADw8PNGvWDObm5pDL5WjdujVWrVqF9PR0jY41depUaVe0o6OjRvsmIiIiIt2iX5qHHRwccO3aNaxatQo7duxAWFhYnvt2dnaYNGkSZs2aBTMzM5X71dPTK01YEkEQkJmZqfbzYWFhcHFxQWhoKADA1NQUaWlpCA4ORnBwMHbt2gV/f39YWVmVOtaAgABs2bKl1P0QERHRu6fZzmbaDqFQyrT/WyLdZlcbyIzUnicuU3fcKt55P6X+SZmammLBggUICQnB06dPceXKFVy8eBEhISGIiIjA999/X6IkHQBEUdTYS12ZmZno27cvQkNDUaNGDZw6dQpJSUlITk6Gj48PLCwscOPGDYwZM0btMXIkJydj0qRJ0NfX54ZbIiIiIgJQyhn1N9na2sLW1rbU/QQEBGggmtLZuXOndJLq/v37pdNVZTIZhg8fDqVSiVGjRuHYsWPw9/dH165d1R5r/vz5ePLkCebPn4+nT58iODhYI++BiIiIiHSX2on6//73P4wdOxa1atXSZDwAAGdnZ433WVI7d+4EAHTu3FlK0nMbMWIE5s+fj5CQEHh7e6udqF+6dAk///wz6tevj++++w5Tp04tVdxERERE9HZQe+nLggULUKdOHXTp0gVeXl5ISEjQZFxalZycjPPnzwMAevfuXWAbQRDQq1cvAMDJkyfVGictLQ3u7u4QRRFbtmyBsbGxegETERFRmUhKSpIKPSQlJWk7HHrHqJ2o16pVC0qlEmfOnMGECRNgY2OD0aNH48SJE0XWVdcF9+/fl95D06ZNC22Xcy86OhoxMTElHmfx4sW4f/8+JkyYUCF+i0BEREREFYfaS19CQ0MRGBgIb29v7N+/HwqFAj4+PvDx8UH16tUxatQojB07Fh988IEm45WIoojY2FgkJSUVuWlUnaU5UVFR0td2dnaFtst9LyoqCpUrV1Z5jBs3bmDFihWoXr06fvrppxLHmCMtLQ1paWnS9/Hx8Wr3RUREREQVR6mqvjg5OWHbtm2Ijo7Gnj170KdPH+jp6SE6Ohpr1qzBhx9+iA8++ACrVq1CdHS0RgI+cuQIevToAUtLS1SrVg2Ojo6oXbt2ga86deqoNUbuZTympqaFtst9ryRLfzIzM+Hu7o7MzEz8/PPPqFSpklpxAsDSpUshl8ulV82aNdXui4iIiEjTZEYyNPVqiqZeTStsacaKSiM/LSMjIwwdOhSHDx9GVFQU1q1bh1atWkEURdy5cwdz585FzZo1C13vraq5c+eif//+OH36tDSTXtSroi7BWbZsGW7evIlPPvkEw4YNK1Vf8+bNg0KhkF4REREaipKIiIiItEnjH2uqVq2KGTNm4PLly/j3338xf/581KpVC1lZWWpvugSAEydOYOXKldDX18fKlSvxzz//AACqVauGx48f49y5c1i4cCEqV66MqlWr4vDhwwgJCVFrLAsLC+nr5OTkQtvlvpf7maLcu3cPS5Ysgbm5OTZu3KhWfLkZGRnB0tIyz4uIiIiIdF+Z/v4hISEB8fHxRSa7qvr1118hCAIWLFiAmTNnolGjRgCyTzGtU6cO2rdvj4ULF+LmzZuQy+WYMGECjIyM1Bordy34yMjIQtvlvqdq/fjp06cjPT0d8+fPh5WVFRITE/O8ck5SFUVRupaRkaHW+yAiIiIi3aXxRD0iIgJLly5F48aN0aZNG6xfvx4vX76EoaEhBgwYoHa/V65cAQBMmjQpz/U3N5La29tj/fr1ePHiBZYvX67WWI0aNYJMlv2juXv3bqHtcu7Z2NiovJE0Z5Z/3rx5sLCwyPfatWsXACA8PFy6tmHDBrXeBxERERHpLo0k6omJifD09ESXLl1Qu3ZtfPfdd/j3338hiqKUrEdFRWH//v1qj/H69WuYmpqievXq0jU9Pb0CZ+u7d+8OY2NjHD16VK2xTE1N0aFDBwDZS24KIooi/Pz8AAA9evRQaxwiIiIiosKoXZ5RqVTCz88Pv/32Gw4ePIjU1FRpdtvBwQFjxoyBq6sr3nvvPY0EamlpKS0LySGXy6USjWZmZtJ1mUwGfX39IpetFMfNzQ1BQUEICAjA5cuX0bZt2zz39+7di//++w8A4OrqqnK/oaGhRd4fN24cdu7cCQcHh2LbEhEREdHbS+0ZdTs7O3zyySfYs2cPUlJSYGFhAXd3d5w5cwYhISFYsmSJxpL0nPHi4+ORmpoqXatfvz4ASKeI5nj06BESExOhr6/25xC4ubmhWbNmEEURgwcPhr+/P4DsDyh79+6VluD07t0bXbt2zfOsh4eHdIoZk20iIiIiUofaifrz588hk8nQq1cv7N69G9HR0di2bRucnJw0GZ/k/fffhyiKuHHjhnSte/fuEEUR3377rVSn/eXLl5g0aRIEQUCrVq3UHk9fXx+HDh2Co6MjIiMj0a1bN5iZmcHMzAzDhg1DfHw8WrRoIa0pJyIiIiLSJLUT9dWrVyMyMhJHjx7F8OHDYWxsrMm48unVqxdEUcSBAweka9OnT0elSpVw48YN1KpVC3Z2dqhRowaCgoIAAHPmzCnVmI6Ojrh9+za+//57NG3aFIIgwMDAAC1btsTKlStx6dIlWFlZlWoMIiIiIqKCCOKbZVMqqJSUFPj6+sLKygr9+vWTrl+8eBEjR45EeHi4dM3MzAwrV67ElClTtBGqVsXHx0Mul0OhUJRbTXXHb9TbtEv/J9R4lLZD0GnNatfSdgg6747bHW2HQFQhJSUlwdzcHEB28Yzce+IqkmY7m2k7BJ1Xnv8Oqpqvqb+Iu5yZmJjAzc0t3/V27drhyZMnuHjxIiIiIiCXy9GxY0ce/ENEREREOk2lRD0wMBBAdtnCnHXfOddKqizWsOvp6aFjx44a75eIiIiISFtUStRdXFwgCAIaNmyIf/75J8+1khAEIV+JRSIiIiJ4yLUdQcHSc60Q/l8NwLBkuU+54RLAt5LKS19EUYRSqcx3rSR0ZDk8EREREZHWqZSov5mgF3atrImiCE9PT/j4+OD27duIjY0tcoaeM/hEREREpKt0ZjNpYmIi+vTpg/Pnz3NmnoiIiIjeejqTqHt4eODcuXPQ09PDqFGj0LNnT1SvXr1Up48SEREREVVUame5MpkMNWrUQGRkpErta9eujYiICLWXouzduxeCIGDdunX49NNP1eqDiIiIiEhXqH0yKVC+m0lfvHgBfX19TJw4Ue0+iIiIiIh0RakS9ZJIT0+HTKb+cDVq1ICpqSkMDQ01GBURERERUcVULol6XFwcXrx4ASsrK7X76NmzJ+Lj4/Hvv/9qMDIiIiIioopJ5TXqt2/fxs2bN/NcS0lJgbe3d6HPiKKIuLg47Nu3D0qlEi1atFA70Hnz5mHfvn34/PPPcfToURgYGKjdFxERERFRRadyov7XX39h8eLFea7Fx8dj/PjxxT4riiIEQcDMmTNLHuH/V6tWLRw7dgzDhg1Dy5YtMWvWLLRq1QoWFhbFPkdEREREpGtUTtQrVaqUJ+kNCwuDTCaDvb19oc/IZDJYWlqiadOmmDx5Mjp16lSqYBs0aIC+ffti/fr1cHd3L7Y9DzwiIiIiIl2lcqL+xRdf4IsvvpC+l8lkqFatGkJCQsoksDe9evUKLi4uuH//PgDVKsjwYCQiIiIi0lVq11FfuHAhzM3NNRlLkRYtWoR79+7B1NQUs2bN4oFHRERERPRWK1WiXp4OHz4MQRCwY8cODBs2rFzHJiIiIiIqbzozHf3ixQsYGhpi8ODB2g6FiIiI3hFmhgLEhZbaDoPeURpJ1C9cuIBz587h6dOnSEpKKnRtuCAI2L59u1pj2Nra4sWLF9DT0ytNqEREREREOqFUifqjR48watQoXL9+Pc/1nHKMBV1TN1Hv168f1q1bh+DgYLRq1UrtmImIiIiIdIHaJ5O+fv0aXbp0wbVr12BtbY2hQ4dCFEUYGxtjzJgx6Nq1K8zNzSGKIqpUqQI3Nze4urqqHeh3330HW1tbTJ06FXFxcWr3Q0RERESkC9SeUV+7di0iIyPRtm1b+Pv7w9TUFL6+vpDL5dJppUlJSVi8eDF++uknmJiYYOPGjWoHevfuXfz444/44osv0LhxY0yaNAlt2rQp9sAjJycntcckIiIiItIWtRP1o0ePQhAE/PjjjzA1NS2wjZmZGZYvX4709HT8/PPP6Ny5M4YOHarWeC4uLtJyGoVCgR9++KHYZ3jgERERERHpKrWXvjx58gSCIOQ7bTQ9PT1f22+++QYAsGXLFnWHA5C9zr0kL6VSWarxiIiIiIi0Re0Z9YyMDFhZWeU5cMjU1BQJCQn52lavXh1yuRy3b99Wdzgm3URERET0TlF7Rt3W1hbJycl5rlWvXh2ZmZn477//8lzPyMhAfHw8FAqFusMREREREb1T1E7UHRwckJqaiqdPn0rXWrduDQD4/fff87T18vKCUqmEnZ2dusMREREREb1T1E7Uc9amnzlzRro2duxYiKKIH374AdOnT8fWrVvx2Wef4bPPPoMgCBgwYEBp4yUiIiIieieovUZ96NCh2LlzJ/z9/TFmzBgAwMcff4wRI0bAx8cHmzdvltqKoohGjRrh+++/V6nvnPKOcrkc/fv3z3OtpEpTu52IiIiISFvUTtSbNGmCkJCQfNd37dqFzp07Y8+ePYiIiIBcLkevXr0wa9YsyOVylfoeN24cBEFAgwYNpEQ951pJCILARJ2IiIiIdJLaiXphBEHApEmTMGnSJLX7qFWrFgRBgK2tbb5rRERERETvAo0n6poQGhqq0jUiIiIioreV2ptJiYiIiIio7Kg0ox4YGKixAZ2cnNR6Ljw8HHp6eiqXeIyKikJmZiZq1aql1nhERERERNqkUqLu4uKikfXhgiAgMzNTrWcdHR1Ro0YNREZGqtS+Q4cOiIiIUHs8IiIiIiJtUnmNuiiKpR6stH2U9HlNxExEREREpA0qJepKpbKs49C41NRU6OtXyL2yRERERETFeis3k0ZFReHly5eoUqWKtkMhIiIiIlJLhZ1yDgwMxJkzZ/JcS0xMxOLFiwt9RhRFxMXF4dixYxBFEW3bti3jKImIiIiIyobGEvWXL18iLCwMycnJald2yS0gIACLFi3Ks4k1KSkJixYtKvZZURRhbGyMefPmlToOIiIiIiJtKHWifujQIXh4eODWrVsA8ld2iY2NxciRIwEAe/bsgVwuV6lfR0dHODs7S9+fPXsWBgYGaNeuXaHPyGQyWFpaomnTpnBzc0O9evXUeUtERERERFpXqkR92bJlmD9/fpHVVaysrGBiYoJDhw5h3759mDBhgkp9u7m5wc3NTfpeJpOhcuXKCAgIKE3IREREREQ6Qe3NpJcuXcL8+fOhr6+PNWvW4NWrV6hevXqBbceMGQNRFHHq1Cm1A/X09MTatWvVfp6IiIiISJeoPaO+bt06AMC8efPwxRdfFNk2ZwnLjRs31B0uz+w6EREREdHbTu1E/fz58wCAzz77rNi2VatWhZmZGaKiotQdLo/09HScOnUKwcHBePHiBQDA2toarVq1Qvfu3WFoaKiRcYiIiIiItEXtRP3FixewsLBA1apVVWpvZGSEhIQEdYeTrF+/HosWLUJMTEyB9ytXrozvv/8eM2bMKPVYRERERETaonaibmZmhoSEBGRlZUFPT6/ItomJiYiLi0O1atXUHQ4AMHHiRHh6ekqbV+3t7WFnZwcAiIyMxNOnT/H69Wt8+eWXuHHjBnbs2FGq8YiIiIiItEXtzaQNGjRAVlYWbt++XWzbAwcOQKlUonnz5uoOh927d2PHjh0QRRFjxozBw4cPER4ejosXL+LixYsIDw/Ho0eP4OrqClEUsXPnTvzxxx9qj0dEREREpE1qJ+r9+vWDKIpYunRpke2ePn2Kb775BoIgYPDgweoOh40bN0IQBMyYMQPe3t4F1kivW7cuvLy8MGPGDIiiiI0bN6o9HhERERGRNqmdqH/22Wews7PD/v374erqirt370r3MjIy8OjRI6xevRotW7ZEVFQU6tevX6rKLbdv34YgCPj++++Lbfv9999DEATcuXNH7fGIiIiIiLRJ7TXq5ubmOHz4MHr27Inff/8du3btku4ZGxtLX4uiCFtbWxw4cAAGBgalCrZSpUqoUqVKse2qVKmCSpUqISsrq1TjERERERFpi9oz6gDQvHlz3Lp1C+PHj4eRkRFEUczzMjAwwLhx4xAcHIwGDRqUKtAGDRpAoVAgMTGx2LaJiYmIj48v9ZhERERERNpSqkQdAGxsbLB9+3bExsbi3Llz8PX1xe7duxEQEICYmBjs2LEDNjY2pQ7U3d0dWVlZ+OWXX4ptu379emRlZcHd3b3U4xIRERERaYPaS1/eZGRkhPbt2xd6PyMjA7/++qtKByQVZOrUqTh79iwWLFiA9PR0zJo1C+bm5nnaJCcnY+XKlViyZAlGjBiBKVOmqDUWEREREZG2aSxRL0xWVha2b9+O//3vf4iMjFQ7UXd3d4eJiQksLCywePFi/PTTT2jVqlWeOurBwcFISUmBXC6HsbFxgTPqgiBg+/btpXpPRERERERlTa1EPTk5GY8ePUJWVhZq164NKyurfG1yapkvWbIEoaGhEEURgiCoHaiXlxcEQZAOO0pOTkZgYGCBbePi4rBz506pLQDpWSbqRERERKQLSpSoKxQKfP755/D19UV6ejqA7AS4X79+2LBhA2rUqAEAOHPmDGbMmIF79+5JyXH//v0xf/58tQN1dXUtVaJPRERERKRLVE7UMzMz0b17d1y7di3PTLUoijh48CAePnyI69ev45dffsHXX38NpVIJPT09DB8+HPPmzUOTJk1KFaiXl1epniciIiIi0iUqJ+o7d+5EcHAwAKBLly7o1asXRFGEn58f/v77b9y/fx9TpkzBzp07IQgCXF1d8f3336NOnTplFjwRERER0dtK5UR97969EAQBkyZNwubNm6Xrc+bMweTJk7Ft2zZ4e3vDysoKf/75J5ydncskYCIiIiKid4HKifqdO3cAAN99912+ewsWLMC2bdsAAMuWLSvzJP358+fYt28fgoOD8eLFCwCAtbU1WrdujcGDB6N69eplOj4RERERUVlTOVF//fo1TE1NYW9vn+9ezZo1YWpqipSUFPTr10+jAeaWlZWFBQsWYPXq1cjIyAAAab28IAjw9vbGzJkzMWvWLCxevBh6enplFgsRERERUVlSOVFPT09H5cqVC71vYWGBlJSUMp3NdnV1hY+PD0RRhJGREVq1aiV9cHj69CmCg4ORlpaGZcuWITw8HL/99luZxUJEREREVJZk2g5AVQcOHMDu3bshiiJmzpyJZ8+eISgoCLt378bu3bsRFBSE6OhozJ49G6Io4o8//sChQ4e0HTYRERERkVp0JlHfvn07BEHA/PnzsXLlSlSqVClfG7lcjhUrVmD+/PkQRRFbt24t/0CJiIiIiDSgRIn68+fPoaenV+ArZ1NnYff19PSgr6/WQagAgKtXr0Imk2H27NnFtp09ezZkMhmuXr2q9nhERERERNpUosw590FH5S02NhZyuRxyubzYtjntYmNjyyEyIiIiIiLNUzlRX7hwYVnGUSwrKyu8fv0a8fHxsLS0LLKtQqGAQqFA1apVyyk6IiIiIiLN0plEvXXr1jh27BjWrFlTbCxr1qyBUqlEq1atyik6IiIiIiLN0pnNpOPHj4coiliyZAkWLFiAxMTEfG0SEhLw3XffYcmSJRAEARMmTCj1uAkJCfDw8ECzZs1gbm4OuVyO1q1bY9WqVUhPT1erz8jISGzcuBFDhw5FvXr1YGJiAhMTE9SuXRsjR47E33//Xeq4iYiIiEi3CaI2F56X0IgRI+Dr6wtBEGBsbIzWrVvDzs4OwP/VUU9NTYUoihg+fDh2795dqvHCwsLg4uKC0NBQAICpqSmysrKQlpYGAGjRogX8/f1hZWWlcp8RERFwcHDIs97f1NQUoigiJSVFuubu7o4tW7aU+NCm+Ph4yOVyKBSKYpcIaYrjN0fLZZy3WajxKG2HoNOa1a6l7RB03h23O9oOgd51HsXvQaPC8d/B0ivPfwdVzdd0ZkYdAH777TfMnDkTenp6SElJQWBgIHx8fODj44OgoCCkpKRAT08Ps2bNgre3d6nGyszMRN++fREaGooaNWrg1KlTSEpKQnJyMnx8fGBhYYEbN25gzJgxJeo3KysLoiiia9eu2LlzJyIjI5GUlITExET8888/6N+/PwBgx44d8PDwKNV7ICIiIiLdpVMz6jmioqKwf/9+BAcHS2Uhra2t0apVKwwePBi2tralHmP79u2YOHEiAODChQto165dnvu7d+/GqFHZs6CnT59G165dVepXoVDgyZMn+PDDDwu8L4oi+vTpgxMnTsDc3BwvX76EsbGxynFzRl03cUa9dDiTVHqcUSet44x6qfDfwdKriDPq6hc21yJbW1vMmDGjTMfYuXMnAKBz5875knQgexnO/PnzERISAm9vb5UTdblcXmiSDgCCIMDd3R0nTpxAYmIi7t+/jxYtWqj3JoiIiIhIZ+nU0pfykpycjPPnzwMAevfuXWAbQRDQq1cvAMDJkyc1On7uGfSsrCyN9k1EREREuoGJegHu378PpVIJAGjatGmh7XLuRUdHIyYmRmPjnzlzBgBgaGiI+vXra6xfIiIiItIdOrn0paxFRUVJX+dUlSlI7ntRUVGoXLlyqccOCQnB5s2bAQDDhw8vdp15WlqaVIUGyF7zRERERES6jzPqBUhISJC+NjU1LbRd7nu5n1FXSkoKhg4diuTkZFStWhXLli0r9pmlS5dCLpdLr5o1a5Y6DiIiIiLSPibqFURmZiZGjRqFa9euwcDAALt27VKpes28efOgUCikV0RERDlES0RERERljUtfCmBhYSF9nZycXGi73PdyP1NSWVlZGD16NA4cOAB9fX388ccf6NGjh0rPGhkZwcjISO2xiYiIiKhi4ox6AXLPZEdGRhbaLvc9dWu3Z2VlYcyYMfD19YWenh5+//13DBkyRK2+iIiIiOjtwUS9AI0aNYJMlv2juXv3bqHtcu7Z2NiotZE0Zybdx8dHStKHDx+uXtBERERE9FZhol4AU1NTdOjQAQBw4sSJAtuIogg/Pz8AUHmZSm5ZWVkYNWoU9uzZIyXpI0aMUD9oIiIiInqrMFEvhJubGwAgICAAly9fznd/7969+O+//wAArq6uJeo7Zybd19cX+vr62LVrF5N0IiIiIsrjrUzUw8PDpZe63Nzc0KxZM4iiiMGDB8Pf3x8AoFQqsXfvXkyaNAlA9smlXbt2zfOsh4cHBEGAIAgIDQ3Ncy9nTfqePXukjaNc7kJEREREb3orq77Url0bACAIAjIzM9XqQ19fH4cOHULnzp0RGhqKbt26wdTUFEqlEqmpqQCAFi1aYNeuXSXq9/z58/Dx8ZHimzFjBmbMmFFo+3Xr1jGRJyIiInoHvZWJuiiKGunH0dERt2/fxsqVK/Hnn38iJCQEBgYGaNKkCUaOHIkZM2bA0NCwRH0qlUrp64yMDDx//rzI9ikpKWrFTkRERES67a1M1BcuXKixviwsLLBo0SIsWrRI5Wc8PDzg4eFR4D0XFxeNfZAgIiIiorcXE3UiIiIiogrordxMSkRERESk63QmUQ8MDMSlS5dUbn/lyhUEBgaWYURERERERGVHZ5a+uLi4oEaNGoiMjFSp/fDhwxEREaF21RciIiIiIm3SmRl1oOTVXLhpk4iIiIh0lU4l6iWRlJQEAwMDbYdBRERERKSWtzJRf/DgAV69egVra2tth0JEREREpJYKu0b94MGDOHjwYJ5rCoUC7u7uhT4jiiLi4uIQFBQEQRDQqVOnsg6TiIiIiKhMVNhE/ebNm/Dy8oIgCNJa85SUFHh5ean0fLVq1VhPnYiIiIh0VoVN1Js3bw43Nzfp+507d8LExATDhg0r9BmZTAZLS0s0bdoUgwcPRqVKlcohUiIiIiIizauwiXr//v3Rv39/6fudO3dCLpfD09NTi1EREREREZWPCpuovykgIACGhobaDoOIiIiIqFzoTKLu7Oys7RCIiIiIiMrNW1mekYiIiIhI1+nMjHpJeHt7S1+7urpqMRIiIiIiIvW8lYn6uHHjIAgCBEFgok5ERP+vvXsPjrK6/zj+2c2VzZ1LRBCxUkEQsJTgEEOS3ahFsXScYgemWChTwbZSK9RqoVzCoGIr0ulMFcXKxQ6WKWIHBhKoJoEEGCgRBREL7TBJQArhUsLmtrmd3x/8sgazG5KQZPdJ3q+ZzCR5zjnPdzNw9pMnz3MOAFhStwzqkrxrrwMAAABWZJmgnp+fL0kaPXr0DddHz8vL64KKAAAAgM5jmaDudDoVEhKi0tLSG7ZlhRgAAABYnWWCelxcnEJCQpSQkBDoUgAAAIBOZ5nlGb/5zW/K7XbL4/EEuhQAAACg01kmqE+bNk21tbX629/+FuhSAAAAgE5nmaD+y1/+UsnJyZo7d66ysrICXQ4AAADQqSxzj/rLL7+stLQ0ffbZZ5o8ebLuuecepaSkKDExUSEhIX77LVmypAurBAAAADqGZYJ6ZmambDabd330Y8eO6fPPP79hP4I6AAAArMgyQT0tLU02my3QZQAAAABdwjJBfffu3YEuAQAAAOgylnmYFAAAAOhJCOoAAABAECKoAwAAAEHIMveot0V+fr7387S0tABWAgAAALRPtwzqTqdTNptNNptNdXV1gS4HAAAAaLNuGdQleddbBwAAAKyoWwb1devWBboEAAAA4KZYJqh7PB5FRES0qu3MmTM7uRoAAACgc1kmqMfHxys5OVlOp1Mul0vjx49XWFhYoMsCAAAAOoVlgrrH49Hu3bu1Z88eLVu2TJGRkbr//vvlcrnkcrl03333KSQkJNBlAgAAAB3CMkF9//79ys3NVV5envbv36+qqirl5OQoNzdXkhQVFaUJEyZ4g/vYsWNls9kCXDUAAADQPpYJ6uPHj9f48eO1cOFC1dTU6MCBA97gfvDgQZWXl2vnzp3atWuXJCk2NlZpaWnaunVrgCsHAAAA2s4yQb2p8PBwpaWlKS0tTZmZmaqqqtLevXuVl5en7OxsHTlyRGVlZdq+fXugSwUAAADaxR7oAm5WQ0ODjhw5okOHDumf//ynTp48GeiSAAAAgJtmySvqn376qXJzc5Wbm6uCggKVl5dLurbJUVRUlCZOnCiXy6WMjIwAVwoAAAC0j2WC+htvvKHc3Fzt2bNHly9f9u48GhkZ6V2yMSMjQ/fdd59CQy3zsgAAAACfLJNo586dK5vNptDQUCUnJysjI0Mul0v3339/qzdCAgAAAKzCcveoh4WFyeFwyOFwKCoqik2PAAAA0C1Z5or6Sy+9pLy8PO3bt08fffSRcnJyJEkxMTFKS0tTRkaGMjIyNHr06ABXCgAAANw8ywT1BQsWaMGCBaqtrdWBAweUl5ennJwcHTx4UNu3b9eOHTskSX369JHT6fQG96FDhwa4cgAAAqOiokLR0dGSpPLyckVFRQW4IgBtYZmg3igsLEypqalKTU3VkiVLVF1d7V1DPS8vT4WFhdqyZYu2bNkim82murq6QJcMAAAAtJnlgvrXRUZGyuVyyeFwKDIyUrW1tTp8+LB3VRgAAADAiiwb1A8fPuxdS33v3r2qqKiQJG9ADw8PV3JyciBLBAAAANrNMkH9+PHj3mC+Z88eXblyRdJXwTw0NFRJSUneZRtTUlIUGRkZwIoBAACA9rNMUB81apSkr4K53W7XmDFjvBsdpaam8pAMAAAAug3LBHVjjEaNGuUN5unp6YqLiwt0WQAAAECnsExQv3Dhgvr06RPoMgAAAIAuYZmdSQnpAAAA6EksE9QBAACAniQog/qrr76qqqqqDh2zsLBQ2dnZHTomAAAA0FmC8h71F154QatWrdLzzz+vWbNmKT4+vt1j7d27V6+88oqys7O1dOlSPfLIIx1XKAAAku74zY5Al+BTQ0219/Phi3fKHh68yxYXBW9pQMAE5RX1hQsX6urVq3ruued066236vHHH9eWLVtUWlp6w761tbU6dOiQFi9erCFDhig9PV1ZWVkaN26cHnvssc4vHgAAAOgAQXlF/cUXX9TPfvYzLVy4UO+9954++OAD/f3vf5ckDRo0SPfee6/69eun3r17KyIiQv/73/90+fJlnTp1SkeOHFFNTY2ka0s6DhkyRMuXL9e0adMC+ZIAAACANgnKoC5JAwcO1IYNG7RixQqtWbNGa9eu1ZkzZ1RSUqKSkhLZbLZmfZruUvroo4/qqaee0sSJE322BQAAAIJZ0Ab1RgMGDFBmZqYyMzN17Ngx5efn6+DBgzp79qwuXLig6upq9enTR/369dOIESOUlpamlJQUxcTEBLp0AAAAoN2CPqg3NXLkSI0cOVI///nPA10KAAAA0KmC8mFSAAAAoKcjqAMAAABBiKAOAAAABCGCOgAAABCELPUwKQAAaD17eKQGv7A90GUAaCeuqAMAAABBiKB+A263W5mZmRo1apSio6MVFxencePG6bXXXvPugNpe58+f169+9SsNGzZMvXr1Uu/evZWamqo///nP3s2bAAAA0DNx60sLiouL5XQ6VVRUJElyOBzyeDwqLCxUYWGhNm7cqJycHCUkJLR57I8//lgTJ07UpUuXJEnR0dFyu93au3ev9u7dq/fff1/btm1TeHh4R74kAAAAWARX1P2oq6vT5MmTVVRUpFtvvVUffvihKioqVFlZqU2bNikmJkaffPKJnnjiiTaPXVZWpu9+97u6dOmS7r77bh06dEhut1sVFRX605/+pLCwMO3atUvPPvtsx78wAAAAWAJB3Y8NGzbos88+kyRt2bJFDz74oCTJbrdr6tSpeuuttyRJWVlZysnJadPYK1eu1Llz59SrVy9lZWUpKSlJkhQeHq6nn35ay5YtkyStWbNGJ0+e7KiXBAAAAAshqPuxYcMGSZLL5VJycnKz49OmTdM3vvENSdK7777bprEb2zcdo6lf/OIXio6OVn19vTZu3NjW0gEAANANENR9qKys1L59+yRJjzzyiM82NptNDz/8sCTpH//4R6vHPnHihEpKSlocOzo6WqmpqW0eGwAAAN0HQd2HL774Qg0NDZKkkSNH+m3XeOzcuXO6fPlyq8Y+duxYs/4tjX38+PFWjQsAAIDuhVVffDh79qz384EDB/pt1/TY2bNn1bt37w4f++rVqyovL1d0dLTPdh6PRx6Px/t1WVmZt19XafBUdtm5uqurNpbjvBn1VfWBLsHyunLO6I6YB28e8+DNYR68eV05Dzae60bLcRPUfXC73d7PHQ6H33ZNjzXt0xlj+wvqK1as8D582tSgQYNaVQ+CQ1ygC7C8LwJdgOXF/Yx/hQgs/gXeLObBmxWIedDtdisuzv95CeoWt2DBAs2fP9/7dUNDgy5fvqw+ffrIZrMFsDJ0F1evXtWgQYN0+vRpxcbGBrocAOhyzIPoaMYYud1uDRgwoMV2BHUfYmJivJ9XVvr/c2bTY037tGVsf//hWzt2RESEIiIirvtefHx8q2oB2iI2NpY3KAA9GvMgOlJLV9Ib8TCpD01/u/nyyy/9tmt67Ea/EbV37NjYWL+3vQAAAKD7Iqj7MHz4cNnt1340TVdp+brGY/3792/Vg6TS9Su9tGbsESNGtGpcAAAAdC8EdR8cDodSUlIkSTt37vTZxhijXbt2SZK+853vtHrsoUOH6vbbb29x7IqKChUUFLR5bKAzREREaOnSpc1usQKAnoJ5EIFCUPdj5syZkqS8vDwdPHiw2fHNmzfr1KlTkqQZM2a0elybzeZtv2nTJhUVFTVr8/rrr6u8vFwhISGaPn16O6oHOk5ERIQyMzN5gwLQYzEPIlAI6n7MnDlTo0aNkjFGU6ZMUU5OjqRrq6ps3rxZs2fPlnRtd9EHHnjgur6ZmZmy2Wyy2Ww+g/hzzz2n/v37q7KyUo8++qg+/vhjSVJNTY1Wr16txYsXS5LmzJmjoUOHduKrBAAAQLBi1Rc/QkNDtW3bNrlcLhUVFenBBx+Uw+FQQ0ODqqurJUljxozRxo0b2zx2XFyctm/frokTJ+r48eNKSkpSTEyMqqurVVtbK+naLS9/+MMfOvQ1AQAAwDq4ot6CO+64Q0ePHtWSJUs0cuRI2Ww2hYWFaezYsVq5cqUOHDighISEdo09duxYff7555o3b57uuusu1dbWKioqShMmTNDbb7+t7Oxs/sQGAADQg9nMjfYuBQAAANDluKIOdGN33HGHbDab1q9fH+hSAEAS81JHaXwezul0BqQ/ugb3qAMWtX79ehUVFcnpdDLRAggKzEtAxyKoAxa1fv167dmzR5L8viEOGTJEkZGRrdqmGABuFvNS1+nbt6+GDRvm3ZsF3RNBHejGGpcVBYBgwbzUMebOnau5c+cGugx0Mu5RBwAAAIIQQR2wmPXr18tms3n/vLxs2TLvBltf32jrRg9tVVRUaOnSpRo+fLh69eqlxMRETZo0yXvFqzUPfe3YsUNTpkzRwIEDFRERoYSEBKWlpWn16tWqqanx2cfpdMpmsykzM1O1tbV67bXXlJSUpPj4eNlsNu3evbu9Px4AAcC8dL3a2lqtWrVK3/rWtxQVFaXevXvL6XTq/fffb3aur2v8ee3evVulpaWaP3++hg4dKofDIZvN5m3XmodBs7Oz9dBDDyk+Pl7R0dG699579fvf/967ZwuCH7e+ABbTq1cv3XLLLbp8+bJ3/f3o6Ojr2oSEhNxwnNLSUrlcLh0/flySFBYWptraWmVnZ2vnzp164403WuxfVVWlGTNmeN94JCk2NlZlZWUqKChQQUGB3n33XWVlZfndb6C6ulpOp1P79+9XaGioYmJirnsjAmANzEtfqaio0KRJk5Sfny/p2uuOiIhQfn6+9uzZowULFrRqnP/85z+aNm2azp8/r8jISIWFhbW6BulakF+2bJn36/j4eB0/flwvvPCCduzYoZSUlDaNhwAxACwpPT3dSDJLly7122bw4MFGklm3bl2zYw8//LCRZHr16mXeeecdU11dbYwxpqSkxEydOtWEh4cbh8Pht/8TTzxhJJk777zTbNy40ZSVlRljjKmqqjJbt241d955p5FkHnvsMb+1R0dHm+joaLNu3TpTWVlpjDHm4sWL5tKlS23/gQAIOOYlY5566ikjydjtdvO73/3OuN1uY4wxFy5cMM8884yRZOLj4/3+nCR56xg2bJjJyckx9fX1xhhjTpw44W23dOlSI8mkp6c3G2Pr1q3ecX7wgx+YkpISY4wxlZWV5vXXXzfh4eHeGnz1R/AgqAMWdTNviAUFBd5J/C9/+UuzfvX19cblcnnbfL1/fn6+kWQSExO9bwBfd/r0aRMVFWUkmU8++cRn7ZLMtm3bWvNyAVhAT5+XiouLjd1uN5LM8uXLfbaZOXOm9zwtBfXY2Fhz+vRpv+dqKaiPGDHCe6wx5Df15ptves9DUA9u3KMO9ECbN2+WdO1ez+nTpzc7brfbtWjRIr/933nnHUnS9OnTNWjQIJ9tbrvtNrlcLknSrl27fLa55557NHny5DbVDqB76g7z0pYtW9TQ0CCHw6F58+b5bLN48eJWjfWjH/1It912W5trOHr0qPfWoUWLFslubx71Zs+erYEDB7Z5bHQ97lEHeqDDhw9LktLS0vzee5mSkqLQ0FDV1dU1O7Zv3z5J194Y33vvPb/nKSsrkyQVFxf7PQcASN1jXmp8DUlJSYqKivLZZsiQIRo0aJBOnz7d4ljtraOwsFCSFBoaqtTUVJ9t7Ha7nE6nNm7c2K5zoOsQ1IEe6MKFC5KkAQMG+G0TERGhvn376ty5c82OnT17VpJ09epVXb169Ybnq6ys9Pn9xMTE1pQLoAcI9nmpf//+Pr8/depU/fGPf5TUutcgSQMHDrxhUG/v/FhaWirp2oZIERERftu152o9uh5BHejB2rvCSn19vSRp9erV+ulPf9ru87dmFQgAPUuwzkvnz5/3+f3GK/RNdcTqVcyPkFhHHeiR+vXrJ+mrK1C+eDweXbx40eexxitL/v50DABtFezzkrm2AEezj6brubfmNUjSl19+2Sk1Sl9dib948aLfNeM7uwZ0HII6YFGNDwgZY9rc99vf/rYkeTcn8WXfvn0+7wOVvrp3cvv27W0+N4Duq6fPS42vobCwUBUVFT7bnDp16oa3vdyMpKQkSVJdXZ0KCgp8tmloaGBjOYsgqAMWFRsbK0m6cuVKm/s+/vjjkqSioiKfD10ZY/Tyyy/77T9nzhxJ0rFjx7R69eoWz1VRUdHiVR0A3UdPn5e+//3vy263q6Kiwnvf+te99NJLHX7epkaPHq3hw4d7z9XQ0NCszdq1a3XmzJlOrQMdg6AOWNTIkSMlSVlZWW3+E2ZqaqoeeughSdeW6Vq/fr08Ho8k6cyZM5o+fboKCgrkcDh89k9PT9esWbMkSU8//bTmzZunU6dOeY97PB4dOHBAzz//vAYPHux9uAlA99bT56XBgwfrJz/5iSRpyZIlWrlypcrLyyVJly5d0vz587V27VrFx8d3+LmbavxlIC8vTz/84Q+9oby6ulpvvvmm5s6d2+k1oIMEagF3ADfn5MmTJjIy0rsD3i233GIGDx5sBg8e7N0ko6UdAP/73/+au+++27vpRVhYmHenOrvdbtasWWNuv/12I8n89a9/bdbf4/GYJ5980ttf/7+TXkJCgnfDj8aPM2fOXNe3NZuiALAe5iVj3G63mTBhgvc8ISEhJiEhwdhsNiPJLFq0yKSlpRlJZsWKFc36N/bLy8tr8TwtbXhkjDG//e1vr3u9CQkJJjQ01EgyqampZsGCBWx4ZAFcUQcs6q677lJeXp6+973vqV+/frp06ZKKi4tVXFzs9x7Opvr3769Dhw5p8eLFGjZsmOx2u0JDQzVp0iTl5uZq9uzZ3tUMfF15CQ8P19tvv639+/frxz/+sYYMGaL6+nqVl5crMTFRTqdTS5Ys0dGjR9lYA+ghmJek6Oho5eTk6NVXX9Xo0aMVHh4uY4zS09P1wQcfaPny5d5bgzrzqvaLL76o7du3KyMjQ7GxsfJ4PBo+fLheeeUV5eTkKDw8vNPOjY5jM6YdT3wA6Pb+/e9/a+jQoZKkkpISvzv9AUBX6Q7zUnl5ufr06aOamhrl5+f73ZQIkLhHHYAfK1askCSNGDHCkm+GALqf7jAvrVq1SjU1Nerdu7fGjRsX6HIQ5AjqQA/1r3/9S08++aTy8/Pldruv+/6sWbO0bt06SdJvfvObQJUIoIfpDvOS2+3WtGnTtHPnzutWvykuLtavf/1rZWZmSpKeffZZRUZGBqZIWAa3vgA91KeffqoxY8Z4v46Li1Ntbe1122o/88wzfpcYA4CO1h3mpStXrighIcH7dUxMjCRd94vHlClTtGnTJoWGskE8WkZQB3oot9utNWvW6KOPPtKJEydUWlqquro6JSYmKjk5WXPmzNEDDzwQ6DIB9CDdYV6qq6vTW2+9pQ8//FDHjh3ThQsXVFVVpb59+yopKUkzZszQlClTZLPZAl0qLICgDgAAAAQh7lEHAAAAghBBHQAAAAhCBHUAAAAgCBHUAQAAgCBEUAcAAACCEEEdAAAACEIEdQAAACAIEdQBAACAIPR/+bMp2/goazcAAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "def maybe_spec_map(id: str):\n", - " spec_map = {\n", - " '4x3.95': '4x3',\n", - " 'cheese.95': 'cheese',\n", - " 'paint.95': 'paint',\n", - " 'shuttle.95': 'shuttle',\n", - " 'example_7': 'ex. 7',\n", - " 'tmaze_5_two_thirds_up': 'tmaze',\n", - " 'tiger-alt-start': 'tiger'\n", - " }\n", - " \n", - "# spec_map |= prisoners_spec_map\n", - " \n", - " if id not in spec_map:\n", - " return id\n", - " return spec_map[id]\n", - "\n", - "groups = normalized_df.groupby(split_by, as_index=False)\n", - "means = groups.mean()\n", - "means['init_improvement_perf'].clip(lower=0, upper=1, inplace=True)\n", - "means['final_mem_perf'].clip(lower=0, upper=1, inplace=True)\n", - "\n", - "std_errs = groups.std()\n", - "std_errs['init_improvement_perf'] /= np.sqrt(len(seeds))\n", - "std_errs['final_mem_perf'] /= np.sqrt(len(seeds))\n", - "\n", - "# SORTING\n", - "sorted_mean_df = pd.DataFrame()\n", - "sorted_std_err_df = pd.DataFrame()\n", - "\n", - "for spec in spec_plot_order:\n", - " mean_spec_df = means[means['spec'] == spec]\n", - " std_err_spec_df = std_errs[std_errs['spec'] == spec]\n", - " sorted_mean_df = pd.concat([sorted_mean_df, mean_spec_df])\n", - " sorted_std_err_df = pd.concat([sorted_std_err_df, std_err_spec_df])\n", - " \n", - "means = sorted_mean_df\n", - "std_errs = sorted_std_err_df\n", - " \n", - "num_n_mem = list(sorted(normalized_df['n_mem_states'].unique()))\n", - "\n", - "group_width = 1\n", - "bar_width = group_width / (len(num_n_mem) + 2)\n", - "fig, ax = plt.subplots(figsize=(8, 6))\n", - "\n", - "specs = means[means['n_mem_states'] == num_n_mem[0]]['spec']\n", - "# spec_order_mapping = [spec_plot_order.index(s) for s in specs]\n", - "spec_order_mapping = np.arange(len(specs), dtype=int)\n", - "\n", - "xlabels = [maybe_spec_map(l) for l in specs]\n", - "x = np.arange(len(specs))\n", - "\n", - "init_improvement_perf_mean = np.array(means[means['n_mem_states'] == num_n_mem[0]]['init_improvement_perf'])\n", - "init_improvement_perf_std = np.array(std_errs[std_errs['n_mem_states'] == num_n_mem[0]]['init_improvement_perf'])\n", - "\n", - "ax.bar(x + (0 + 1) * bar_width,\n", - " init_improvement_perf_mean,\n", - " bar_width,\n", - " yerr=init_improvement_perf_std,\n", - " label='Memoryless')\n", - "\n", - "for i, n_mem_states in enumerate(num_n_mem):\n", - " curr_mem_mean = np.array(means[means['n_mem_states'] == n_mem_states]['final_mem_perf'])\n", - " curr_mem_std = np.array(std_errs[std_errs['n_mem_states'] == n_mem_states]['final_mem_perf'])\n", - " ax.bar(x + (i + 2) * bar_width,\n", - " curr_mem_mean,\n", - " bar_width,\n", - " yerr=curr_mem_std,\n", - " label=f\"{int(np.log2(n_mem_states))} Memory Bits\")\n", - "\n", - "ax.set_ylim([0, 1.05])\n", - "ax.set_ylabel(f'Relative Performance\\n (w.r.t. optimal {compare_to} & initial policy)')\n", - "ax.set_xticks(x + group_width / 2)\n", - "ax.set_xticklabels(xlabels)\n", - "ax.legend(bbox_to_anchor=(0.5, 0.62), framealpha=0.95)\n", - "# ax.set_title(\"Performance of Memory Iteration in POMDPs\")\n", - "\n", - "downloads = Path().home() / 'Downloads'\n", - "fig_path = downloads / f\"{results_dir.stem}.pdf\"\n", - "fig.savefig(fig_path, bbox_inches='tight')" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "id": "d6db67eb", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
specn_mem_statesseedinit_improvement_perffinal_mem_perf
0tiger-alt-start22034.50.7164350.716435
1tiger-alt-start42034.50.7178850.717885
2tiger-alt-start82034.50.7183400.718340
3tiger-grid22024.50.0468050.372141
4tiger-grid42024.50.0623310.417175
\n", - "
" - ], - "text/plain": [ - " spec n_mem_states seed init_improvement_perf \\\n", - "0 tiger-alt-start 2 2034.5 0.716435 \n", - "1 tiger-alt-start 4 2034.5 0.717885 \n", - "2 tiger-alt-start 8 2034.5 0.718340 \n", - "3 tiger-grid 2 2024.5 0.046805 \n", - "4 tiger-grid 4 2024.5 0.062331 \n", - "\n", - " final_mem_perf \n", - "0 0.716435 \n", - "1 0.717885 \n", - "2 0.718340 \n", - "3 0.372141 \n", - "4 0.417175 " - ] - }, - "execution_count": 66, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "sorted_mean_df" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "id": "b6c3c84c", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([], dtype=int64)" - ] - }, - "execution_count": 36, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "seeds" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "69829ba6", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.6" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/plotting/mi_performance.py b/scripts/plotting/mi_performance.py index 4803b735..d32ae123 100644 --- a/scripts/plotting/mi_performance.py +++ b/scripts/plotting/mi_performance.py @@ -1,194 +1,102 @@ +# %% codecell import numpy as np import jax.numpy as jnp import matplotlib.pyplot as plt import matplotlib.cm as cm import pandas as pd +from argparse import Namespace from jax.nn import softmax -from jax.config import config +from jax import config from pathlib import Path from collections import namedtuple +from tqdm import tqdm config.update('jax_platform_name', 'cpu') np.set_printoptions(precision=4) plt.rcParams['axes.facecolor'] = 'white' plt.rcParams.update({'font.size': 18}) -from grl.utils import load_info +from scripts.plotting.parse_experiments import parse_baselines, parse_dirs from definitions import ROOT_DIR -# %% -# results_dir = Path(ROOT_DIR, 'results', 'pomdps_mi_pi') -results_dir = Path(ROOT_DIR, 'results', 'all_pomdps_mi_pi_obs_space') +# %% codecell +# results_dir = Path(ROOT_DIR, 'results', 'final_analytical_kitchen_sinks') +experiment_dirs = [ + Path(ROOT_DIR, 'results', 'mem_tde_kitchen_sinks_pg'), + Path(ROOT_DIR, 'results', 'final_discrep_kitchen_sinks_pg'), +] + +# results_dir = Path(ROOT_DIR, 'results', 'prisoners_dilemma') # results_dir = Path(ROOT_DIR, 'results', 'pomdps_mi_dm') vi_results_dir = Path(ROOT_DIR, 'results', 'vi') pomdp_files_dir = Path(ROOT_DIR, 'grl', 'environment', 'pomdp_files') -args_to_keep = ['spec', 'n_mem_states', 'seed'] -split_by = [arg for arg in args_to_keep if arg != 'seed'] +args_to_keep = ['spec', 'n_mem_states', 'seed', 'alpha', 'residual'] +split_by = [arg for arg in args_to_keep if arg != 'seed'] + ['experiment'] # this option allows us to compare to either the optimal belief state soln # or optimal state soln. ('belief' | 'state') compare_to = 'belief' +# compare_to = 'state' + +# policy_optim_alg = 'policy_grad' +policy_optim_alg = 'policy_grad' -# spec_plot_order = ['example_7', 'slippery_tmaze_5_two_thirds_up', -# 'tiger', 'paint.95', 'cheese.95', -# 'network', 'shuttle.95', '4x3.95'] spec_plot_order = [ - 'example_7', 'tmaze_5_two_thirds_up', 'tiger-alt-start', 'paint.95', 'cheese.95', 'network', - 'shuttle.95', '4x3.95', 'hallway' + 'network', 'paint.95', '4x3.95', 'tiger-alt-start', 'shuttle.95', 'cheese.95', 'tmaze_5_two_thirds_up' ] -spec_to_belief_state = {'tmaze_5_two_thirds_up': 'tmaze5'} - -# %% - -compare_to_list = [] - -# if compare_to == 'belief': -for spec in spec_plot_order: - for fname in pomdp_files_dir.iterdir(): - if 'pomdp-solver-results' in fname.stem: - if (fname.stem == f"{spec_to_belief_state.get(spec, spec)}-pomdp-solver-results"): - belief_info = load_info(fname) - coeffs = belief_info['coeffs'] - max_start_vals = coeffs[belief_info['max_start_idx']] - spec_compare_indv = { - 'spec': spec, - 'compare_perf': np.dot(max_start_vals, belief_info['p0']) - } - compare_to_list.append(spec_compare_indv) - break - # print(f"loaded results for {hparams.spec} from {fname}") - else: - for vi_path in vi_results_dir.iterdir(): - for spec in spec_plot_order: - if spec_to_belief_state.get(spec, spec) in vi_path.name: - vi_info = load_info(vi_path) - spec_compare_indv = { - 'spec': spec, - 'compare_perf': np.dot(max_start_vals, belief_info['p0']) - } - compare_to_list.append(spec_compare_indv) - -# elif compare_to == 'state': -# else: -# raise NotImplementedError - -compare_to_df = pd.DataFrame(compare_to_list) - -# %% +# %% codecell -all_results = [] +compare_to_dict = parse_baselines(spec_plot_order, + vi_results_dir, + pomdp_files_dir, + compare_to=compare_to) -for results_path in results_dir.iterdir(): - if results_path.is_dir() or results_path.suffix != '.npy': - continue - info = load_info(results_path) - args = info['args'] +# %% codecell +all_res_df = parse_dirs(experiment_dirs, + compare_to_dict, + args_to_keep) - # agent = info['agent'] - init_policy_info = info['logs']['initial_policy_stats'] - init_improvement_info = info['logs']['greedy_initial_improvement_stats'] - final_mem_info = info['logs']['greedy_final_mem_stats'] - def get_perf(info: dict): - return (info['state_vals_v'] * info['p0']).sum() - single_res = {k: args[k] for k in args_to_keep} +# %% codecell - single_res.update({ - 'init_policy_perf': get_perf(init_policy_info), - 'init_improvement_perf': get_perf(init_improvement_info), - 'final_mem_perf': get_perf(final_mem_info), - # 'init_policy': info['logs']['initial_policy'], - # 'init_improvement_policy': info['logs']['initial_improvement_policy'], - # 'final_mem': np.array(agent.memory), - # 'final_policy': np.array(agent.policy) - }) - all_results.append(single_res) +# FILTER OUT for what we want to plot +# alpha = 1. +# +# all_res_df +residual = False +filtered_df = all_res_df[(all_res_df['residual'] == residual)].reset_index() -all_res_df = pd.DataFrame(all_results) +# %% codecell +all_res_groups = filtered_df.groupby(split_by, as_index=False) +all_res_means = all_res_groups.mean() +del all_res_means['seed'] +# all_res_means.to_csv(Path(ROOT_DIR, 'results', 'all_pomdps_means.csv')) -# %% +# %% codecell cols_to_normalize = ['init_improvement_perf', 'final_mem_perf'] -merged_df = all_res_df.merge(compare_to_df, on='spec') +# merged_df = filtered_df.merge(compare_to_df, on='spec') +merged_df = filtered_df # for col_name in cols_to_normalize: normalized_df = merged_df.copy() -normalized_df['init_improvement_perf'] = ( - normalized_df['init_improvement_perf'] - - merged_df['init_policy_perf']) / (merged_df['compare_perf'] - merged_df['init_policy_perf']) -normalized_df['final_mem_perf'] = (normalized_df['final_mem_perf'] - merged_df['init_policy_perf'] - ) / (merged_df['compare_perf'] - merged_df['init_policy_perf']) +normalized_df['init_improvement_perf'] = (normalized_df['init_improvement_perf'] - merged_df['init_policy_perf']) / (merged_df['compare_to_perf'] - merged_df['init_policy_perf']) +normalized_df['final_mem_perf'] = (normalized_df['final_mem_perf'] - merged_df['init_policy_perf']) / (merged_df['compare_to_perf'] - merged_df['init_policy_perf']) del normalized_df['init_policy_perf'] -del normalized_df['compare_perf'] - -# all_normalized_perf_results = {} -# for hparams, res in all_results.items(): -# max_key = 'compare_perf' -# # if max_key not in res: -# # max_key = 'final_mem_perf' -# max_v = res[max_key] -# min_v = res['init_policy_perf'] -# for k, v in res.items(): -# if '_perf' in k: -# all_results[hparams][k] = (v - min_v) / (max_v - min_v) -# %% -normalized_df.groupby(split_by).mean() +del normalized_df['compare_to_perf'] -# %% +# %% codecell +normalized_df.loc[(normalized_df['spec'] == 'hallway') & (normalized_df['n_mem_states'] == 8), 'final_mem_perf'] = 0 -# all_table_results = {} -# all_plot_results = {'x': [], 'xlabels': []} -# -# for i, spec in enumerate(spec_plot_order): -# hparams = sorted([k for k in all_results.keys() if k.spec == spec], -# key=lambda hp: hp.n_mem_states) -# -# first_res = all_results[hparams[0]] -# all_plot_results['x'].append(i) -# all_plot_results['xlabels'].append(spec) -# -# # we first add initial and first improvement stats -# for k, v in first_res.items(): -# if 'perf' in k and k != 'final_mem_perf': -# mean = v.mean(axis=0) -# std_err = v.std(axis=0) / np.sqrt(v.shape[0]) -# -# stripped_str = k.replace('_perf', '') -# if stripped_str not in all_plot_results: -# all_plot_results[stripped_str] = {'mean': [], 'std_err': []} -# all_plot_results[stripped_str]['mean'].append(mean) -# all_plot_results[stripped_str]['std_err'].append(std_err) -# -# # now we add final memory perf, for each n_mem_states -# for hparam in hparams: -# res = all_results[hparam] -# for k, v in res.items(): -# if k == 'final_mem_perf': -# mean = v.mean(axis=0) -# std_err = v.std(axis=0) / np.sqrt(v.shape[0]) -# stripped_str = k.replace('_perf', '') -# mem_label = f"mem, |ms| = {hparam.n_mem_states}" -# if mem_label not in all_plot_results: -# all_plot_results[mem_label] = {'mean': [], 'std_err': []} -# all_plot_results[mem_label]['mean'].append(mean) -# all_plot_results[mem_label]['std_err'].append(std_err) -# -# ordered_plot = [] -# # ordered_plot.append(('init_policy', all_plot_results['init_policy'])) -# ordered_plot.append(('init_improvement', all_plot_results['init_improvement'])) -# for k in sorted(all_plot_results.keys()): -# if 'mem' in k: -# ordered_plot.append((k, all_plot_results[k])) -# ordered_plot.append(('state_optimal', all_plot_results['vi'])) - -# %% - -# %% +# %% codecell +# normalized_df[normalized_df['spec'] == 'prisoners_dilemma_all_c'] +seeds = normalized_df[normalized_df['spec'] == normalized_df['spec'][0]]['seed'].unique() +# %% codecell def maybe_spec_map(id: str): spec_map = { '4x3.95': '4x3', @@ -199,41 +107,102 @@ def maybe_spec_map(id: str): 'tmaze_5_two_thirds_up': 'tmaze', 'tiger-alt-start': 'tiger' } + +# spec_map |= prisoners_spec_map + if id not in spec_map: return id return spec_map[id] groups = normalized_df.groupby(split_by, as_index=False) -means = groups.mean() -std_errs = groups.std() -num_n_mem = list(sorted(normalized_df['n_mem_states'].unique())) +all_means = groups.mean() +all_means['init_improvement_perf'].clip(lower=0, upper=1, inplace=True) +all_means['final_mem_perf'].clip(lower=0, upper=1, inplace=True) + +all_std_errs = groups.std() +all_std_errs['init_improvement_perf'] /= np.sqrt(len(seeds)) +all_std_errs['final_mem_perf'] /= np.sqrt(len(seeds)) + +# %% + +# SORTING +sorted_mean_df = pd.DataFrame() +sorted_std_err_df = pd.DataFrame() + +for spec in spec_plot_order: + mean_spec_df = all_means[all_means['spec'] == spec] + std_err_spec_df = all_std_errs[all_std_errs['spec'] == spec] + sorted_mean_df = pd.concat([sorted_mean_df, mean_spec_df]) + sorted_std_err_df = pd.concat([sorted_std_err_df, std_err_spec_df]) + +# %% +experiments = normalized_df['experiment'].unique() group_width = 1 -bar_width = group_width / (len(num_n_mem) + 2) +num_n_mem = list(sorted(normalized_df['n_mem_states'].unique())) +specs = means['spec'].unique() + +spec_order_mapping = np.arange(len(specs), dtype=int) + +exp_group_width = group_width / (len(experiments)) +bar_width = exp_group_width / (len(num_n_mem) + 3) + fig, ax = plt.subplots(figsize=(12, 6)) -x = np.arange(len(means)) -xlabels = [maybe_spec_map(l) for l in list(means['spec'])] +xlabels = [maybe_spec_map(l) for l in specs] +x = np.arange(len(specs)) + +init_improvement_perf_mean = np.array(all_means[ + (all_means['n_mem_states'] == num_n_mem[0]) & + (all_means['experiment'] == experiments[0]) + ]['init_improvement_perf']) +init_improvement_perf_std = np.array(all_std_errs[ + (all_std_errs['n_mem_states'] == num_n_mem[0]) & + (all_std_errs['experiment'] == experiments[0]) + ]['init_improvement_perf']) -ax.bar(x + (0 + 1) * bar_width, - means[means['n_mem_states'] == num_n_mem[0]]['init_improvement_perf'], +ax.bar(x + 0 * exp_group_width + (0 + 1) * bar_width, + init_improvement_perf_mean, bar_width, - yerr=std_errs[std_errs['n_mem_states'] == num_n_mem[0]]['init_improvement_perf'], - label='Memoryless') - -for i, n_mem_states in enumerate(num_n_mem): - ax.bar(x + (i + 2) * bar_width, - means[means['n_mem_states'] == n_mem_states]['final_mem_perf'], - bar_width, - yerr=std_errs[std_errs['n_mem_states'] == n_mem_states]['final_mem_perf'], - label=f"{int(np.log(n_mem_states))} Memory Bits") -ax.set_ylim([0, 1]) + yerr=init_improvement_perf_std, + label='Memoryless', + color='#5B97E0') + +mem_colors = ['#E0B625', '#DD8453', '#C44E52'] +exp_hatches = ['/', 'o', '+', '.'] + +for i, exp_name in enumerate(experiments): + means = sorted_mean_df[sorted_mean_df['experiment'] == exp_name] + std_errs = sorted_std_err_df[sorted_std_err_df['experiment'] == exp_name] + + # means = sorted_mean_df + # std_errs = sorted_std_err_df + + for j, n_mem_states in enumerate(num_n_mem): + curr_mem_mean = np.array(means[means['n_mem_states'] == n_mem_states]['final_mem_perf']) + curr_mem_std = np.array(std_errs[std_errs['n_mem_states'] == n_mem_states]['final_mem_perf']) + to_add = i * exp_group_width + (j + 3) * bar_width + if i != 0: + to_add -= 2 * bar_width + ax.bar(x + to_add, + curr_mem_mean, + bar_width, + yerr=curr_mem_std, + label=f"{int(np.log2(n_mem_states))} Memory Bits", + hatch=exp_hatches[i], + color=mem_colors[j]) + +ax.set_ylim([0, 1.05]) ax.set_ylabel(f'Relative Performance\n (w.r.t. optimal {compare_to} & initial policy)') ax.set_xticks(x + group_width / 2) ax.set_xticklabels(xlabels) -ax.legend(bbox_to_anchor=(0.7, 0.6), framealpha=0.95) -ax.set_title("Performance of Memory Iteration in POMDPs") +# ax.legend(bbox_to_anchor=(0.317, 0.62), framexalpha=0.95) +# ax.set_title(f"Memory Iteration ({policy_optim_alg})") +alpha_str = 'uniform' if alpha == 1. else 'occupancy' +residual_str = 'semi_grad' if not residual else 'residual' +ax.set_title(f"Memory: (MSTDE (dashes, {residual_str}) vs LD (dots, {alpha_str}))") downloads = Path().home() / 'Downloads' -fig_path = downloads / f"{results_dir.stem}.pdf" -fig.savefig(fig_path) +fig_path = downloads / f"{results_dir.stem}_{residual_str}_{alpha_str}.pdf" +fig.savefig(fig_path, bbox_inches='tight') +# %% codecell diff --git a/scripts/plotting/parse_experiments.py b/scripts/plotting/parse_experiments.py new file mode 100644 index 00000000..20cb0390 --- /dev/null +++ b/scripts/plotting/parse_experiments.py @@ -0,0 +1,113 @@ +from pathlib import Path + +import numpy as np +import pandas as pd +from tqdm import tqdm + +from grl.memory.analytical import memory_cross_product +from grl.environment import load_pomdp +from grl.utils import load_info +from grl.utils.lambda_discrep import lambda_discrep_measures +from grl.utils.math import greedify + +from definitions import ROOT_DIR + +def parse_dirs(exp_dirs: list[Path], + baseline_dict: dict, + args_to_keep: list[str]): + all_results = [] + + def parse_exp_dir(exp_dir: Path): + print(f"Parsing {exp_dir}") + for results_path in tqdm(list(exp_dir.iterdir())): + if results_path.is_dir() or results_path.suffix != '.npy': + continue + + info = load_info(results_path) + args = info['args'] + + if args['spec'] not in baseline_dict: + continue + + def get_perf(info: dict): + return (info['state_vals_v'] * info['p0']).sum() + + # Greedification + agent_path = results_path.parent / 'agent' / f'{results_path.stem}.pkl.npy' + agent = load_info(agent_path) + pomdp, _ = load_pomdp(args['spec']) + final_mem_pomdp = memory_cross_product(agent.mem_params, pomdp) + + greedy_policy = greedify(agent.policy) + + greedy_measures = lambda_discrep_measures(final_mem_pomdp, greedy_policy) + greedy_pi_mem_perf = float(get_perf(greedy_measures)) + + init_policy_info = info['logs']['initial_policy_stats'] + init_improvement_info = info['logs']['greedy_initial_improvement_stats'] + final_mem_info = info['logs']['final_mem_stats'] + + single_res = {k: args[k] for k in args_to_keep} + + final_mem_perf = get_perf(final_mem_info) + + compare_to_perf = baseline_dict[args['spec']] + init_policy_perf = get_perf(init_policy_info) + init_improvement_perf = get_perf(init_improvement_info) + + if init_policy_perf > init_improvement_perf: + print("Initial policy performance is better than improvement performance\n" + f"For {results_path}.\n" + "Setting the bottom line to the initial policy performance.") + init_policy_perf = init_improvement_perf + + single_res.update({ + 'experiment': exp_dir.name, + 'init_policy_perf': init_policy_perf, + 'init_improvement_perf': init_improvement_perf, + 'final_mem_perf': final_mem_perf, + 'greedy_pi_mem_perf': greedy_pi_mem_perf, + 'compare_to_perf': compare_to_perf, + }) + all_results.append(single_res) + + for exp_dir in exp_dirs: + parse_exp_dir(exp_dir) + + all_res_df = pd.DataFrame(all_results) + + return all_res_df + +def parse_baselines( + plot_order: list[str], + vi_dir: Path = Path(ROOT_DIR, 'results', 'vi'), + pomdp_files_dir: Path = Path(ROOT_DIR, 'grl', 'environment', 'pomdp_files'), + compare_to: str = 'belief'): + compare_to_dict = {} + spec_to_belief_state = {'tmaze_5_two_thirds_up': 'tmaze5'} + + def load_state_val(spec: str): + for vi_path in vi_dir.iterdir(): + if spec_to_belief_state.get(spec, spec) in vi_path.name: + vi_info = load_info(vi_path) + max_start_vals = vi_info['optimal_vs'] + return np.dot(max_start_vals, vi_info['p0']) + + for spec in plot_order: + if compare_to == 'belief': + + for fname in pomdp_files_dir.iterdir(): + if 'pomdp-solver-results' in fname.stem: + if (fname.stem == + f"{spec_to_belief_state.get(spec, spec)}-pomdp-solver-results" + ): + belief_info = load_info(fname) + compare_to_dict[spec] = belief_info['start_val'] + break + else: + compare_to_dict[spec] = load_state_val(spec) + + elif compare_to == 'state': + compare_to_dict[spec] = load_state_val(spec) + + return compare_to_dict \ No newline at end of file diff --git a/scripts/plotting/rnn_performance.ipynb b/scripts/plotting/rnn_performance.ipynb deleted file mode 100644 index 294e3275..00000000 --- a/scripts/plotting/rnn_performance.ipynb +++ /dev/null @@ -1,821 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "9bd5d2b9", - "metadata": {}, - "outputs": [], - "source": [ - "from argparse import Namespace\n", - "import jax.numpy as jnp\n", - "import matplotlib.pyplot as plt\n", - "from matplotlib.pyplot import cm\n", - "import numpy as np\n", - "import pandas as pd\n", - "from tqdm.notebook import tqdm\n", - "import seaborn as sns\n", - "\n", - "from jax.nn import softmax\n", - "from jax.config import config\n", - "from pathlib import Path\n", - "from collections import namedtuple\n", - "\n", - "config.update('jax_platform_name', 'cpu')\n", - "np.set_printoptions(precision=4)\n", - "plt.rcParams['axes.facecolor'] = 'white'\n", - "plt.rcParams.update({'font.size': 18})\n", - "\n", - "from grl.utils import load_info\n", - "from grl.utils.mdp import all_t_discounted_returns\n", - "from grl.utils.data import uncompress_episode_rewards\n", - "from grl.utils.loss import mse\n", - "from grl.environment import get_env\n", - "\n", - "from definitions import ROOT_DIR" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "8780676c", - "metadata": {}, - "outputs": [], - "source": [ - "results_dir_td = Path(ROOT_DIR, 'results', 'rnn_rocksample_sweep_td')\n", - "results_dir_mc = Path(ROOT_DIR, 'results', 'rnn_rocksample_sweep_mc')\n", - "results_dir_split = Path(ROOT_DIR, 'results', 'rnn_split')" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "48aa4613", - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "641031ca75864091a8725303fe54ff9c", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/120 [00:00 0.):\n", - " # continue\n", - "\n", - " no_gamma_terminal = not args.no_gamma_terminal\n", - "\n", - " offline_evals = info['episodes_info']['offline_eval']\n", - " eval_freq = args.offline_eval_freq\n", - " total_steps = args.total_steps\n", - " env = get_env(args)\n", - " reward_scale = 1.\n", - " normalize_rewards = args.normalize_rewards\n", - " if args.normalize_rewards:\n", - " assert hasattr(env, 'R_max') and hasattr(env, 'R_min')\n", - " reward_scale = 1 / (env.R_max - env.R_min)\n", - "\n", - "\n", - " all_t_undisc_returns = []\n", - "\n", - " # For each offline eval\n", - " for i, oe in enumerate(offline_evals):\n", - " d = {**info['args']}\n", - "\n", - " avg_undisc_returns = 0\n", - " avg_value_err = 0\n", - "\n", - " # iterate through number of seeds\n", - " for ep_rewards, ep_qs in zip(oe['episode_rewards'], oe['episode_qs']):\n", - " # relevant qs are everything but the last index\n", - " # b/c last index is terminal q\n", - " relevant_ep_qs = ep_qs[:-1]\n", - "\n", - " episode_rewards = np.array(uncompress_episode_rewards(ep_rewards['episode_length'], ep_rewards['most_common_reward'], ep_rewards['compressed_rewards']))\n", - "\n", - " discounts = np.ones(episode_rewards.shape[0])\n", - " if no_gamma_terminal:\n", - " discounts *= env.gamma\n", - "\n", - " discounts[-1] = 0.\n", - "\n", - " discounted_returns = all_t_discounted_returns(discounts, reward_scale * episode_rewards)\n", - " avg_value_err += mse(relevant_ep_qs, discounted_returns).item()\n", - "\n", - " # TODO: calculate value estimation error through q_t - g_t\n", - " undisc_returns = episode_rewards.sum()\n", - " avg_undisc_returns += undisc_returns\n", - "\n", - " avg_undisc_returns /= len(oe['episode_rewards'])\n", - " avg_value_err /= len(oe['episode_rewards'])\n", - "\n", - " d['timestep'] = int(i * eval_freq)\n", - " d['undisc_returns'] = avg_undisc_returns\n", - " d['value_err'] = avg_value_err\n", - " \n", - " offline_eval.append(d)\n", - "\n", - "all_res_df = pd.DataFrame(offline_eval)\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "7ded083b", - "metadata": {}, - "outputs": [], - "source": [ - "arch = \"gru\"\n", - "measure_col = 'undisc_returns'\n", - "all_args = list(info['args'].keys())\n", - "\n", - "offline_eval_df = all_res_df[\n", - " (all_res_df['arch'] == arch)\n", - " & (all_res_df['multihead_lambda_coeff'] >= 0.)\n", - "# & (all_res_df['multihead_lambda_coeff'] != 1.)\n", - "# & (all_res_df['multihead_loss_mode'] != 'td')\n", - "# & (all_res_df['multihead_action_mode'] == 'td')\n", - "]\n", - "\n", - "unique_seeds = offline_eval_df.seed.unique()\n", - "\n", - "split_by_args = ['spec', 'multihead_action_mode', 'multihead_loss_mode', 'multihead_lambda_coeff', 'lr']\n", - "\n", - "# For split exp\n", - "# split_by_args = ['spec', 'multihead_loss_mode', 'lr']\n", - "\n", - "rnn_grouped = offline_eval_df.groupby(split_by_args, as_index=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "6f698bcf", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
specno_gamma_terminalmax_episode_stepsfeature_encodingalgoarchepsilonlroptimizerhidden_size...offline_eval_epsiloncheckpoint_freqsave_all_checkpointstotal_stepsplatformseedstudy_nametimestepundisc_returnsvalue_err
0rocksampleFalse1000one_hotmultihead_rnngru0.10.000001adam64...0.1-1False1500000cpu2024rnn_rocksample_sweep_td0-31.00.469165
1rocksampleFalse1000one_hotmultihead_rnngru0.10.000001adam64...0.1-1False1500000cpu2024rnn_rocksample_sweep_td10000.00.116940
2rocksampleFalse1000one_hotmultihead_rnngru0.10.000001adam64...0.1-1False1500000cpu2024rnn_rocksample_sweep_td20004.00.109426
3rocksampleFalse1000one_hotmultihead_rnngru0.10.000001adam64...0.1-1False1500000cpu2024rnn_rocksample_sweep_td3000-2.00.131880
4rocksampleFalse1000one_hotmultihead_rnngru0.10.000001adam64...0.1-1False1500000cpu2024rnn_rocksample_sweep_td40007.00.130211
..................................................................
179995rocksampleFalse1000one_hotmultihead_rnngru0.10.000001adam64...0.1-1False1500000cpu2020rnn_rocksample_sweep_mc149500014.00.045318
179996rocksampleFalse1000one_hotmultihead_rnngru0.10.000001adam64...0.1-1False1500000cpu2020rnn_rocksample_sweep_mc149600018.00.034946
179997rocksampleFalse1000one_hotmultihead_rnngru0.10.000001adam64...0.1-1False1500000cpu2020rnn_rocksample_sweep_mc149700015.00.027271
179998rocksampleFalse1000one_hotmultihead_rnngru0.10.000001adam64...0.1-1False1500000cpu2020rnn_rocksample_sweep_mc149800012.00.016947
179999rocksampleFalse1000one_hotmultihead_rnngru0.10.000001adam64...0.1-1False1500000cpu2020rnn_rocksample_sweep_mc149900013.00.037840
\n", - "

180000 rows × 32 columns

\n", - "
" - ], - "text/plain": [ - " spec no_gamma_terminal max_episode_steps feature_encoding \\\n", - "0 rocksample False 1000 one_hot \n", - "1 rocksample False 1000 one_hot \n", - "2 rocksample False 1000 one_hot \n", - "3 rocksample False 1000 one_hot \n", - "4 rocksample False 1000 one_hot \n", - "... ... ... ... ... \n", - "179995 rocksample False 1000 one_hot \n", - "179996 rocksample False 1000 one_hot \n", - "179997 rocksample False 1000 one_hot \n", - "179998 rocksample False 1000 one_hot \n", - "179999 rocksample False 1000 one_hot \n", - "\n", - " algo arch epsilon lr optimizer hidden_size ... \\\n", - "0 multihead_rnn gru 0.1 0.000001 adam 64 ... \n", - "1 multihead_rnn gru 0.1 0.000001 adam 64 ... \n", - "2 multihead_rnn gru 0.1 0.000001 adam 64 ... \n", - "3 multihead_rnn gru 0.1 0.000001 adam 64 ... \n", - "4 multihead_rnn gru 0.1 0.000001 adam 64 ... \n", - "... ... ... ... ... ... ... ... \n", - "179995 multihead_rnn gru 0.1 0.000001 adam 64 ... \n", - "179996 multihead_rnn gru 0.1 0.000001 adam 64 ... \n", - "179997 multihead_rnn gru 0.1 0.000001 adam 64 ... \n", - "179998 multihead_rnn gru 0.1 0.000001 adam 64 ... \n", - "179999 multihead_rnn gru 0.1 0.000001 adam 64 ... \n", - "\n", - " offline_eval_epsilon checkpoint_freq save_all_checkpoints \\\n", - "0 0.1 -1 False \n", - "1 0.1 -1 False \n", - "2 0.1 -1 False \n", - "3 0.1 -1 False \n", - "4 0.1 -1 False \n", - "... ... ... ... \n", - "179995 0.1 -1 False \n", - "179996 0.1 -1 False \n", - "179997 0.1 -1 False \n", - "179998 0.1 -1 False \n", - "179999 0.1 -1 False \n", - "\n", - " total_steps platform seed study_name timestep \\\n", - "0 1500000 cpu 2024 rnn_rocksample_sweep_td 0 \n", - "1 1500000 cpu 2024 rnn_rocksample_sweep_td 1000 \n", - "2 1500000 cpu 2024 rnn_rocksample_sweep_td 2000 \n", - "3 1500000 cpu 2024 rnn_rocksample_sweep_td 3000 \n", - "4 1500000 cpu 2024 rnn_rocksample_sweep_td 4000 \n", - "... ... ... ... ... ... \n", - "179995 1500000 cpu 2020 rnn_rocksample_sweep_mc 1495000 \n", - "179996 1500000 cpu 2020 rnn_rocksample_sweep_mc 1496000 \n", - "179997 1500000 cpu 2020 rnn_rocksample_sweep_mc 1497000 \n", - "179998 1500000 cpu 2020 rnn_rocksample_sweep_mc 1498000 \n", - "179999 1500000 cpu 2020 rnn_rocksample_sweep_mc 1499000 \n", - "\n", - " undisc_returns value_err \n", - "0 -31.0 0.469165 \n", - "1 0.0 0.116940 \n", - "2 4.0 0.109426 \n", - "3 -2.0 0.131880 \n", - "4 7.0 0.130211 \n", - "... ... ... \n", - "179995 14.0 0.045318 \n", - "179996 18.0 0.034946 \n", - "179997 15.0 0.027271 \n", - "179998 12.0 0.016947 \n", - "179999 13.0 0.037840 \n", - "\n", - "[180000 rows x 32 columns]" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "all_res_df" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "0daad22f", - "metadata": {}, - "outputs": [], - "source": [ - "# take mean and std_err over mean over seeds\n", - "rnn_mean = rnn_grouped.mean(numeric_only=True)\n", - "rnn_sem = rnn_grouped.sem(numeric_only=True)\n", - "\n", - "# Now we take the max over everything but learning rate\n", - "max_over_args = ['lr']\n", - "grouped_args = [arg for arg in split_by_args if arg not in max_over_args]\n", - "rnn_less_lr_grouped = rnn_mean.groupby(grouped_args, as_index=False)\n", - "\n", - "rnn_arg_maxes = rnn_mean.sort_values(measure_col).drop_duplicates(grouped_args, keep='last')" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "0896e4d0", - "metadata": {}, - "outputs": [], - "source": [ - "# drop columns we need in original df\n", - "pruned_rnn_arg_maxes = rnn_arg_maxes.drop(columns=['timestep', 'seed'])\n", - "\n", - "# A df of results over all the maxes.\n", - "max_offline_eval_df = pruned_rnn_arg_maxes.merge(offline_eval_df, on=split_by_args, how='inner')\n" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "id": "30194301", - "metadata": {}, - "outputs": [], - "source": [ - "w = 100\n", - "mode = 'same'\n", - "max_offline_eval_df[f'avged_returns'] = max_offline_eval_df.groupby(split_by_args)['undisc_returns_y'].transform(lambda x: np.convolve(x, np.ones(w), mode=mode) / w)" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "id": "21d18833", - "metadata": {}, - "outputs": [], - "source": [ - "df = max_offline_eval_df\n", - "# pivot_value_key = 'undisc_returns_y'\n", - "pivot_value_key = 'avged_returns'\n", - "\n", - "# pivot_value_key = 'value_err_y'\n", - "\n", - "\n", - "# df = offline_eval_df\n", - "# pivot_value_key = 'undisc_returns'\n", - "\n", - "all_labels = \\\n", - " sorted(['_'.join([str(el) for el in row]) for row in df[[arg for arg in grouped_args if (arg != 'spec') and (arg != 'timestep')]]\\\n", - " .drop_duplicates()\\\n", - " .to_numpy()])\n", - "\n", - "label_cmap = {}\n", - "\n", - "color = iter(cm.tab10(np.linspace(0, 1, 10)))\n", - "for i in range(len(all_labels)):\n", - " c = next(color)\n", - " label_cmap[all_labels[i]] = c" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "id": "622b6cec", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1499000" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 41, - "id": "9e3ccaec", - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "2fb529efeb6e408cb5b9e566973719ed", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/1 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "spec_strs = df['spec'].unique()\n", - "\n", - "pivot_cols = [arg for arg in grouped_args if (arg != 'spec') and (arg != 'timestep')] + ['seed']\n", - "\n", - "n_specs = len(spec_strs)\n", - "\n", - "n_rows, n_cols = 2, np.ceil(n_specs / 2).astype(int)\n", - "if n_specs == 1:\n", - " n_rows = 1\n", - "\n", - "fig, axes = plt.subplots(n_rows, n_cols, figsize=(24, 10))\n", - "\n", - "for i, spec in tqdm(list(enumerate(sorted(spec_strs)))):\n", - " row = i // n_cols\n", - " col = i % n_cols\n", - " \n", - " ax = axes\n", - " if n_specs > 1:\n", - " ax = axes[row, col]\n", - "\n", - " spec_df = df[df['spec'] == spec]\n", - " sorted_spec_df = spec_df.sort_values(by=grouped_args)\n", - " \n", - " # Make a pivot table\n", - " pivot_df = sorted_spec_df.pivot(index='timestep', columns=pivot_cols, values=pivot_value_key)\n", - " \n", - " # Flatten all columns except seed\n", - " col_name = '_'.join([col for col in pivot_cols if col != 'seed'])\n", - " multi_index = pd.MultiIndex.from_tuples([('_'.join([str(c) for c in col[:-1]]), col[-1]) for col in pivot_df.columns], names=[col_name, \"seed\"])\n", - "\n", - " pivot_df.columns = multi_index\n", - "\n", - " sns.lineplot(ax=ax, data=pivot_df, palette=label_cmap)\n", - " \n", - " ax.get_legend().remove()\n", - " ax.set_title(spec)\n", - " if col > 0:\n", - " ax.set_ylabel(None)\n", - " if row < n_rows - 1:\n", - " ax.set_xlabel(None)\n", - " \n", - " handles, labels = ax.get_legend_handles_labels()\n", - " \n", - " if spec == 'rocksample':\n", - " ax.set_ylim(0, 35)\n", - " step_per_eval = 1000\n", - " ax.set_xlim(w * step_per_eval // 2, df['timestep'].max() - w * step_per_eval // 2)\n", - "# if i >= 1:\n", - "# break\n", - " \n", - "fig.legend(handles, labels, loc='lower right')\n", - "fig.suptitle(f\"{measure_col} {arch}\")\n", - "fig.tight_layout()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "01ddc2b7", - "metadata": {}, - "outputs": [], - "source": [ - "# fig_path = Path(ROOT_DIR, 'results', f'{measure_col}_{arch}_rnn_td.pdf')\n", - "# fig.savefig(fig_path, bbox_inches='tight')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f0d7f37c", - "metadata": {}, - "outputs": [], - "source": [ - "# for results_path in tqdm(all_results_paths):\n", - "# if results_path.is_dir() or results_path.suffix != '.npy':\n", - "# continue\n", - " \n", - "# info = load_info(results_path)\n", - "# args = info['args']\n", - "# if args['multihead_lambda_coeff'] == -1:\n", - "# results_path.unlink()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "28787036", - "metadata": {}, - "outputs": [], - "source": [ - "()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.6" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/plotting/rnn_performance.py b/scripts/plotting/rnn_performance.py new file mode 100644 index 00000000..53d77b53 --- /dev/null +++ b/scripts/plotting/rnn_performance.py @@ -0,0 +1,241 @@ +# %% codecell +from argparse import Namespace +import jax.numpy as jnp +import matplotlib.pyplot as plt +from matplotlib.pyplot import cm +import numpy as np +import pandas as pd +from tqdm.notebook import tqdm +import seaborn as sns + +from jax.nn import softmax +from jax.config import config +from pathlib import Path +from collections import namedtuple + +config.update('jax_platform_name', 'cpu') +np.set_printoptions(precision=4) +plt.rcParams['axes.facecolor'] = 'white' +plt.rcParams.update({'font.size': 18}) + +from grl.utils import load_info +from grl.utils.mdp import all_t_discounted_returns +from grl.utils.data import uncompress_episode_rewards +from grl.utils.loss import mse +from grl.environment import get_env + +from definitions import ROOT_DIR +# %% codecell +results_dir_td = Path(ROOT_DIR, 'results', 'rnn_rocksample_sweep_td') +results_dir_mc = Path(ROOT_DIR, 'results', 'rnn_rocksample_sweep_mc') +results_dir_split = Path(ROOT_DIR, 'results', 'rnn_split') +# %% codecell + +offline_eval = [] +all_results_paths = list(results_dir_td.iterdir()) + list(results_dir_mc.iterdir()) +# all_results_paths = list(results_dir_td.iterdir()) + +# all_results_paths = list(results_dir_split.iterdir()) + +for results_path in tqdm(all_results_paths): + if results_path.is_dir() or results_path.suffix != '.npy': + continue + + info = load_info(results_path) + args = Namespace(**info['args']) + + # if (args.multihead_loss_mode != 'both') and (args.multihead_lambda_coeff > 0.): + # continue + + no_gamma_terminal = not args.no_gamma_terminal + + offline_evals = info['episodes_info']['offline_eval'] + eval_freq = args.offline_eval_freq + total_steps = args.total_steps + env = get_env(args) + reward_scale = 1. + normalize_rewards = args.normalize_rewards + if args.normalize_rewards: + assert hasattr(env, 'R_max') and hasattr(env, 'R_min') + reward_scale = 1 / (env.R_max - env.R_min) + + + all_t_undisc_returns = [] + + # For each offline eval + for i, oe in enumerate(offline_evals): + d = {**info['args']} + + avg_undisc_returns = 0 + avg_value_err = 0 + + # iterate through number of seeds + for ep_rewards, ep_qs in zip(oe['episode_rewards'], oe['episode_qs']): + # relevant qs are everything but the last index + # b/c last index is terminal q + relevant_ep_qs = ep_qs[:-1] + + episode_rewards = np.array(uncompress_episode_rewards(ep_rewards['episode_length'], ep_rewards['most_common_reward'], ep_rewards['compressed_rewards'])) + + discounts = np.ones(episode_rewards.shape[0]) + if no_gamma_terminal: + discounts *= env.gamma + + discounts[-1] = 0. + + discounted_returns = all_t_discounted_returns(discounts, reward_scale * episode_rewards) + avg_value_err += mse(relevant_ep_qs, discounted_returns).item() + + # TODO: calculate value estimation error through q_t - g_t + undisc_returns = episode_rewards.sum() + avg_undisc_returns += undisc_returns + + avg_undisc_returns /= len(oe['episode_rewards']) + avg_value_err /= len(oe['episode_rewards']) + + d['timestep'] = int(i * eval_freq) + d['undisc_returns'] = avg_undisc_returns + d['value_err'] = avg_value_err + + offline_eval.append(d) + +all_res_df = pd.DataFrame(offline_eval) + + +# %% codecell +arch = "gru" +measure_col = 'undisc_returns' +all_args = list(info['args'].keys()) + +offline_eval_df = all_res_df[ + (all_res_df['arch'] == arch) + & (all_res_df['multihead_lambda_coeff'] >= 0.) +# & (all_res_df['multihead_lambda_coeff'] != 1.) +# & (all_res_df['multihead_loss_mode'] != 'td') +# & (all_res_df['multihead_action_mode'] == 'td') +] + +unique_seeds = offline_eval_df.seed.unique() + +split_by_args = ['spec', 'multihead_action_mode', 'multihead_loss_mode', 'multihead_lambda_coeff', 'lr'] + +# For split exp +# split_by_args = ['spec', 'multihead_loss_mode', 'lr'] + +rnn_grouped = offline_eval_df.groupby(split_by_args, as_index=False) +# %% codecell +all_res_df +# %% codecell +# take mean and std_err over mean over seeds +rnn_mean = rnn_grouped.mean(numeric_only=True) +rnn_sem = rnn_grouped.sem(numeric_only=True) + +# Now we take the max over everything but learning rate +max_over_args = ['lr'] +grouped_args = [arg for arg in split_by_args if arg not in max_over_args] +rnn_less_lr_grouped = rnn_mean.groupby(grouped_args, as_index=False) + +rnn_arg_maxes = rnn_mean.sort_values(measure_col).drop_duplicates(grouped_args, keep='last') +# %% codecell +# drop columns we need in original df +pruned_rnn_arg_maxes = rnn_arg_maxes.drop(columns=['timestep', 'seed']) + +# A df of results over all the maxes. +max_offline_eval_df = pruned_rnn_arg_maxes.merge(offline_eval_df, on=split_by_args, how='inner') + +# %% codecell +w = 100 +mode = 'same' +max_offline_eval_df[f'avged_returns'] = max_offline_eval_df.groupby(split_by_args)['undisc_returns_y'].transform(lambda x: np.convolve(x, np.ones(w), mode=mode) / w) +# %% codecell +df = max_offline_eval_df +# pivot_value_key = 'undisc_returns_y' +pivot_value_key = 'avged_returns' + +# pivot_value_key = 'value_err_y' + + +# df = offline_eval_df +# pivot_value_key = 'undisc_returns' + +all_labels = \ + sorted(['_'.join([str(el) for el in row]) for row in df[[arg for arg in grouped_args if (arg != 'spec') and (arg != 'timestep')]]\ + .drop_duplicates()\ + .to_numpy()]) + +label_cmap = {} + +color = iter(cm.tab10(np.linspace(0, 1, 10))) +for i in range(len(all_labels)): + c = next(color) + label_cmap[all_labels[i]] = c +# %% codecell + +# %% codecell +spec_strs = df['spec'].unique() + +pivot_cols = [arg for arg in grouped_args if (arg != 'spec') and (arg != 'timestep')] + ['seed'] + +n_specs = len(spec_strs) + +n_rows, n_cols = 2, np.ceil(n_specs / 2).astype(int) +if n_specs == 1: + n_rows = 1 + +fig, axes = plt.subplots(n_rows, n_cols, figsize=(24, 10)) + +for i, spec in tqdm(list(enumerate(sorted(spec_strs)))): + row = i // n_cols + col = i % n_cols + + ax = axes + if n_specs > 1: + ax = axes[row, col] + + spec_df = df[df['spec'] == spec] + sorted_spec_df = spec_df.sort_values(by=grouped_args) + + # Make a pivot table + pivot_df = sorted_spec_df.pivot(index='timestep', columns=pivot_cols, values=pivot_value_key) + + # Flatten all columns except seed + col_name = '_'.join([col for col in pivot_cols if col != 'seed']) + multi_index = pd.MultiIndex.from_tuples([('_'.join([str(c) for c in col[:-1]]), col[-1]) for col in pivot_df.columns], names=[col_name, "seed"]) + + pivot_df.columns = multi_index + + sns.lineplot(ax=ax, data=pivot_df, palette=label_cmap) + + ax.get_legend().remove() + ax.set_title(spec) + if col > 0: + ax.set_ylabel(None) + if row < n_rows - 1: + ax.set_xlabel(None) + + handles, labels = ax.get_legend_handles_labels() + + if spec == 'rocksample': + ax.set_ylim(0, 35) + step_per_eval = 1000 + ax.set_xlim(w * step_per_eval // 2, df['timestep'].max() - w * step_per_eval // 2) +# if i >= 1: +# break + +fig.legend(handles, labels, loc='lower right') +fig.suptitle(f"{measure_col} {arch}") +fig.tight_layout() +# %% codecell +# fig_path = Path(ROOT_DIR, 'results', f'{measure_col}_{arch}_rnn_td.pdf') +# fig.savefig(fig_path, bbox_inches='tight') +# %% codecell +# for results_path in tqdm(all_results_paths): +# if results_path.is_dir() or results_path.suffix != '.npy': +# continue + +# info = load_info(results_path) +# args = info['args'] +# if args['multihead_lambda_coeff'] == -1: +# results_path.unlink() +# %% codecell +() diff --git a/scripts/plotting/tiger_dm_mi_performance.ipynb b/scripts/plotting/tiger_dm_mi_performance.ipynb deleted file mode 100644 index 67113ed4..00000000 --- a/scripts/plotting/tiger_dm_mi_performance.ipynb +++ /dev/null @@ -1,967 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "6f06962c", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import jax.numpy as jnp\n", - "import matplotlib.pyplot as plt\n", - "import matplotlib.cm as cm\n", - "from jax.nn import softmax\n", - "from jax.config import config\n", - "from pathlib import Path\n", - "from collections import namedtuple\n", - "\n", - "config.update('jax_platform_name', 'cpu')\n", - "np.set_printoptions(precision=4)\n", - "plt.rcParams['axes.facecolor'] = 'white'\n", - "plt.rcParams.update({'font.size': 22})\n", - "\n", - "from grl.utils import load_info\n", - "from grl.utils.mdp import get_perf\n", - "from definitions import ROOT_DIR" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "739315d8", - "metadata": {}, - "outputs": [], - "source": [ - "# results_dir = Path(ROOT_DIR, 'results', 'pomdps_mi_pi')\n", - "# results_dir = Path(ROOT_DIR, 'results', 'pomdps_mi_pi_og')\n", - "\n", - "# results_dir = Path(ROOT_DIR, 'results', 'tiger-alt-start_mi_pi_q_abs')\n", - "# results_dir = Path(ROOT_DIR, 'results', 'tiger-alt-start_mi_dm_q_abs')\n", - "results_dir = Path(ROOT_DIR, 'results', 'old', 'tiger-alt-start_mi_dm_q_abs(2 mi_iterations, 100k mi_steps)')\n", - "\n", - "# results_dir = Path(ROOT_DIR, 'results', 'pomdps_mi_dm')\n", - "vi_results_dir = Path(ROOT_DIR, 'results', 'pomdps_vi')\n", - "pomdp_files_dir = Path(ROOT_DIR, 'grl', 'environment', 'pomdp_files')\n", - "\n", - "split_by = ['spec', 'algo', 'n_mem_states']\n", - "Args = namedtuple('args', split_by)\n", - "# this option allows us to compare to either the optimal belief state soln\n", - "# or optimal state soln. ('belief' | 'state')\n", - "compare_to = 'belief'\n", - "\n", - "# spec_plot_order = ['example_7', 'slippery_tmaze_5_two_thirds_up',\n", - "# 'tiger', 'paint.95', 'cheese.95',\n", - "# 'network', 'shuttle.95', '4x3.95']\n", - "spec_plot_order = [\n", - "# 'example_7', \n", - " 'tiger-alt-start'\n", - "]\n", - "\n", - "spec_to_belief_state = {'tmaze_5_two_thirds_up': 'tmaze5'}" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "dee0c91f", - "metadata": {}, - "outputs": [], - "source": [ - "all_results = {}\n", - "\n", - "for results_path in results_dir.iterdir():\n", - " if results_path.is_dir() or results_path.suffix != '.npy':\n", - " continue\n", - " info = load_info(results_path)\n", - "\n", - " args = info['args']\n", - " if 'n_mem_states' not in args:\n", - " args['n_mem_states'] = 2\n", - " # agent = info['agent']\n", - " init_policy_info = info['logs']['initial_policy_stats']\n", - " init_improvement_info = info['logs']['greedy_initial_improvement_stats']\n", - " final_mem_info = info['logs']['greedy_final_mem_stats']\n", - " greedy_td_optimal_policy_stats = info['logs']['greedy_td_optimal_policy_stats']\n", - "\n", - "\n", - " single_res = {\n", - " 'init_policy_perf': get_perf(init_policy_info),\n", - "# 'init_improvement_perf': get_perf(init_improvement_info),\n", - " 'init_improvement_perf': get_perf(greedy_td_optimal_policy_stats),\n", - " 'final_mem_perf': get_perf(final_mem_info),\n", - " 'init_policy': info['logs']['initial_policy'],\n", - " 'init_improvement_policy': info['logs']['initial_improvement_policy'],\n", - " # 'final_mem': np.array(agent.memory),\n", - " # 'final_policy': np.array(agent.policy)\n", - " }\n", - "\n", - " hparams = Args(*tuple(args[s] for s in split_by))\n", - "\n", - " if hparams not in all_results:\n", - " all_results[hparams] = {}\n", - "\n", - " for k, v in single_res.items():\n", - " if k not in all_results[hparams]:\n", - " all_results[hparams][k] = []\n", - " all_results[hparams][k].append(v)\n", - " all_results[hparams]['args'] = args\n", - "\n", - "for hparams, res_dict in all_results.items():\n", - " for k, v in res_dict.items():\n", - " if k != 'args':\n", - " all_results[hparams][k] = np.stack(v)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a304c71a", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "c4f4572e", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[2.8284e-01, 4.0065e-01, 3.1650e-01],\n", - " [5.8773e-01, 4.1227e-01, 1.3896e-07],\n", - " [1.0000e+00, 4.5790e-07, 3.9510e-08],\n", - " [9.9999e-01, 6.5812e-06, 6.5430e-06]])" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# results_path = Path(ROOT_DIR, 'results', 'tiger-alt-start_mi_pi(dm)_miit(2)_s(2023)_Sat Jan 21 11:41:13 2023.npy')\n", - "results_path = list(results_dir.iterdir())[-1]\n", - "agent_path = results_path.parent / 'agents' / f\"{results_path.stem}.pkl.npy\"\n", - "info = load_info(results_path)\n", - "agent = load_info(agent_path)\n", - "\n", - "init_improvement_policy = info['logs']['initial_improvement_policy']\n", - "init_improvement_policy" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "0c355643", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(-7.174999999999999,\n", - " -1.8474502548598393,\n", - " {'v_discrep': 0.45380196587980703, 'q_discrep': 0.35620424562337677})" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "def calc_lambda_discrep(stats: dict):\n", - " v_diff_discrep = np.abs(stats['td_vals_v'] - stats['mc_vals_v']).mean()\n", - " q_diff_discrep = np.abs(stats['td_vals_q'] - stats['mc_vals_q']).mean()\n", - " return {'v_discrep': v_diff_discrep, 'q_discrep': q_diff_discrep}\n", - "\n", - "init_improvement_stats = info['logs']['initial_improvement_stats']\n", - "greedy_init_improvement_stats = info['logs']['greedy_initial_improvement_stats']\n", - "\n", - "td_optimal_policy_stats = info['logs']['td_optimal_policy_stats']\n", - "greedy_td_optimal_policy_stats = info['logs']['greedy_td_optimal_policy_stats']\n", - "\n", - "final_mem_stats = info['logs']['final_mem_stats']\n", - "greedy_final_mem_stats = info['logs']['greedy_final_mem_stats']\n", - "\n", - "# get_perf(init_improvement_stats), get_perf(final_mem_stats)\n", - "get_perf(greedy_td_optimal_policy_stats), get_perf(greedy_final_mem_stats), calc_lambda_discrep(final_mem_stats)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "efa47c75", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "({'v': array([1.5309e-18, 0.0000e+00, 1.3496e+01, 0.0000e+00, 3.1554e-30,\n", - " 5.9657e-03, 0.0000e+00, 0.0000e+00]),\n", - " 'q': array([[1.5309e-18, 0.0000e+00, 1.3496e+01, 2.6350e+01, 4.0644e-03,\n", - " 5.9657e-03, 1.2326e-32, 0.0000e+00],\n", - " [0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 3.1554e-30,\n", - " 7.0997e-30, 0.0000e+00, 0.0000e+00],\n", - " [0.0000e+00, 0.0000e+00, 2.0195e-28, 0.0000e+00, 0.0000e+00,\n", - " 2.0195e-28, 0.0000e+00, 0.0000e+00]]),\n", - " 'mc_vals_q': array([[ -1.8475, 0. , 1.5687, 2.6335, 6.631 , 4.7138,\n", - " -1. , -1. ],\n", - " [-45. , 0. , -16.6646, -83.5794, 8.9147, -6.5797,\n", - " 0. , 0. ],\n", - " [-45. , 0. , -73.3354, -6.4206, -98.9147, -83.4203,\n", - " 0. , 0. ]]),\n", - " 'td_vals_q': array([[ -1.8475, 0. , 5.2423, -2.4997, 6.5673, 4.6365,\n", - " -1. , -1. ],\n", - " [-45. , 0. , -16.6646, -83.5794, 8.9147, -6.5797,\n", - " 0. , 0. ],\n", - " [-45. , 0. , -73.3354, -6.4206, -98.9147, -83.4203,\n", - " 0. , 0. ]]),\n", - " 'mc_vals_v': array([-1.8475, 0. , 1.5687, -6.4206, 8.9147, 4.7138, 0. ,\n", - " 0. ]),\n", - " 'td_vals_v': array([-1.8475, 0. , 5.2423, -6.4206, 8.9147, 4.6365, 0. ,\n", - " 0. ]),\n", - " 'state_vals_v': array([ 7.387 , -6.6362, -11.0819, -3.3474, -27.6856, 8.8285,\n", - " 9.5269, -10.6126, 0. , 0. ]),\n", - " 'state_vals_q': array([[ 7.387 , -6.6362, -11.0819, -3.3474, -14.9242, 2.1898,\n", - " 6.8458, 5.1617, -1. , -1. ],\n", - " [-100. , -100. , 10. , 10. , -100. , -100. ,\n", - " 10. , 10. , 0. , 0. ],\n", - " [ 10. , 10. , -100. , -100. , 10. , 10. ,\n", - " -100. , -100. , 0. , 0. ]]),\n", - " 'p0': array([0.5, 0. , 0.5, 0. , 0. , 0. , 0. , 0. , 0. , 0. ]),\n", - " 'q_sum': array(1.1563)},\n", - " {'v': array([1.6808e-18, 0.0000e+00, 1.0570e+01, 2.3548e-02, 5.7825e-05,\n", - " 4.7619e-02, 2.0639e-20, 2.0817e-20]),\n", - " 'q': array([[1.9009e-18, 0.0000e+00, 1.2134e+01, 2.1193e+01, 5.2043e-02,\n", - " 5.4665e-02, 2.0727e-20, 2.0921e-20],\n", - " [2.0865e-20, 0.0000e+00, 2.0796e-20, 2.0752e-20, 2.0766e-20,\n", - " 2.0779e-20, 2.0880e-20, 2.0819e-20],\n", - " [2.0865e-20, 0.0000e+00, 2.0891e-20, 2.0799e-20, 2.0908e-20,\n", - " 2.0900e-20, 2.0627e-20, 2.0814e-20]]),\n", - " 'mc_vals_q': array([[ -7.0565, 0. , -3.6726, -2.174 , 1.7474, -0.1976,\n", - " -1.6333, -1.6333],\n", - " [-45.6333, 0. , -19.8034, -84.5582, 8.1442, -7.5674,\n", - " -0.6333, -0.6333],\n", - " [-45.6333, 0. , -71.4632, -6.7084, -99.4108, -83.6992,\n", - " -0.6333, -0.6333]]),\n", - " 'td_vals_q': array([[ -7.0565, 0. , -0.1892, -6.7776, 1.5193, -0.4314,\n", - " -1.6333, -1.6333],\n", - " [-45.6333, 0. , -19.8034, -84.5582, 8.1442, -7.5674,\n", - " -0.6333, -0.6333],\n", - " [-45.6333, 0. , -71.4632, -6.7084, -99.4108, -83.6992,\n", - " -0.6333, -0.6333]]),\n", - " 'mc_vals_v': array([-9.6283, 0. , -6.4699, -9.1523, 4.3458, -3.2266, -0.6667,\n", - " -0.6667]),\n", - " 'td_vals_v': array([-9.6283, 0. , -3.2188, -9.3057, 4.3382, -3.4448, -0.6667,\n", - " -0.6667]),\n", - " 'state_vals_v': array([ -0.7057, -13.5244, -18.5509, -11.0609, -32.0741, 3.6879,\n", - " 4.4573, -16.4384, -0.6667, -0.6667]),\n", - " 'state_vals_q': array([[ 2.5034, -11.2309, -16.6165, -8.5915, -19.3482, -2.5867,\n", - " 1.9845, 0.2371, -1.6333, -1.6333],\n", - " [-100.6333, -100.6333, 9.3667, 9.3667, -100.6333, -100.6333,\n", - " 9.3667, 9.3667, -0.6333, -0.6333],\n", - " [ 9.3667, 9.3667, -100.6333, -100.6333, 9.3667, 9.3667,\n", - " -100.6333, -100.6333, -0.6333, -0.6333]]),\n", - " 'p0': array([0.5, 0. , 0.5, 0. , 0. , 0. , 0. , 0. , 0. , 0. ]),\n", - " 'q_sum': array(1.2769)})" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "greedy_final_mem_stats, final_mem_stats" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "24e13ec6", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(DeviceArray([[[[9.9719e-07, 1.0000e+00],\n", - " [4.0426e-01, 5.9574e-01]],\n", - " \n", - " [[5.8023e-01, 4.1977e-01],\n", - " [9.2616e-07, 1.0000e+00]],\n", - " \n", - " [[1.0000e+00, 6.8763e-07],\n", - " [9.9883e-01, 1.1668e-03]],\n", - " \n", - " [[3.8166e-01, 6.1834e-01],\n", - " [7.2587e-01, 2.7413e-01]]],\n", - " \n", - " \n", - " [[[6.2555e-01, 3.7445e-01],\n", - " [5.2054e-01, 4.7946e-01]],\n", - " \n", - " [[7.0851e-01, 2.9149e-01],\n", - " [3.8941e-01, 6.1059e-01]],\n", - " \n", - " [[4.0307e-01, 5.9693e-01],\n", - " [5.0259e-01, 4.9741e-01]],\n", - " \n", - " [[6.5246e-01, 3.4754e-01],\n", - " [5.4487e-01, 4.5513e-01]]],\n", - " \n", - " \n", - " [[[6.2514e-01, 3.7486e-01],\n", - " [5.5853e-01, 4.4147e-01]],\n", - " \n", - " [[5.4279e-01, 4.5721e-01],\n", - " [4.1864e-01, 5.8136e-01]],\n", - " \n", - " [[7.3612e-01, 2.6388e-01],\n", - " [7.8108e-01, 2.1892e-01]],\n", - " \n", - " [[2.0307e-01, 7.9693e-01],\n", - " [5.3494e-01, 4.6506e-01]]]], dtype=float32),\n", - " DeviceArray([[0.9333, 0.0333, 0.0333],\n", - " [0.9333, 0.0333, 0.0333],\n", - " [0.9333, 0.0333, 0.0333],\n", - " [0.0333, 0.0333, 0.9333],\n", - " [0.0333, 0.9333, 0.0333],\n", - " [0.9333, 0.0333, 0.0333],\n", - " [0.0333, 0.0333, 0.9333],\n", - " [0.0333, 0.0333, 0.9333]], dtype=float32))" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "agent.memory, agent.policy" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "0ec56846", - "metadata": {}, - "outputs": [], - "source": [ - "if compare_to == 'belief':\n", - " for fname in pomdp_files_dir.iterdir():\n", - " if 'pomdp-solver-results' in fname.stem:\n", - " for hparams in all_results.keys():\n", - " if (fname.stem ==\n", - " f\"{spec_to_belief_state.get(hparams.spec, hparams.spec)}-pomdp-solver-results\"\n", - " ):\n", - " belief_info = load_info(fname)\n", - " coeffs = belief_info['coeffs']\n", - " max_start_vals = coeffs[belief_info['max_start_idx']]\n", - " all_results[hparams]['compare_perf'] = np.array(\n", - " [np.dot(max_start_vals, belief_info['p0'])])\n", - " # print(f\"loaded results for {hparams.spec} from {fname}\")\n", - "\n", - "elif compare_to == 'state':\n", - " for hparams, res_dict in all_results.items():\n", - " for vi_path in vi_results_dir.iterdir():\n", - " if hparams.spec in vi_path.name:\n", - " vi_info = load_info(vi_path)\n", - " all_results[hparams]['compare_perf'] = np.array([\n", - " (vi_info['optimal_vs'] * vi_info['p0']).sum()\n", - " ])\n", - "else:\n", - " raise NotImplementedError" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "d677a428", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'init_policy_perf': array([-52.4978, -52.2354, -51.5068, -50.295 , -52.2378, -54.4271,\n", - " -53.1488, -46.8697, -49.3594, -50.4529, -49.1381, -48.8586,\n", - " -50.0219, -49.5627, -50.4812, -51.1004, -50.6628, -47.7137,\n", - " -50.4953, -49.3584, -52.8769, -49.8966, -51.5698, -47.8228,\n", - " -50.8433, -49.6393, -52.4657, -51.9119, -50.9637, -50.0235]),\n", - " 'init_improvement_perf': array([-7.175, -7.175, -7.175, -7.175, -7.175, -7.175, -7.175, -7.175,\n", - " -7.175, -7.175, -7.175, -7.175, -7.175, -7.175, -7.175, -7.175,\n", - " -7.175, -7.175, -7.175, -7.175, -7.175, -7.175, -7.175, -7.175,\n", - " -7.175, -7.175, -7.175, -7.175, -7.175, -7.175]),\n", - " 'final_mem_perf': array([ -7.175 , -7.175 , -7.5399, -7.54 , -1.8474, -1.7827,\n", - " -7.175 , -7.5402, -7.5399, -7.175 , -7.1751, -7.175 ,\n", - " -9.2015, -10.3617, -11.9383, -9.7863, -9.5748, -7.5402,\n", - " -1.8798, -11.5764, -9.8554, -1.7969, -7.5399, -10.6936,\n", - " -7.175 , -7.5399, -7.175 , -7.175 , -7.175 , -1.8475]),\n", - " 'init_policy': array([[[0.3217, 0.4049, 0.2733],\n", - " [0.32 , 0.3791, 0.301 ],\n", - " [0.4068, 0.2736, 0.3197],\n", - " [0.3784, 0.371 , 0.2506]],\n", - " \n", - " [[0.3708, 0.3295, 0.2997],\n", - " [0.3412, 0.4134, 0.2454],\n", - " [0.3161, 0.2882, 0.3957],\n", - " [0.2803, 0.3432, 0.3765]],\n", - " \n", - " [[0.3071, 0.3221, 0.3708],\n", - " [0.2874, 0.401 , 0.3116],\n", - " [0.2662, 0.3569, 0.3769],\n", - " [0.3325, 0.406 , 0.2615]],\n", - " \n", - " [[0.2812, 0.3653, 0.3535],\n", - " [0.3276, 0.3055, 0.3668],\n", - " [0.2559, 0.4078, 0.3363],\n", - " [0.3609, 0.3111, 0.3279]],\n", - " \n", - " [[0.3449, 0.3358, 0.3193],\n", - " [0.4127, 0.3566, 0.2306],\n", - " [0.3178, 0.3135, 0.3686],\n", - " [0.3297, 0.2121, 0.4582]],\n", - " \n", - " [[0.3839, 0.3904, 0.2257],\n", - " [0.3113, 0.503 , 0.1857],\n", - " [0.4522, 0.2537, 0.2941],\n", - " [0.3552, 0.2428, 0.402 ]],\n", - " \n", - " [[0.3615, 0.1842, 0.4543],\n", - " [0.3583, 0.3769, 0.2648],\n", - " [0.3511, 0.2777, 0.3712],\n", - " [0.3669, 0.3603, 0.2728]],\n", - " \n", - " [[0.3225, 0.3374, 0.3401],\n", - " [0.2553, 0.3002, 0.4444],\n", - " [0.4333, 0.3432, 0.2235],\n", - " [0.2576, 0.4341, 0.3083]],\n", - " \n", - " [[0.3209, 0.3947, 0.2844],\n", - " [0.2139, 0.2999, 0.4863],\n", - " [0.2368, 0.3185, 0.4447],\n", - " [0.284 , 0.3624, 0.3536]],\n", - " \n", - " [[0.3483, 0.309 , 0.3427],\n", - " [0.3342, 0.4151, 0.2507],\n", - " [0.3122, 0.4161, 0.2717],\n", - " [0.3161, 0.3484, 0.3355]],\n", - " \n", - " [[0.3554, 0.3122, 0.3324],\n", - " [0.3455, 0.2627, 0.3917],\n", - " [0.3382, 0.3239, 0.338 ],\n", - " [0.3194, 0.3909, 0.2897]],\n", - " \n", - " [[0.3897, 0.3213, 0.289 ],\n", - " [0.2246, 0.4108, 0.3646],\n", - " [0.3063, 0.3828, 0.3109],\n", - " [0.2602, 0.3834, 0.3564]],\n", - " \n", - " [[0.2851, 0.3057, 0.4092],\n", - " [0.3078, 0.3713, 0.3208],\n", - " [0.371 , 0.3316, 0.2974],\n", - " [0.2931, 0.4282, 0.2786]],\n", - " \n", - " [[0.2522, 0.4825, 0.2654],\n", - " [0.3238, 0.3296, 0.3466],\n", - " [0.3339, 0.3993, 0.2668],\n", - " [0.3232, 0.353 , 0.3238]],\n", - " \n", - " [[0.3919, 0.3127, 0.2954],\n", - " [0.4017, 0.2496, 0.3487],\n", - " [0.3519, 0.2899, 0.3582],\n", - " [0.3615, 0.3235, 0.315 ]],\n", - " \n", - " [[0.3814, 0.318 , 0.3006],\n", - " [0.3579, 0.3168, 0.3253],\n", - " [0.3326, 0.2673, 0.4 ],\n", - " [0.3004, 0.4372, 0.2624]],\n", - " \n", - " [[0.2838, 0.2952, 0.421 ],\n", - " [0.3719, 0.3013, 0.3268],\n", - " [0.3065, 0.314 , 0.3795],\n", - " [0.3178, 0.3209, 0.3613]],\n", - " \n", - " [[0.3872, 0.3058, 0.307 ],\n", - " [0.4181, 0.228 , 0.3539],\n", - " [0.3618, 0.377 , 0.2613],\n", - " [0.3371, 0.3372, 0.3257]],\n", - " \n", - " [[0.3678, 0.2438, 0.3884],\n", - " [0.3177, 0.343 , 0.3393],\n", - " [0.3839, 0.3132, 0.3028],\n", - " [0.3388, 0.3163, 0.3449]],\n", - " \n", - " [[0.4689, 0.289 , 0.2421],\n", - " [0.3359, 0.3099, 0.3543],\n", - " [0.3179, 0.2927, 0.3894],\n", - " [0.2496, 0.3286, 0.4218]],\n", - " \n", - " [[0.3767, 0.3506, 0.2727],\n", - " [0.3252, 0.3399, 0.3349],\n", - " [0.3576, 0.2706, 0.3718],\n", - " [0.4044, 0.2984, 0.2972]],\n", - " \n", - " [[0.3549, 0.3824, 0.2626],\n", - " [0.298 , 0.3186, 0.3835],\n", - " [0.2917, 0.4109, 0.2974],\n", - " [0.3842, 0.3084, 0.3074]],\n", - " \n", - " [[0.3204, 0.3431, 0.3364],\n", - " [0.282 , 0.405 , 0.313 ],\n", - " [0.3043, 0.4414, 0.2543],\n", - " [0.4269, 0.2844, 0.2887]],\n", - " \n", - " [[0.3591, 0.3306, 0.3103],\n", - " [0.317 , 0.2487, 0.4343],\n", - " [0.1838, 0.3916, 0.4246],\n", - " [0.2635, 0.3626, 0.3738]],\n", - " \n", - " [[0.252 , 0.2961, 0.4519],\n", - " [0.3215, 0.3053, 0.3733],\n", - " [0.3228, 0.3453, 0.3319],\n", - " [0.3665, 0.3131, 0.3204]],\n", - " \n", - " [[0.3793, 0.3449, 0.2757],\n", - " [0.294 , 0.273 , 0.433 ],\n", - " [0.3109, 0.294 , 0.3951],\n", - " [0.3163, 0.2601, 0.4236]],\n", - " \n", - " [[0.2988, 0.3482, 0.353 ],\n", - " [0.2972, 0.2676, 0.4352],\n", - " [0.438 , 0.2451, 0.3169],\n", - " [0.4657, 0.2212, 0.3131]],\n", - " \n", - " [[0.4758, 0.2557, 0.2685],\n", - " [0.2962, 0.3528, 0.3511],\n", - " [0.3174, 0.3105, 0.3721],\n", - " [0.3792, 0.3454, 0.2754]],\n", - " \n", - " [[0.4024, 0.3347, 0.2629],\n", - " [0.219 , 0.3936, 0.3874],\n", - " [0.3398, 0.3123, 0.348 ],\n", - " [0.3358, 0.3376, 0.3266]],\n", - " \n", - " [[0.2827, 0.4007, 0.3166],\n", - " [0.3143, 0.3631, 0.3226],\n", - " [0.3658, 0.3862, 0.2481],\n", - " [0.3409, 0.3806, 0.2785]]]),\n", - " 'init_improvement_policy': array([[[3.2187e-01, 4.0485e-01, 2.7328e-01],\n", - " [5.8773e-01, 4.1227e-01, 1.3897e-07],\n", - " [1.0000e+00, 4.4552e-07, 4.0512e-08],\n", - " [9.9999e-01, 6.5877e-06, 6.5366e-06]],\n", - " \n", - " [[3.7089e-01, 3.2948e-01, 2.9963e-01],\n", - " [5.8773e-01, 4.1227e-01, 1.3897e-07],\n", - " [1.0000e+00, 4.5282e-07, 3.9932e-08],\n", - " [9.9999e-01, 6.5572e-06, 6.5674e-06]],\n", - " \n", - " [[3.0722e-01, 3.2206e-01, 3.7072e-01],\n", - " [5.8773e-01, 4.1227e-01, 1.3896e-07],\n", - " [1.0000e+00, 4.6587e-07, 3.8816e-08],\n", - " [9.9999e-01, 6.5887e-06, 6.5352e-06]],\n", - " \n", - " [[2.8109e-01, 3.6536e-01, 3.5355e-01],\n", - " [1.0000e+00, 1.2381e-07, 6.9446e-08],\n", - " [3.5955e-06, 1.0000e+00, 1.5655e-07],\n", - " [5.9466e-05, 2.6464e-01, 7.3530e-01]],\n", - " \n", - " [[3.4505e-01, 3.3571e-01, 3.1923e-01],\n", - " [5.8773e-01, 4.1227e-01, 1.3897e-07],\n", - " [1.0000e+00, 4.5431e-07, 3.9807e-08],\n", - " [9.9999e-01, 6.5137e-06, 6.6102e-06]],\n", - " \n", - " [[3.8398e-01, 3.9031e-01, 2.2571e-01],\n", - " [5.8773e-01, 4.1227e-01, 1.3898e-07],\n", - " [1.0000e+00, 4.3299e-07, 4.1440e-08],\n", - " [9.9999e-01, 6.5305e-06, 6.5947e-06]],\n", - " \n", - " [[3.6164e-01, 1.8421e-01, 4.5416e-01],\n", - " [5.8773e-01, 4.1227e-01, 1.3896e-07],\n", - " [1.0000e+00, 4.5343e-07, 3.9883e-08],\n", - " [9.9999e-01, 6.5797e-06, 6.5444e-06]],\n", - " \n", - " [[3.2239e-01, 3.3749e-01, 3.4012e-01],\n", - " [3.5339e-06, 1.5926e-07, 1.0000e+00],\n", - " [1.0000e+00, 7.0194e-08, 1.2259e-07],\n", - " [4.9749e-05, 9.6141e-01, 3.8538e-02]],\n", - " \n", - " [[3.2103e-01, 3.9459e-01, 2.8439e-01],\n", - " [1.0000e+00, 3.9407e-08, 4.5904e-07],\n", - " [5.8773e-01, 1.3897e-07, 4.1227e-01],\n", - " [9.9999e-01, 6.5641e-06, 6.5613e-06]],\n", - " \n", - " [[3.4845e-01, 3.0893e-01, 3.4262e-01],\n", - " [5.8773e-01, 4.1227e-01, 1.3897e-07],\n", - " [1.0000e+00, 4.5186e-07, 4.0001e-08],\n", - " [9.9999e-01, 6.5649e-06, 6.5605e-06]],\n", - " \n", - " [[3.5555e-01, 3.1211e-01, 3.3234e-01],\n", - " [1.0000e+00, 4.1262e-08, 4.3517e-07],\n", - " [5.8773e-01, 1.3902e-07, 4.1227e-01],\n", - " [9.9999e-01, 6.5827e-06, 6.5469e-06]],\n", - " \n", - " [[3.8980e-01, 3.2126e-01, 2.8895e-01],\n", - " [5.8773e-01, 4.1227e-01, 1.3897e-07],\n", - " [1.0000e+00, 4.5233e-07, 3.9973e-08],\n", - " [9.9999e-01, 6.5663e-06, 6.5585e-06]],\n", - " \n", - " [[2.8521e-01, 3.0564e-01, 4.0915e-01],\n", - " [5.8773e-01, 4.1227e-01, 1.3896e-07],\n", - " [1.0000e+00, 4.5686e-07, 3.9599e-08],\n", - " [9.9999e-01, 6.5866e-06, 6.5376e-06]],\n", - " \n", - " [[2.5233e-01, 4.8234e-01, 2.6532e-01],\n", - " [5.8773e-01, 4.1227e-01, 1.3898e-07],\n", - " [1.0000e+00, 4.5160e-07, 4.0021e-08],\n", - " [9.9999e-01, 6.5680e-06, 6.5577e-06]],\n", - " \n", - " [[3.9199e-01, 3.1265e-01, 2.9536e-01],\n", - " [1.0000e+00, 3.8932e-08, 4.6457e-07],\n", - " [5.8773e-01, 1.3896e-07, 4.1227e-01],\n", - " [9.9999e-01, 6.5636e-06, 6.5603e-06]],\n", - " \n", - " [[3.8150e-01, 3.1791e-01, 3.0059e-01],\n", - " [1.0000e+00, 3.9543e-08, 4.5753e-07],\n", - " [5.8773e-01, 1.3896e-07, 4.1227e-01],\n", - " [9.9999e-01, 6.5917e-06, 6.5323e-06]],\n", - " \n", - " [[2.8399e-01, 2.9510e-01, 4.2091e-01],\n", - " [1.0000e+00, 3.9714e-08, 4.5548e-07],\n", - " [5.8773e-01, 1.3896e-07, 4.1227e-01],\n", - " [9.9999e-01, 6.5552e-06, 6.5691e-06]],\n", - " \n", - " [[3.8714e-01, 3.0582e-01, 3.0704e-01],\n", - " [3.6538e-06, 1.5382e-07, 1.0000e+00],\n", - " [1.0000e+00, 6.8050e-08, 1.2596e-07],\n", - " [6.0987e-05, 6.6198e-01, 3.3796e-01]],\n", - " \n", - " [[3.6787e-01, 2.4380e-01, 3.8833e-01],\n", - " [5.8773e-01, 4.1227e-01, 1.3896e-07],\n", - " [1.0000e+00, 4.5809e-07, 3.9496e-08],\n", - " [9.9999e-01, 6.5569e-06, 6.5673e-06]],\n", - " \n", - " [[4.6897e-01, 2.8893e-01, 2.4211e-01],\n", - " [1.0000e+00, 3.9565e-08, 4.5725e-07],\n", - " [5.8773e-01, 1.3896e-07, 4.1227e-01],\n", - " [9.9999e-01, 6.5493e-06, 6.5754e-06]],\n", - " \n", - " [[3.7683e-01, 3.5051e-01, 2.7267e-01],\n", - " [1.0000e+00, 3.9597e-08, 4.5686e-07],\n", - " [5.8773e-01, 1.3896e-07, 4.1227e-01],\n", - " [9.9999e-01, 6.5623e-06, 6.5618e-06]],\n", - " \n", - " [[3.5483e-01, 3.8252e-01, 2.6265e-01],\n", - " [1.0000e+00, 1.2629e-07, 6.7830e-08],\n", - " [3.6602e-06, 1.0000e+00, 1.5351e-07],\n", - " [6.2302e-05, 5.1812e-01, 4.8182e-01]],\n", - " \n", - " [[3.2059e-01, 3.4304e-01, 3.3637e-01],\n", - " [5.8773e-01, 4.1227e-01, 1.3896e-07],\n", - " [1.0000e+00, 4.5577e-07, 3.9687e-08],\n", - " [9.9999e-01, 6.5610e-06, 6.5631e-06]],\n", - " \n", - " [[3.5923e-01, 3.3051e-01, 3.1026e-01],\n", - " [1.0000e+00, 4.0878e-08, 4.4070e-07],\n", - " [5.8773e-01, 1.3897e-07, 4.1227e-01],\n", - " [9.9999e-01, 6.5611e-06, 6.5644e-06]],\n", - " \n", - " [[2.5221e-01, 2.9603e-01, 4.5176e-01],\n", - " [1.0000e+00, 4.0897e-08, 4.4020e-07],\n", - " [5.8773e-01, 1.3900e-07, 4.1227e-01],\n", - " [9.9999e-01, 6.5626e-06, 6.5655e-06]],\n", - " \n", - " [[3.7945e-01, 3.4486e-01, 2.7568e-01],\n", - " [1.0000e+00, 4.1280e-08, 4.3492e-07],\n", - " [5.8773e-01, 1.3902e-07, 4.1227e-01],\n", - " [9.9999e-01, 6.5354e-06, 6.5942e-06]],\n", - " \n", - " [[2.9867e-01, 3.4826e-01, 3.5307e-01],\n", - " [3.5956e-06, 1.5655e-07, 1.0000e+00],\n", - " [1.0000e+00, 6.9429e-08, 1.2383e-07],\n", - " [4.8743e-05, 2.4331e-02, 9.7562e-01]],\n", - " \n", - " [[4.7585e-01, 2.5571e-01, 2.6844e-01],\n", - " [1.0000e+00, 3.9997e-08, 4.5194e-07],\n", - " [5.8773e-01, 1.3897e-07, 4.1227e-01],\n", - " [9.9999e-01, 6.5771e-06, 6.5478e-06]],\n", - " \n", - " [[4.0232e-01, 3.3474e-01, 2.6294e-01],\n", - " [3.5999e-06, 1.5635e-07, 1.0000e+00],\n", - " [1.0000e+00, 6.9371e-08, 1.2392e-07],\n", - " [6.1084e-05, 6.5634e-01, 3.4360e-01]],\n", - " \n", - " [[2.8284e-01, 4.0065e-01, 3.1650e-01],\n", - " [5.8773e-01, 4.1227e-01, 1.3896e-07],\n", - " [1.0000e+00, 4.5790e-07, 3.9510e-08],\n", - " [9.9999e-01, 6.5812e-06, 6.5430e-06]]]),\n", - " 'args': {'spec': 'tiger-alt-start',\n", - " 'run_generated': None,\n", - " 'algo': 'mi',\n", - " 'mi_iterations': 2,\n", - " 'mi_steps': 100000,\n", - " 'pi_steps': 100000,\n", - " 'policy_optim_alg': 'dm',\n", - " 'pomdp_id': None,\n", - " 'mem_fn_id': None,\n", - " 'method': 'a',\n", - " 'n_random_policies': 0,\n", - " 'use_memory': 0,\n", - " 'n_mem_states': 2,\n", - " 'weight_discrep': False,\n", - " 'use_grad': 'm',\n", - " 'value_type': 'q',\n", - " 'error_type': 'abs',\n", - " 'lr': 1.0,\n", - " 'heatmap': False,\n", - " 'n_episodes': 500,\n", - " 'generate_pomdps': None,\n", - " 'log': False,\n", - " 'experiment_name': 'tiger-alt-start_mi_dm_q_abs',\n", - " 'platform': 'cpu',\n", - " 'seed': 2025,\n", - " 'tmaze_corridor_length': None,\n", - " 'tmaze_discount': None,\n", - " 'tmaze_junction_up_pi': None},\n", - " 'compare_perf': array([3.7702])}" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "list(all_results.values())[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "3c147eb8", - "metadata": {}, - "outputs": [], - "source": [ - "for hparams, res in all_results.items():\n", - " max_key = 'compare_perf'\n", - " if max_key not in res:\n", - " print(hparams)\n", - " # max_key = 'final_mem_perf'\n", - " max_v = res[max_key]\n", - " min_v = res['init_policy_perf'].min()\n", - " for k, v in res.items():\n", - " if '_perf' in k:\n", - " all_results[hparams][k] = (v - min_v) / (max_v - min_v)" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "7c946a76", - "metadata": {}, - "outputs": [], - "source": [ - "all_table_results = {}\n", - "all_plot_results = {'x': [], 'xlabels': []}\n", - "\n", - "for i, spec in enumerate(spec_plot_order):\n", - " hparams = sorted([k for k in all_results.keys() if k.spec == spec],\n", - " key=lambda hp: hp.n_mem_states)\n", - "\n", - " first_res = all_results[hparams[0]]\n", - " all_plot_results['x'].append(i)\n", - " all_plot_results['xlabels'].append(spec)\n", - "\n", - " # we first add initial and first improvement stats\n", - " for k, v in first_res.items():\n", - " if 'perf' in k and k != 'final_mem_perf':\n", - " mean = v.max(axis=0)\n", - " std_err = v.std(axis=0) / np.sqrt(v.shape[0])\n", - "\n", - " stripped_str = k.replace('_perf', '')\n", - " if stripped_str not in all_plot_results:\n", - " all_plot_results[stripped_str] = {'mean': [], 'std_err': []}\n", - " all_plot_results[stripped_str]['mean'].append(mean)\n", - " all_plot_results[stripped_str]['std_err'].append(std_err)\n", - "\n", - " # now we add final memory perf, for each n_mem_states\n", - " for hparam in hparams:\n", - " res = all_results[hparam]\n", - " for k, v in res.items():\n", - " if k == 'final_mem_perf':\n", - " mean = v.max(axis=0)\n", - " std_err = v.std(axis=0) / np.sqrt(v.shape[0])\n", - " stripped_str = k.replace('_perf', '')\n", - " mem_label = f\"mem_{hparam.n_mem_states}\"\n", - " if mem_label not in all_plot_results:\n", - " all_plot_results[mem_label] = {'mean': [], 'std_err': []}\n", - " all_plot_results[mem_label]['mean'].append(mean)\n", - " all_plot_results[mem_label]['std_err'].append(std_err)\n", - "\n", - "ordered_plot = []\n", - "# ordered_plot.append(('init_policy', all_plot_results['init_policy']))\n", - "ordered_plot.append(('init_improvement', all_plot_results['init_improvement']))\n", - "for k in sorted(all_plot_results.keys()):\n", - " if 'mem' in k:\n", - " ordered_plot.append((k, all_plot_results[k]))\n", - "# ordered_plot.append(('state_optimal', all_plot_results['vi']))\n" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "id": "3db92c53", - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvIAAAIqCAYAAABG9q4OAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAACBnUlEQVR4nO3dd1gU594+8HvoSFNBRAXBFiPGoFhALGCJKGpULMFeEsQWkxiTmGaLKZ6oJ8eW2LFE0VhiV1BBlIhdATVWSFAB6UUFKfP7g9/OC6HvDgu7uT/XtVdmd5/yXfA9773DM88IoiiKICIiIiIijaJT0wUQEREREVHVMcgTEREREWkgBnkiIiIiIg3EIE9EREREpIEY5ImIiIiINBCDPBERERGRBmKQJyIiIiLSQAzyREREREQaiEGeiIiIiEgDMcgTEREREWkgBnmZ5OfnIyoqCv7+/nj//ffRtWtX1KlTB4IgQBAETJo0qdrmPnToEEaOHAkHBwcYGRnB2toabm5u+PHHH5GRkVFt8xIRERFRzdGr6QK0xahRo7B//361zpmVlYWxY8fi0KFDxV5PTExEYmIiLly4gFWrVmHPnj1wdXVVa21EREREVL14Rl4m+fn5xZ7Xr18frVq1qtb5Ro4cKYX4hg0b4quvvsLOnTuxevVqdOvWDQAQGxsLLy8v3Llzp9pqISIiIiL14xl5mXTp0gVt2rRBx44d0bFjRzRr1gz+/v6YPHlytcy3ceNGnDhxAgDg6OiIM2fOoGHDhtL7M2fOxNy5c7F8+XKkpqbCz88PoaGh1VILEREREamfIIqiWNNFaKuiQX7ixInw9/eXZdz8/HzY2dkhLi4OAHD16lU4OzuX2q5Tp064ceMGAODkyZPo16+fLDUQERERUc3i0hoNFBoaKoV4d3f3UkM8AOjq6mL27NnS8127dqmlPiIiIiKqfgzyGuj48ePSsZeXV7ltBwwYUGo/IiIiItJsDPIaKDIyUjru3LlzuW1tbGxgZ2cHAEhISEBiYmK11kZERERE6sEgr4Hu3r0rHTdr1qzC9kXbFO1LRERERJqLQV4DpaWlScdWVlYVtre0tCy1LxERERFpLm4/qYGysrKkYyMjowrbGxsbS8eZmZlltsvJyUFOTo70vKCgACkpKbC0tIQgCEpWS0RERESVJYoiMjMz0bhxY+jolH/OnUGeJN9//z0WLVpU02UQERER/evFxsbC1ta23DYM8hrI1NQUqampAIDs7GyYmpqW2/7ly5fSsZmZWZntPv/8c8yZM0d6np6ejqZNmyI2Nhbm5uYqVk1EREREFcnIyICdnV25mU2BQV4D1a1bVwrySUlJFQb55OTkYn3LYmhoCENDwxKvm5ubM8gTERERqVFlljXzYlcN1Lp1a+k4Ojq6wvZF2xTtS0RERESai0FeA7Vr1046vnz5crltExISEBsbCwCwtrZGgwYNqrU2IiIiIlIPBnkN1L9/f+m4oru1Hjt2TDqu6C6wRERERKQ5GOQ1kLu7O2xsbAAAISEhuHbtWqnt8vPzsXLlSum5j4+PWuojIiIiourHIF/L+Pv7QxAECIIADw+PUtvo6upi/vz50vMJEybg2bNnJdrNmzcPN27cAAB069YNnp6e1VEyEREREdUA7lojk+joaGzatKnYaxEREdLx9evX8dVXXxV7v3fv3ujdu7dS8/n6+uLAgQMICgrCrVu34OTkBF9fXzg6OiIlJQW7du3C+fPnARTuVLNu3Tql5iEiIiKi2olBXiZ//fUXvv322zLfj4iIKBbsAUBPT0/pIK+np4d9+/ZhzJgxOHLkCOLj4/HNN9+UaGdra4vdu3ejbdu2Ss1DRERERLUTl9ZoMDMzMxw+fBi///47vL29YWdnB0NDQ1hZWcHFxQVLly5FVFQU3NzcarpUIiIiIpKZIIqiWNNFUO2UkZEBCwsLpKen84ZQRERERGpQlfzFM/JERERERBqIQZ6IiIiISAMxyBMRERERaSAGeSIiIiIiDcQgT0RERESkgRjkiYiIiIg0EG8IRUREBCAzMxOBgYEIDg7GtWvXcP/+faSlpcHY2BiNGzdGly5dMGbMGHh6ekIQBFnmdHBwwF9//SU9d3FxQXh4eKX63r9/H6+99lqx1xYsWICFCxfKUhvVrIKCApw4cQJ79uzBlStX8OTJE2RmZsLIyAhWVlZo3rw5nJyc4Orqij59+sDKyqrUcRT/HhwcHDBp0iS11R8TEwN/f38AgIeHBzw8PNQ2978JgzwREf3rrVixAl9++SWys7NLvJeZmYm7d+/i7t272L59O3r06IEdO3agadOmstdx8eJF3L59G46OjhW23bx5s+zzU+1w//59jBs3DpcuXSrx3vPnz/H8+XP89ddfCA4OBgDo6OggIyMDJiYmJdovWrQIAODu7q72IK+YGwCDfDVhkCcion+9e/fuSSG+SZMm6Nu3Lzp27Ahra2tkZ2cjPDwcO3bsQFZWFs6dOwcPDw+Eh4fD2tpathr09PSQl5eHzZs3Y9myZeW2zc/Px7Zt24r1I+3w999/o2fPnoiPjwcA1KtXD97e3ujYsSOsrKzw6tUrxMXF4fr16zh9+jQSEhJQUFAA3t/z34lBnoiI/vUEQUC/fv0wd+5c9OnTBzo6xS8hmzhxIubNmwdPT0/cvXsX0dHRmDdvnqxnxb28vHDo0CHs2LEDP/zwA/T0yv5/0SdOnMDTp08BAAMHDsTBgwdlq4Nq1kcffSSF+H79+mH37t2oW7duqW1FUURYWBjWrVtX4t8s/Tvwt05ERP963377LU6ePIm33nqrzEBkb2+P3bt3S893796NFy9eyFbDlClTAAAJCQk4evRouW0VXyAaNWqEAQMGyFYD1az09HQcPnwYAGBsbIyAgIAyQzxQ+AW0e/fu2L59O+rUqaOmKqk2YZAnIqJ/vfr161eqnZOTE1q3bg0AePHiBR48eCBbDf3790fjxo0BlL/+PSkpSQp7EyZMgK6ubpXmuXHjBj744AM4OTmhfv36MDQ0ROPGjTFw4EBs3ry5wmU6giBAEARpzXNGRga+//57dOzYEfXq1UPdunXRuXNnrF+/Hrm5ucX6RkdH4+OPP0bbtm1hamqKunXrolevXti/f3+l67969SqmT5+ONm3awMLCAsbGxrC3t8eoUaMqNY6DgwMEQYCDgwMAICcnB2vWrIGHhwcaNWoEXV1dODg4IC8vD40bN4YgCKhbt26lvrRlZWXB3NwcgiDA1tYW+fn5lf5cAPDw4UPpZ9auXTvUq1evSv2LUvyeFM6ePSu9VvShuCBVIS8vD0FBQfj000/h7u6ORo0awcDAACYmJnBwcMDIkSOxd+9eFBQUlDpvSEgIBEFAr169pNcWLVpU6twxMTGljpGeno7ly5ejb9++aNy4MQwNDVG/fn107NgRn3/+OZ48eVLh54+Pj8eiRYvQrVs3WFlZQV9fHxYWFmjRogW6du2KGTNm4NixY2V+Do0hEpUhPT1dBCCmp6fXdClERLVG586dRQAiADE8PFylsezt7aWxcnNzxXnz5okARD09PTE+Pr7UPitWrJD6/Pnnn+KGDRuk5wsWLChzruzsbHHKlCmiIAhS+9Iebdu2FR8+fFjmOIp27u7u4p07d8QWLVqUOdbAgQPFnJwcURRF8dChQ6KpqWmZbefNm1fuzyovL0+cMWNGhfX36NFDfPbsWYU/c3t7ezE6Olp84403Soxhb28viqIofv3119JrmzdvLrc+URTFdevWSe3nz59fYft/unjxYokalFXez6joY8uWLcX69erVq1L93NzcSv03GhwcXOm5o6OjS/Tfs2ePWL9+/XL7GRkZif7+/mV+9mPHjolmZmaVqiExMVGln3N1qEr+EkSRV0dQ6TIyMmBhYYH09HSYm5vXdDlERDXu1atXsLa2Rnp6OgAgLi4ONjY2So9XdPvJ3NxcREdHS1tK/vjjj5g7d26JPm+++SYiIyPh5uaGsLAwbNy4Eb6+vgDK3n4yLy8Pb731FkJCQgAAjRs3ho+PD958803UqVMHjx8/xv79+3H+/HkAhRf8Xr9+HQ0aNCgxluIsb/v27ZGVlYWHDx9i+PDh6NevH8zNzXH79m2sXr0aKSkpUk1eXl7o1q0bjIyMMGnSJHTp0gW6uroICQnB5s2bpTPXp0+fRu/evUv9WY0fPx47duwAAOjr62PcuHHo2bMnDAwMEBERgc2bNyMxMREA8Prrr+Py5cswNTUt82duY2ODRo0a4fr163B1dcWIESNga2uLlJQU3Lp1C6tXr0ZsbCyaNWuG/Px8uLq64sKFC6XWptCxY0dcu3YNurq6iI6Ohp2dXbnt/yk5ORkNGjSQLlzdv38/hg0bVqUxFH7//XcAkPq3bdsWS5YsKdHO2dm52A5Mrq6uiIqKgoeHBzp27IhmzZrBzMwMz58/x507d/Dbb7/h4cOHAICuXbsiNDS02PUcSUlJOH/+PKKiovD1118DAN555x34+PiUmLtfv37FlgRt2LABfn5+EEURBgYGGDJkCHr27ImGDRsiKysL58+fx86dO6UL03fu3InRo0cXG/Pp06do3bo1srKyABTu1jNw4EDY2NjA0NAQSUlJiIqKwunTp3Hv3j0kJiaWuXVnTalS/qrmLxWkwXhGnoiouC1btkhn8pydnVUe759n5EVRFLt37y4CEB0dHUu0v3TpktR+48aNoiiKlTojrzjTD0D09fUVX758WWq7//3vf1K7sWPHltoGRc5mGhoaisePHy/R5s6dO6KxsbEIQLSwsBBbtmwpNm/evNQzsJs3b5bG8/LyKnXOPXv2SG3q168vXr16tUSbxMREsUOHDlK76dOnlzpW0Z85APGHH34otZ3CkCFDpLYRERFltrty5YrUbtCgQeWOWR5PT09pHH19fdHPz08MDg4WX7x4odR4irHc3d0r1T4oKEh8/vx5me/n5uaKM2fOlMbdvn17qe2Knpkv7y9FCjdv3hQNDAxEAGKrVq3EO3fulNru9u3bYuPGjUUAopmZmZicnFzs/R9//FGad+XKleXOGR4eXub/LdSkquQvBnkqE4M8EdH/efbsmWhtbS2FhP3796s8ZmlBvuiXhQsXLhRrP23aNBGAaGJiImZmZoqiWHGQT0hIEI2MjEQAYt++fSusacyYMSIAUVdXV3z8+HGJ94uG4G+//bbMcd57771ibf/5WYpq2bKl9MVA8XMoytnZWRpnz549ZY4THR0tfYEwNDQUExISSrQp+jMfMmRImWMpnDx5Umo/a9asMtv5+vpK7Q4fPlzhuGW5f/++2KBBgxJLQHR1dcV27dqJkyZNEtevXy8+ePCgUuNVNchXRm5urujg4FDuv6mqBvlhw4ZJy2bu379fbtvAwMAyv4j5+flJ75X3haQ2q0r+4sWuREREFXj16hWGDx+OZ8+eAQCGDh2q9JKHiowcORJmZmYAgC1btkivZ2dnIyAgQGpT2rKR0uzevVtaivDJJ59U2H7ixIkACveqP336dJntdHV1MWPGjDLf7969u3TcqVMnuLq6Vtg2JydHWrah8Ndff+HatWsAgObNm2PEiBFljuPg4CAttcjJyalw95/Zs2eX+z4AvPXWW2jZsiUAYMeOHXj58mWJNpmZmdi1axcAwM7OTqWdhFq2bImrV69i2LBhxS5Wzc/PR2RkJPz9/TF16lS0bNkSLi4uOHLkiNJzKUtPT0/6fV66dEnlPezT0tKkLVSHDRsm/bzL8tZbb6FRo0YAgJMnTxZ7r+hNsa5evapSXZqA+8gTERGVo6CgAFOmTMG5c+cAAC1atKjWu6qamJhg1KhR2LRpEwICAvDTTz/B2NgY+/btQ1paGgBg8uTJlR4vNDRUOk5ISJDWTpel6I4gt2/fLrNd69aty90asei1A126dCl3zqJtU1NTi7138eJF6fitt94qFm5L4+npKf1+wsPDy/xZ6erqws3NrdyxgMJrAvz8/PDJJ58gLS0Nu3fvLnGH1J07d0prst97770q7yT0T3Z2dti/fz+io6Oxd+9enD17FpcuXZKuAVC4dOkSBg8ejKlTp+KXX36p8GdTWS9evMDu3btx+PBhREZGIiEhAVlZWaUG9oyMDGlNt7LCwsKk3WMMDQ0r/DcKAGZmZoiLiyvxb7Rfv35YsWIFAMDb2xufffYZRo4cCXt7e6Xrq80Y5ImIiMogiiKmTZuGX3/9FQDQtGlTnDp1SqVtAStjypQp2LRpEzIyMrB3716MHz9eCqctW7ZEz549Kz1W0S3+JkyYUKU6FBeslsbS0rLcvoaGhkq1Vfz1QCEuLk46VlwIXJ6ibYr2/SdLS0sYGRlVOB5Q+Pv4+uuvkZ2djfXr15cI8uvXrwdQ+OXg3XffrdSYldGsWTN88skn0l9SYmNjcfHiRQQGBmLXrl3Sl4f169fD3t4eX3zxhcpz/vHHH/Dx8UFsbGyl+6ga5Iv+G/X39y+xJWZ5/vlv1NPTExMmTMC2bduQlJQk/fyaNWuGrl27omfPnvDy8qryhci1FZfWEBERlUIURcyYMQMbNmwAANja2uLMmTPS/uPVyc3NDa+//jqAwuU1MTExCA4OBlC1s/EApLP4ynj16lWZ71XlTqKq3HU0MzNTOi66bKIsRZccFe37T8bGxpWuoX79+hg1ahQA4MKFC4iKipLeu3LlirT0Z9CgQWjSpEmlx60qOzs7jBgxAuvXr8ejR4/QtWtX6b0ffvihxJegqoqOjoanp6cU4lu2bIlZs2Zh5cqV2LlzJ/bv348DBw7gwIEDxfaJr+p++f+kyr/Rf96rAPi/LwNvvvmm9Fp0dDR27tyJadOmwd7eHgMHDsTdu3eVnre2YJAnIiL6B1EUMXPmTPzyyy8ACrdjDA4ORosWLdRWgyKwh4SEYP78+RBFEbq6utIa9soqGmwzMjIgFm50UalHVc6MVhfF9QIA8Pz58wrbK85S/7OvqqZPny4dK87A//PYz89Ptvkq0qBBA+zevVva+jEzMxPh4eEqjfndd99JP7/PPvsM9+7dw6pVq/D+++9j9OjRGDZsGIYOHYqhQ4dW+FeWqij6b3TlypVV+jda2nIfQRAwceJE3Lx5EzExMdixYwdmzpyJtm3bAij8v+9jx46hc+fOiIyMlO1z1AQGeSIioiIUIf7nn38GULjnenBwcIUX4MltwoQJ0NPTgyiK2L59O4DC9b9VPeNra2srHVdluURtobioEQDu379fYft79+5Jx4o75crB1dUVHTp0AABs374dL1++LHaRq729PTw9PWWbrzLs7OwqvZSoMgIDAwEA1tbW+Pbbb8tdcx8dHa3SXEVV579Re3t7jB07FqtXr0ZUVBRu374Nd3d3AIVffuRYjlSTGOSJiIj+v3+G+EaNGiE4OBitWrVSey02NjYldj+p6rIaAFJoAYDjx4+rXJe6ubi4SMdBQUEVti+6i0nRvnJQnJVPS0vDnj17il3k6uvrq9ISImXp6+tLx6XtZKQI45XZWSY+Ph5A4dr88i7YjYuLw82bN8sdq+jPoqK5e/ToIdV54sSJCutURZs2bbBv3z6pPsVF7JqKQZ6IiOj/mzVrlhTibWxsEBwcXKkLLKuzHhcXF7i4uMDDwwNDhgyp8hg+Pj7SxaQrVqxAUlKS3GVWK3t7e3Ts2BEA8PDhQ+zdu7fMtn/99Ze0RaehoSEGDhwoay1jxoyRLupcv369tKxGT08PU6ZMUXn8Fy9elLuu/58ePXpUbGnIG2+8UaKNItxXZlmS4hqEhw8flhu+Fy9ejLy8vHLHKvqloqK5ra2tpS+tkZGR0l85qoulpaV0x9SKPkdtxyBPREQE4P3338fatWsBFIb4kJAQtG7dukZr6tevH8LDwxEeHo7g4GAYGBhUeQxbW1tpv/SnT5/C09MTjx49KrfPzZs31breuyLz5s2Tjv38/HD9+vUSbZKTkzFixAi8ePECAPDuu+/C2tpa1jpMTEyknX/++OMP6SLXt99+u9gSIGX9/fffsLe3x9dff11iP/1/iomJgbe3t7RtY/fu3dGsWbMS7RSv/fnnn6XugV9U586dAQBJSUlYvnx5qW2WL18uXTtSnqK1KH5O5fn222+lf9/vvfdehWE+JSUFK1aswKlTp4q9vmjRIpw8eVL6uZRm165d0gW27du3r7C22ozbTxIR0b/eV199hdWrVwMoXIrwwQcf4M6dO7hz5065/ZydndG0aVN1lKiS7777Djdv3kRgYCCuXbuG119/HW+//TZ69OiBRo0aoaCgAElJSYiKikJwcDDu3bsHXV1drFu3rqZLBwCMGDEC48aNw44dO5CSkgJXV1eMGzcOPXv2hIGBASIjI7Fp0ybphl2vv/46/vOf/1RLLdOmTcOqVauKvSbnl57U1FQsWbIES5YsQYcOHeDm5gZHR0fp4tKnT5/iwoULOHToEHJycgAAFhYW0l+S/qlv376IiIjA8+fPMXjwYEyYMAENGjSQlrK0a9dOuu7iww8/lNbJf/LJJwgODkb//v3RsGFD/P3339izZw8uX76MRo0aoV27dlLb0tSrVw/Ozs64du0agoOD4efnh759+xa7ANnd3V3aPah9+/ZYt24d3n33Xbx48QJjxozBf/7zHwwePBitWrWCsbEx0tPT8eDBA1y6dAmhoaHIy8uTrh9RCA4OxsKFC2FtbQ1PT0+0b98eNjY20NHRQVxcHE6ePFlsiZamr5GHineRJS1WlVsEExFpMnd3d+m27lV5bNmyRaV57e3tpbFyc3OVGmPDhg3SGAsWLCiz3atXr8SPP/5Y1NPTq9Rns7e3L3Ucxfvu7u7l1hUcHFypukRRFBcsWCC1DQ4OLrVNbm6uOH36dFEQhHLr7t69u/js2bMy51L8zMv6fJVR9N9L8+bNxYKCAqXHKurx48diixYtqvRvsEOHDuLNmzfLHPPJkydiw4YNK/1veOHChRX+u7h69ao4ceJE6bXo6OhS5z5x4kS5/95K63fixAmxcePGlfrshoaG4vHjx4v19/DwqFRfExMTcfPmzVX9FalFVfIXz8gTERH9C+jr62PZsmV4//33sXnzZgQHB+P+/ftISUmBjo4OLC0t8dprr8HFxQWenp5VuumUOujp6WHt2rV49913sWHDBoSEhODJkyfIzc2FtbU1XFxcMHr0aHh7e1d7Lf369cPZs2cBFF7kKtcdVZs0aYIHDx4gMjISZ86cwYULF/Dnn38iNjYWGRkZ0NXVhbm5OZo1awZnZ2cMGzYMffv2Lfci28aNG+PatWtYvnw5Tp06hejo6DLv0goACxYsQM+ePbFy5UpcuHABKSkpqFu3Lpo3b45hw4bBz8+v3Dv6FuXp6YkLFy5g5cqV+OOPPxAXFyctfSqvz6NHj7Bz504cO3YMV69eRWJiIrKzs2FmZgYHBwc4OTmhd+/eGDx4cIlaDh8+jFOnTuHs2bO4du0aHjx4gKSkJIiiiLp16+L1119H37598d5778m6q1FNEcSyfpP0r6e4U1t6erp0UQgREdG/Xfv27XHz5k3o6+sjNjYWDRs2rOmSSItUJX/xYlciIiKiSrpw4YK09aK3tzdDPNUoBnkiIiKiShBFEV9//bX0/MMPP6y5YojAXWuIiIiIyhQZGYknT54gNTUVv/76K06fPg0AGDhwIFxdXWu4Ovq3Y5AnIiIiKsPy5cuxdevWYq9ZW1tL9xwgqklcWkNERERUAV1dXTg4OGDKlCm4fPmyRtw/gLQfd62hMnHXGiLSaAstaroCItIWC9PVNhV3rSEiIiIi0nIM8kREREREGohBnoiIiIhIAzHIExERERFpIAZ5IiIiIiINxCBPRERERKSBGOSJiIiIiDQQgzwRERERkQZikCciIiIi0kAM8kREREREGohBnoiIiIhIAzHIExERERFpIAZ5IiIiIiINxCBPRERERKSBGOSJiFQQEhICQRAgCAIWLlxY0+UQEdG/CIM8UTVQBDvFw8fHp9J9g4KCSvT39/evvmKJiIhIIzHIE6nB77//jtTU1Eq13bx5czVXQ0RERNqAQZ6oGunp6QEAcnJy8Ouvv1bYPjU1Fb///nuxvkRERESlYZAnqkYNGzaEs7MzAGDLli0Vtt+5cyeys7MBAAMHDqzW2oiIiEizMcgTVbMpU6YAAK5du4aIiIhy2yqW1XTu3BlvvPFGtddGREREmotBnqiajRkzBkZGRgCATZs2ldkuIiIC165dA/B/4b8qQkNDMXXqVLRp0wZ169aFkZER7OzsMHz4cOzbtw+iKJbZNyYmRrqwdtKkSQCA+Ph4fPnll3jjjTdgbm4OKysr9OjRA3v27CkxVlRUFHx9fdG6dWvUqVMHlpaWGDhwIEJCQipd/5kzZzBp0iS0bNkSpqamMDExQcuWLTFx4kScPn26wv6K+j08PAAAaWlpWLp0KVxdXWFtbQ0dHR14eHggOTkZRkZGEAQBr732WqVqi46Oho6ODgRBgKura6U/U2lU+T0pHD16FKNHj0bLli1hYmICQ0NDNGrUCO3atcOQIUOwbNkyPH78uNS+BQUF2LlzJ4YOHQp7e3sYGxvDyMgITZo0gZOTE0aOHIm1a9ciOTlZpc9JRETVj4twiapZvXr1MHToUAQEBODXX3/Fjz/+CAMDgxLtFCHfyMgIPj4+WLZsWaXGT0tLw/jx43HkyJES7z1+/BiPHz/G/v370bNnT+zbtw9WVlYVjhkWFgZvb288e/as2Ovnz5/H+fPncfr0afzyyy8QBAHr16/HzJkzkZeXJ7V7+fIljh07hmPHjuHnn3/GtGnTypzrxYsXGD9+PPbv31/ivYcPH+Lhw4fYtm0bvL29sX37dtSpU6fC+q9fv46hQ4fi77//LvGepaUlRo0ahe3bt+P+/fsIDg5Gr169yh1vw4YNUsD28/OrcP7SyPF7evnyJd555x0cPny4xHvx8fGIj49HVFQUDh06hJiYGKxevbpYm+TkZAwaNAjh4eEl+j99+hRPnz5FREQE9u7dixcvXmDu3LlKfVYiIlIPBnkiNZgyZQoCAgKQnJyMQ4cOYcSIEcXef/XqlXQxrLe3N+rWrVupcTMyMtCtWzfcvn0bANCqVSuMHDkSbdq0gYGBAR49eoRdu3YhIiICoaGh6Nu3L8LDw6W/EJTm77//xtChQ5Geno5JkybB3d0dRkZGuHz5Mn7++We8fPkS69evR9euXWFubg4/Pz9YWVlhypQpcHJyQl5eHo4ePYo9e/YAAGbPng0PDw+8/vrrJebKz8+Hl5cXzp49CwAwNTXFpEmT0LlzZ+jo6ODSpUvYsmULsrKysH//fqSkpODUqVPQ1dUts/7k5GQMGTIEsbGxeOuttzB48GA0bNgQ8fHx0heT6dOnY/v27QCA9evXlxvkc3NzpesbLCws8M4775T3KymVXL+nL7/8UgrxDRo0wDvvvIO2bdvC0tIS2dnZiI6OxqVLlxAcHFxqHb6+vlKIt7Ozg4+PD1q1aoV69erh+fPnuH//Pi5cuIBz585V+TMSEZH6McgTqUGfPn3QtGlT/P3339i8eXOJIH/w4EFpKUNVltX4+flJ4XDhwoX46quvSoTcTz/9FJ9++imWL1+OmzdvYsmSJViyZEmZYwYHB6N+/fq4cOECOnbsKL3u4+ODwYMHo3fv3hBFEYsWLUJmZiY6d+6MEydOoH79+lLbCRMm4PXXX8fixYuRm5uLVatWYc2aNSXmWrFihRTiHRwccObMGTRr1kx6f9y4cfjoo4/Qq1cv/PXXXwgJCcHy5cvx6aeflll/VFQUdHV1sWPHDowdO7bUNl27doWTkxNu3ryJAwcOICkpqcy/VBw6dAjx8fEAgPHjx1fqLwL/JMfvKT8/X7qGokWLFrh8+TLq1atX6nwZGRl4+PBhsdeePXuGgwcPAgDc3Nxw+vTpMr/QJSYmIikpqcqfk4iI1Itr5InUQEdHB5MnTwYABAYG4smTJ8XeVwQ0BwcH9O7du1JjRkREICAgAADw7rvvYsGCBaWeqdbR0cGyZcvQrVs3AMDq1auRk5NT7tirVq0qFuIVPDw80KdPHwCF6+qzsrLw22+/FQvxCvPmzYOpqSkA4MSJEyXez83NxYoVKwAUrm8PCAgoFuIVmjVrhoCAAAiCAKAw/L969arc+mfNmlVmiFeYMWMGgMKtQbdu3Vpmu3Xr1knHyiyrkev3lJiYiPT0dACFf7UpK8QDgLm5OTp06FDstUePHqGgoAAAMHbs2HL/KtOgQQO0adOmkp+QiIhqCoM8kZpMnjwZgiAgPz8f27Ztk15/8uQJAgMDAQCTJk2SAmtFiobP8s5QK0yYMAEAkJ6ejosXL5bZztrautzlI927d5eOBw8eDHt7+1LbGRsbo1OnTgAKLxZVbKup8Mcff0hnuj08PODi4lLmnK6urtLyl4SEBISFhZXZFihczlORsWPHwtzcHEDh8prSPHr0CKdOnQJQeBZbmZ2E5Po9Ff1LgOKi6KowMTGRjq9evVrl/kREVPtwaQ2Rmtjb26N37944ffo0tmzZgs8//xwA4O/vj4KCgmI7xlRGaGgogMKLY2/fvi0t3ShL0b8C3L59Gz179iy1XadOncpdg25jYyMdd+nSpdw5FW1FUURaWlqxvkVDar9+/codBwA8PT1x5swZAEB4eHiZ69obN26M5s2bVzieiYkJxo8fjzVr1uDevXsICQmRdrxRkOMiV7l+T+bm5nB1dUV4eDhOnz6Nt99+G7NmzYKHh0epF0//k6OjI5o0aYInT55g8+bNyM/Ph6+vL1xdXcv9fRMRUe3FIE+kRlOmTMHp06dx//59nDt3Dj169IC/vz8AoHfv3mWe3S5NTEwMACA7OxvDhg2rUh0pKSllvmdpaVluX0NDQ6Xa/vOMfFxcnHRcmW0gi7Yp2vefbG1tKxxLYfr06dLa/fXr1xcL8kUvcq1Xrx5GjRpV6XGLkvP3tGbNGvTu3Rvp6ek4fPgwDh8+DGNjY3Tu3Blubm7o3bs3evXqVepdgXV1dbF+/Xp4e3tLy4m2bt0Kc3NzuLi4oFu3bujbty/c3Nwq/VchIiKqWVxaQ6RGRXek2bJlC86ePYsHDx4AqPre8WlpaUrXUd4acx2dyv/PQlXa/lNmZqZ0XHTZR1kU6+3/2fefjI2NK11D27ZtpTPe+/fvL7Z3+sGDB5GQkACgcLlLeWvKyyPn78nZ2Rk3b97E5MmTpZ/Zy5cvERoaih9++AH9+vWDra0tfvrpJ2k9fFFeXl64cuUKRowYIZ3Fz8jIQFBQEBYuXIju3bujRYsW2LFjh9I1ExGR+jDIE6mRkZERRo8eDQD47bffsHLlSgBA3bp14e3tXaWxFMG2fv36EEWxSo+FCxfK+rmUYWZmJh0/f/68wvZZWVml9lXV9OnTAZS86LXouvmpU6cqPb7cvyd7e3ts3rwZycnJOHv2LH744QcMGjRImichIQEfffQR3n333VLreeONN/Dbb78hJSUFJ0+exKJFi9C3b1/pryfR0dEYP348Fi1apPRnJiIi9WCQJ1IzxZl3xd7oADB69Ogqn/FVLCFJS0srFnI1RaNGjaTj+/fvV9j+3r170nHjxo1lq2P48OFo2LAhgP8L70Uvcu3RowccHR2VHr+6fk+Ghobo2bMnPvvsMxw+fBiJiYlYt24d9PX1ARRee1HeRa0mJibo168f5s+fj6CgICQmJuKbb76R3v/222+li5GJiKh2YpAnUrNOnTrhzTffLPaaYmvKqnB3dwcAFBQU4OTJk7LUpk5Fd6lR7NpTnqKfsbwdbqpKX19fOnt99+5dnD17VpaLXBXU9XsyMjLC1KlTpW01AVTpxk5mZmb46quvMGTIEACF1wiUdgdYIiKqPRjkiWrAnDlz4OLiAhcXF3h7e6Nz585VHkOxTSEALF68uMTFpLWdm5ubdFY+ODgYly5dKrNt0buV2tjYSHuty2Xq1KnSev81a9ZIF7laWlqWuHlXVan791R0L/68vDy19yciIvVhkCeqARMnTkR4eDjCw8Oxb98+pcbo0qULRo4cCaDwpkNDhgxBYmJime1FUURYWBjmzp2r1Hxy09fXx5w5cwAU1ubj4yPt8FJUTEwMfHx8pDPkc+bMqdR2i1Vhb2+PgQMHAii8dkFxkevEiROL7byjDLl+T9evX8eiRYvK3bHn+fPnxe5R0L59e+n45MmT+O9//4vU1NQy+z979qzYv0cnJ6cy2xIRUc3j9pNEGmzTpk24d+8ebt68icDAQDg4OGD48OFwdXVFgwYNkJubi4SEBERERODUqVN4/PgxWrRogWXLltV06QCAjz76CEeOHMHZs2cRHR2Ndu3aYfLkyejSpQsEQcClS5ewZcsWaZcaDw8PKfzLbfr06Th8+HCx11S5yLUoOX5P6enpWLhwIRYvXgw3Nze4ubmhdevWMDc3R1paGv7880/s2rULT58+BVB4E62idwmOi4vDnDlz8Nlnn8HDwwOurq5o3rw5TE1NkZycjIiICOzatUsK+qNGjUKrVq1k+fxERFQ9GOSJNJiZmRnOnz+PGTNmYMeOHXjx4gW2b9+O7du3l9mnKvusVzddXV0cPXoU48ePx4EDB5CVlYVVq1aV2nbYsGHYsWNHtd28yNPTE82bN8ejR48AFH5paN26tSxjy/F7UuztXlBQgPPnz+P8+fNl9u3Zsyf27t1bbHtQRf/c3FwEBQUhKCiozP4jRoyQlhcREVHtxSBPpOFMTU2xbds2zJs3D/7+/tLZ7dTUVBgYGKBBgwZo3bo13NzcMGDAgArvxqpuJiYm2L9/P86cOYOtW7fi/Pnz0m4pDRs2RPfu3TFx4kT06dOnWuvQ0dFBnz59pCCv6kWu/6Tq78nd3R2RkZEICgrChQsXcOvWLTx+/BjPnz+HkZERmjRpgk6dOsHHxweDBw8uMf+ECRPg6OiIU6dO4eLFi7hz5w6ePn2Kly9fok6dOmjatClcXV0xfvx46QJdIiKq3QRRsfCU6B8yMjJgYWGB9PR0mJub13Q5RNXq1atXsLW1RWJiIho0aIDHjx/Lvhaf1GyhRU1XQETaYmG62qaqSv7ixa5ERAD27dsnXYQ6ZcoUhngiIqr1GORldujQIYwcORIODg4wMjKCtbU13Nzc8OOPPyIjI0P2+WJiYvD111+je/fusLKygr6+PkxNTdG8eXN4e3tjx44dyM3NlX1eIm2Sk5Mj3QxJX1+/2F7sREREtRWX1sgkKysLY8eOxaFDh8psY2dnhz179sDV1VWWOVesWIEvvvgCOTk55bZr3bo19u7dizfeeKNK43NpDWmzS5cuISUlBc+ePcO6devwxx9/AABmzpyJ1atX13B1JAsurSEiudTSpTUM8jLIz8/HoEGDcOLECQCFF+j5+vrC0dERKSkp2LVrF8LCwgAA9erVQ1hYGNq0aaPSnKtXr8b7778vPXdzc8Pbb78NOzs7ZGRk4NatW/D395duCW9lZYXIyEjY2NhUeg4GedJmHh4eOHv2bLHXXnvtNVy+fJn/3rUFgzwRyYVBXnutW7cO06ZNAwA4OjrizJkzaNiwYbE2c+fOxfLlywEAPXr0QGhoqNLzvXz5Eg0bNpT21t6wYQPee++9Eu0SExPRp08fREZGAijcs3vFihWVnodBnrSZIsjr6+vD3t4egwcPxpdffglLS8uaLo3kwiBPRHJhkNdO+fn5sLOzk+62ePXqVTg7O5farlOnTrhx4waAwrss9uvXT6k5T506hbfeegsA0Llz53JvbX/06FEMGjQIANCxY0dcuXKl0vPURJB3mHdULfMQkfaLMRpT0yUQkbaopUGeF7uqKDQ0VArx7u7upYZ4oPDGN7Nnz5ae79q1S+k5nz17Jh1XdOfFou8rltkQERERkeZjkFfR8ePHpWMvL69y2w4YMKDUflVlbW0tHd+7d6/ctkXfb9u2rdJzEhEREVHtwiCvIsX6c6BwmUt5bGxsYGdnBwBISEiQ9qyuKsVWkwBw5coVbNy4sdR2iYmJ+OKLLwAU3rVyzpw5Ss1HRERERLWPXk0XoOnu3r0rHTdr1qzC9s2aNUNsbKzUt0GDBlWe08jICL/88gt8fHyQl5cHX19f+Pv7F9u1JioqClu3bkVmZiZMTU2xceNGdOvWrcpzEREREVHtxCCvorS0NOlYcZa8PEV3xCjat6qGDx+OU6dOYebMmbh16xbCwsKkLS4V9PX18eWXX8LPz0/6S0B5cnJyiu1JXx03sCIiIiIieXBpjYqKXkBqZGRUYXtjY2PpWLF9pLJ69uyJ1atXo0OHDqW+n5ubizVr1mDFihV4+fJlheN9//33sLCwkB6VCf9EREREVDMY5DVUUlIS+vTpg169eiEmJgb//e9/8fDhQ7x69QppaWk4ffo0vLy8kJaWhp9++gkeHh5ITk4ud8zPP/8c6enp0kOxBIiIiIiIah8GeRWZmppKx9nZ2RW2L3pm3MzMTKk5X7x4gR49eiA4OBj16tXDxYsX8eGHH6J58+bQ19eHhYUFevfujaNHj2LmzJkACm9HX/ROsKUxNDSEubl5sQcRERER1U4M8iqqW7eudJyUlFRh+6JnxYv2rYq1a9fizz//BFB4x9jy9pJfunSpNM/u3bsRHx+v1JxEREREVLswyKuodevW0nF0dHSF7Yu2Kdq3Ko4cOSIdV3R3WBMTE7i5uQEACgoKcPnyZaXmJCIiIqLahUFeRe3atZOOKwrJCQkJ0rpza2trpbaeBICnT59KxxYWFhW2L3rmn3d3JSIiItIODPIq6t+/v3Rc0d1ajx07Jh1XdBfY8hRdW1+ZC1L/+usv6bjo9pdEREREpLkY5FXk7u4OGxsbAEBISAiuXbtWarv8/HysXLlSeu7j46P0nEX/CvDrr7+W2/bBgwe4ePEigMK7u3bq1EnpeYmIiIio9mCQV5Guri7mz58vPZ8wYQKePXtWot28efNw48YNAEC3bt3g6elZ6nj+/v4QBAGCIMDDw6PUNmPGjJGOt2zZgk2bNpXaLj4+HqNGjUJeXh4AYNCgQahfv35lPhYRERER1XK8s6sMfH19ceDAAQQFBeHWrVtwcnKCr68vHB0dkZKSgl27duH8+fMACterr1u3TqX5+vXrhxEjRmDv3r0QRRHvvfcetm/fjiFDhsDW1hYvX77ElStXsH37dunusZaWlli+fLmqH5WIiIiIagkGeRno6elh3759GDNmDI4cOYL4+Hh88803JdrZ2tpi9+7daNu2rcpz7tixA+bm5ti8eTMA4OzZszh79mypbVu3bo2AgAC0bNlS5XmJiIiIqHbg0hqZmJmZ4fDhw/j999/h7e0NOzs7GBoawsrKCi4uLli6dCmioqKkrSBVZWhoiE2bNuH69ev44IMP0KlTJ9SvXx96enqoU6cOHBwcMHz4cGzfvh0RERFo3769LPMSERERUe0giKIo1nQRciooKMC+fftw8uRJ3L59GykpKcjNzcXDhw+LtYuKikJGRgYsLCxkOUOujRQ/n/T0dLXd5dVh3lG1zENE2i/GaEzFjYiIKmNhutqmqkr+0qqlNWFhYZgwYQJiYmKk10RRhCAIJdru27cPixcvhrm5OeLi4mBkZKTGSomIiIiIVKM1S2sCAwPRu3dvxMTEQBRF6OrqlnuzpKlTpwIo/NZTdH93IiIiIiJNoBVBPi0tDaNHj0Zubi5MTU2xfv16pKWlYcuWLWX2adSoEVxdXQEAp0+fVlepRERERESy0Iogv2bNGqSmpkJPTw8nTpzAe++9hzp16lTYz83NDaIolnkTJyIiIiKi2korgvyxY8cgCAKGDx+Orl27Vrpf69atAQCPHj2qrtKIiIiIiKqFVgT5e/fuAQD69OlTpX5169YFAKSnq+9KZCIiIiIiOWhFkM/IyAAA1K9fv0r9cnNzARTe0ImIiIiISJNoRZBXBPjk5OQq9VNsU2llZSV3SURERERE1UorgnzLli0BABcuXKhSvxMnTkAQBDg5OVVHWURERERE1UYrgny/fv0giiL27t2L+Pj4SvU5ffo0zp07BwDw9PSszvKIiIiIiGSnFUF+6tSpqFOnDp4/f44RI0ZUePHqhQsXMHr0aABAvXr1MHHiRHWUSUREREQkG624yrNhw4b47rvv8OGHH+LChQto3bo13nvvPeTn50ttjh07hr///hvHjx/H0aNHUVBQAEEQ8NNPP8HExKQGqyciIiIiqjqtCPIAMHv2bDx79gzff/+99F8AEAQBADB48GCprSiKAIBFixZh3Lhx6i+WiIiIiEhFWrG0RmHJkiU4evQoOnToAFEUy3y88cYbOHLkCL766quaLpmIiIiISClac0ZeoX///ujfvz+ioqIQGhqKmJgYpKWlwdTUFLa2tnB3d0fHjh1rukwiIiIiIpVoXZBXeOONN/DGG2/UdBlERERERNVCq5bWEBERERH9WzDIExERERFpIK0I8nFxcejYsSM6duyI48ePV6rPiRMn4OzsjM6dOyM5ObmaKyQiIiIikpdWBPmdO3fi+vXriI6ORp8+fSrVp3fv3vj7779x7do17Ny5s5orJCIiIiKSl1YE+TNnzkAQBAwcOBAGBgaV6mNgYIBBgwZBFEUEBQVVc4VERERERPLSiiAfGRkJAOjSpUuV+nXq1KlYfyIiIiIiTaEVQf7Zs2cAgEaNGlWpX8OGDQEACQkJstdERERERFSdtCLI6+kVboefk5NTpX6vXr0CAIiiKHtNRERERETVSSuCfIMGDQAAd+/erVK/P//8EwBgZWUle01ERERERNVJK4K8s7MzRFHEb7/9Vumz6wUFBfjtt98gCALefPPNaq6QiIiIiEheWhHkvby8AAD37t3Dd999V6k+3333He7duwcAGDx4cLXVRkRERERUHbQiyI8fPx6NGzcGAMyfPx+zZ88u8yZPycnJeP/997FgwQIIggAbGxtMnjxZneUSEREREalMr6YLkIOBgQG2b98OT09P5OfnY82aNdi4cSPc3Nzg6OgIU1NTZGVl4fbt2/jjjz+Qk5MDURShr6+Pbdu2wdDQsKY/AhERERFRlWhFkAeAXr16Yc+ePZg4cSIyMzORnZ2N4OBgBAcHF2unWENvbm6OrVu3VvpOsEREREREtYlWLK1RGDp0KCIjI+Hn5wdzc3OIoljiYWFhgRkzZiAyMhJDhgyp6ZKJiIiIiJSiNWfkFZo2bYqff/4Za9asQUREBB4/foyMjAyYm5vD1tYWb775JnR0tOr7CxERERH9C2ldkFfQ0dFB+/bt0b59+5ouhYiIiIhIdjw1TURERESkgRjkiYiIiIg0kFYurYmLi0NUVBRSU1ORnZ1dqT4TJkyo5qqIiIiIiOSjVUE+ICAAP/zwAyIjI6vUTxAEBnkiIiIi0ihaE+Rnz56NNWvWAPi/veKJiIiIiLSVVgT5Q4cOYfXq1dJzFxcXvPXWW7C1teVdW4mIiIhIK2lFkF+3bh0AQFdXF/7+/hg7dmwNV0REREREVL20YteaK1euQBAEjBs3jiGeiIiIiP4VtCLIp6enAwD69OlTw5UQEREREamHVgT5hg0bAgD09fVruBIiIiIiIvXQiiDv4uICALhz504NV0JEREREpB5aEeSnT58OURSxY8cO5Obm1nQ5RERERETVTiuCfK9evTBr1iw8evQIkyZNYpgnIiIiIq2nFdtP/v333/j444+RkpKCnTt34tq1a5gxYwa6du0KKysr6OhU/H2ladOmaqiUiIiIiEgeWhHkHRwcIAgCAEAQBNy7dw8ffvhhpfsLgoC8vLxqqo6IiIiISH5aEeQBQBTFmi6BiIiIiEhttCLIT5w4saZLICIiIiJSK60I8lu2bKnpEoiIiIiI1Eordq0hIiIiIvq3YZAnIiIiItJAVV5a07x58+qoA4Ig4OHDh9UyNhERERGRtqlykI+JiYEgCJXaJUaxJSRQuKvMP5+X1ZaIiIiIiMpX5SDftGnTckN3bm4u4uLiIIqiFNbr1q0LExMTPH/+HGlpaVJbQRDQqFEj6OvrV73yMsTGxmLHjh0IDw/H48ePkZGRgfz8/HL78K8BRERERKRplDojX5bHjx9j5MiRePr0Kbp06YKPP/4Yffr0Qf369aU2KSkpOHXqFH766SeEh4ejadOm+O2339CkSROlPoBCQUEBvvjiC6xYsUIK7mWd9edfA4iIiIhI08l2sWt2djYGDhyIS5cuYc6cOQgPD8fIkSOLhXgAqF+/PkaNGoU//vgDH3/8McLDwzFo0CBkZ2erNP/MmTPx448/Ii8vD6IoomHDhgAKQ3qDBg1gZWVVbEmQIAiwtbWFvb09mjZtqtLcRERERETqJluQX7t2LSIjI9GpUycsW7asUn1+/PFHdOrUCREREfjll1+Unvvy5ctYt24dAKBr16548OABnj59Kr2/YcMGPHv2DKmpqdi9ezfefPNNiKKI119/HdeuXUN0dLTScxMRERER1QTZgvyePXsgCALGjBlTpX5jx46FKIoICAhQeu4NGzYAAOrVq4cjR46UubOOmZkZRo4cicuXL2PEiBE4ffo0RowYofS8REREREQ1RbYgr7hYtHHjxlXqp2ivysWmYWFhEAQBo0aNQr169Spsr6+vj23btqFJkyYICQnBr7/+qvTcREREREQ1QbYg/+LFCwAotqSlMhTtFf2VoRijU6dOpb6fk5NT4jUjIyNMmjQJoihi586dSs9NRERERFQTZAvydnZ2AFDlUKxor+ivjOfPnwNAibPxderUAQCkp6eX2s/R0REAEBkZqfTcREREREQ1QbYgP2DAAIiiiCtXruCTTz6pVJ/PPvsMly9fhiAI8PLyUnpuMzMzAMDLly+Lva4I9mVdzKr4ApCYmKj03ERERERENUG2ID937lwpUK9YsQJdu3bFb7/9huTk5GLtkpOT8dtvv6Fbt27S7jZmZmaYO3eu0nO3aNECQMllPY6OjhBFEWfPni2136VLlwAAxsbGSs9NRERERFQTZAvyTZo0wc6dO6W7tF66dAk+Pj6wtrZGvXr10KRJE9SrVw/W1tbw8fFBeHg4RFGEgYEBdu7cWeWLZItydnaGKIq4efNmsdf79OkDALhw4QKOHTtW7L3w8HD4+/tDEAQ4OTkpPTcRERERUU2QLcgDwMCBAxESEoLWrVtDFEXpkZ6ejvj4eKSnpxd7vU2bNjh79qxKy2oAoFevXgCAM2fOFHt9/Pjx0jr5oUOHYtSoUfjiiy8watQoeHh4IDc3FwAwceJEleYnIiIiIlI3QVTc6lRGBQUFOHr0KPbv349Lly7h6dOnyMrKgqmpKZo0aYIuXbpg2LBhGDhwIHR0VP8ukZWVhQYNGiAnJwfHjx+Hp6en9N66deswffp0AIV3c1VQfOwBAwbg6NGjKtegjTIyMmBhYYH09HSYm5urZU6HefxdEJE8Yoyqdl8TIqIyLSx945TqUJX8pVcdBejo6GDw4MEYPHhwdQxfgqmpKTIyMlBQUCAt7VHw8/ND/fr18eWXX+LBgwfF+syYMQPffPONWmokIiIiIpJTtQT5mvDPAF/UyJEjMXLkSMTExCA+Ph4mJiZ4/fXXy+1DRERERFSbaU2QrwwHBwc4ODjUdBlERERERCqrtiB/+fJlnDx5Erdv30ZKSgpyc3Nx+vTpYm2SkpLw6tUrGBkZoX79+tVVChERERGR1pE9yD948ABTpkxBWFiY9JooisUuNFX4/vvv8dNPP6FBgwZ48uQJdHV15S6HiIiIiEgryRrkr127ht69eyMzMxOV2Qxn+vTp+O9//4vExEQEBgZiwIABstRRUFCAhw8fIjU1FdnZ2ZXq07NnT1nmJiIiIiJSB9mC/MuXLzF06FBkZGRAT08Pn376KSZOnIibN29i1KhRpfZp2bIl2rdvj5s3byIoKEjlIB8SEoLly5fj9OnTyMnJqXQ/QRCQl5en0twKhw4dwvbt23H58mXEx8fD3NwcLVu2xLBhw+Dn51dt2zhev34dO3fuxKlTp/D48WNkZGTAysoKjRo1gqurKzw8PDBs2DD+1YOIiIhIS8gW5Dds2IDHjx9DEATs3r0bw4YNAwDcvn273H49evTAjRs3cOXKFZXm//LLL/HDDz8AQKX+GiC3rKwsjB07FocOHSr2emJiIhITE3HhwgWsWrUKe/bsgaurq2zzZmRk4IMPPsDWrVtLfO6nT5/i6dOnuHr1KtasWYPU1FTUrVtXtrmJiIiIqObIFuQPHjwIQRAwYMAAKcRXRps2bQCg2B7vVXXgwAF8//330vOWLVuie/fusLGxgaGhodLjVlZ+fj5GjhyJEydOAAAaNmwIX19fODo6IiUlBbt27UJYWBhiY2Ph5eWFsLAw6XOrIiUlBZ6entKXoCZNmsDb2xtOTk6wsLBAZmYm7t+/j6CgIFy9elXl+YiIiIio9pAtyN+6dQsAMHDgwCr1U+xWk5aWpvTcq1atAgDo6elh/fr1mDRpktJjKWPjxo1SiHd0dMSZM2fQsGFD6f2ZM2di7ty5WL58OVJTU+Hn54fQ0FCV5x0zZowU4j/++GMsWbIERkZGJdp99913ePr0KUxNTVWek4iIiIhqBx25BkpNTQUAWFtbV6mfHMtgrl+/DkEQMGXKFLWH+Pz8fCxatEh6vn379mIhXmHp0qVo3749AODcuXMIDAxUaV5/f3+cPHkSQOFFw8uWLSs1xCs0btwYenr/qtsGEBEREWk12YK8hYUFgMI121Xx+PFjAIClpaXScysuVPXw8FB6DGWFhoYiLi4OAODu7g5nZ+dS2+nq6mL27NnS8127dqk079KlSwEApqam0rUBRERERPTvIVuQV9wxtaprsRU3iXJ0dFR6bnt7ewCFZ8fV7fjx49Kxl5dXuW2L7spTtF9VhYWF4c8//wQADBkypNp2wiEiIiKi2ku2IN+nTx+Ioojdu3dX+qz8jRs3cPLkSQiCgL59+yo9t6enJ4DCu8mqW2RkpHTcuXPnctva2NjAzs4OAJCQkIDExESl5jx79qx07OLiAgDYv38/vLy8pAt8GzdujIEDB2LLli2yba1JRERERLWHbEHe19cXenp6SElJwcSJEysMj48ePcKIESMgiiLq1KmDKVOmKD33+++/jzp16mDLli2IjY1Vehxl3L17Vzpu1qxZhe2LtinatyqKbtXZsGFDDB8+HMOHD8fx48eRkJCAV69eIS4uDseOHcOUKVPg7OyM6OhopeYiIiIiotpJtiDfvHlzzJ07F6Io4tChQ2jfvj02btyIR48eSW1u376NEydO4IMPPoCTkxMePXoEQRCwYMECldbIOzg4YOfOnXj58iV69+6t1jPzRXfbsbKyqrB90c+p7E49ijX5ADB//nzs378fBgYGeO+99+Dv749ff/0Vn376qbQjUGRkJHr16oWUlJRyx83JyUFGRkaxBxERERHVTrJuY/Ltt98iNjYWv/76K+7cuQM/Pz8AhXdOBYB27dpJbRW71UyZMgVz585Vee7BgwcjLCwMY8eOhaurKzp16oQuXbrA0tISOjoVf1+ZP3++UvNmZWVJx+XtGqNgbGwsHWdmZio1p2KHIKDwrH69evVw+vRpdOjQQXp9zJgx+Oijj9CnTx/cvn0bf/31F7744gv88ssvZY77/fffF9uBh4iIiIhqL1mDvCAI2L59O7p3747FixcXO3P8Tw0aNMDChQsxffp0WebOy8vD8ePHkZiYCFEUceXKlSrdLVbZIF8TCgoKij1ftmxZsRCvYGNjg507d0rbXvr7++M///lPmRfHfv7555gzZ470PCMjQ1rTT0RERES1S7VsLO7n54fJkycjMDAQoaGhiImJQVpaGkxNTWFrawt3d3cMGDAAderUkWW+vLw8DBs2DMeOHZNeq8r+9Iq/GCjD1NRUOkOenZ1d4U2XXr58KR2bmZkpNWfRfiYmJhg3blyZbZ2cnODq6orw8HDk5OQgLCys2O45RRkaGqrlTrhEREREpLpqu0OQgYEBBg0ahEGDBlXXFJItW7bg6NGjAAqXrowdOxbdu3eXdnCpTnXr1pWCfFJSUoVBPjk5uVhfZdSrV086bteuHQwMDMpt36lTJ4SHhwMAHj58qNScRERERFS7aMWtPjds2ACg8ELS8+fPo3Xr1mqbu3Xr1tKOMNHR0dJ++mUpunuMsnW+/vrr0v77ihtxladoG17ASkRERKQdZNu1Ztu2bdi2bRsOHTpUpX6xsbFSX2Xdv38fgiBg1qxZag3xQPELeCvaLSchIUHaHtPa2hoNGjRQak4nJyfpOD09vcL2RdtUJvgTERERUe0nW5CfNGkSJk+ejGHDhuHdd9+t9F1Wr127hkmTJqm0j7xijbsqd4dVVv/+/aXjiu7WWnQNf0V3gS3PgAEDpM8cGRmJV69eldu+6EW/6v6iQ0RERETVQ7YgryCKIvz9/eHp6Vmps8VF+ymrefPmAGpm2Yi7uztsbGwAACEhIbh27Vqp7fLz87Fy5UrpuY+Pj9JzKi4YBoDnz59jx44dZba9efOmtD7ezMwM3bp1U3peIiIiIqo9ZA/yZmZmEEURwcHB6Nq1q1ruKDp8+HCIooigoKBqn+ufdHV1i21dOWHCBDx79qxEu3nz5uHGjRsAgG7dusHT07PU8fz9/SEIAgRBgIeHR5nzfvfdd9Lx3Llzcf369RJtEhISMHbsWOn57Nmzi+1jT0RERESaS/Ygv3btWnh5eUEURdy9excuLi74448/5J6mmFmzZqF58+bYu3dvjYR5X19fvPXWWwCAW7duwcnJCfPnz0dAQADWrl2LHj16YNmyZQAKd6pZt26dynN27doVn332GYDCG0S5urpi6tSp2LZtG3bt2oXPPvsMjo6OuHXrFoDCnWu++uorleclIiIiotqhWs7IHzp0CLNmzYIoikhKSkKfPn2wa9cuuacqNufRo0fRvHlzDBkyBEuXLlXrMhs9PT3s27dP2mozPj4e33zzDUaPHo2ZM2fi/PnzAAqXxBw9ehRt27aVZd4ffvgBX3zxBXR1dfHq1Sts2LABEydOxJgxY/Cf//wHKSkpAABPT08EBgZW6s6zRERERKQZqmX7SR0dHaxcuRKtWrXCnDlzkJOTg3HjxuH+/fvVcgfV3r17Ayi8OVN2dja++OILfP3112jdujUsLS2ho1P+9xVBEKTtHJVlZmaGw4cP4+DBg9i2bRsuX76MZ8+ewczMDC1atIC3tzf8/Pxk3zXm22+/xahRo7Bp0yYEBQXhyZMnyM3NhbW1Ndzc3DBhwoQybwBFRERERJpLEFW5yrQIHR0dCIKAAwcO4O2335ZeP3bsGEaPHo3MzEwIgoCxY8di06ZN0NfXBwAcPHgQw4YNgyAIld7ppqy5ixJFsVJ3bFW0U3ZubZaRkQELCwukp6fD3NxcLXM6zDuqlnmISPvFGI2p6RKISFssrPwGLqqqSv6SfWnNP3l5eeHcuXOws7ODKIr49ddf0bdvX2nZh1xEUSz2KO210h5ERERERJqo2oM8ALz55psIDw+Hs7MzRFHE+fPn4erqinv37skyfkFBgUoPno0nIiIiIk2jliAPAI0aNcK5c+cwZMgQiKKIhw8fomvXrggJCVFXCUREREREWkNtQR4AjI2NsX//fsyZMweiKCItLa3YTZKUNWXKFEyZMgX/+9//ZKiSiIiIiKj2U2uQBwp3iFm2bBl++eUX6OrqyjKmv78/tm7diry8PFnGIyIiIiKq7WTbfnLChAkQBAFNmzatVPupU6eiWbNmGDVqFNLTVbsSuH79+khNTa303EREREREmk62IO/v71/lPm+99RZSU1NVnrtp06ZITU2VZSwiIiIiIk2g9qU11WHQoEEQRVHlmzoREREREWkKrQjy06dPR7169bBv3z7ugkNERERE/wpaEeQbNWqE3bt3w9TUFG+//TZWrVqFFy9e1HRZRERERETVpspr5Js3bw6gcPeZhw8flnhdWf8cryqmTJkCAGjXrh3CwsLw4Ycf4vPPP0eHDh1ga2sLY2PjCufetGmTUnMTEREREdUEQRRFsSoddHQKT+ILglDsjqg6OjoQBAFVHO7/CvnHeFWtSRCEYq+JoljitfLw7q4lZWRkwMLCAunp6TA3N1fLnA7zjqplHiLSfjFGY2q6BCLSFgtV22GxKqqSv6p8Rr5p06alBuSyXleX0r5AVPZLRU3WTURERESkjCoH+ZiYmCq9rg7R0dE1NjcRERERUU2QbR/5mmRvb1/TJRARERERqZVW7FpDRERERPRvwyBPRERERKSBtGJpTWn++usvhIeHIy4uDpmZmTAzM0Pjxo3h4uLCpThEREREpPGqHOS3bdtWHXUAACZMmKDyGHv37sUPP/yA69evl9mmQ4cO+OKLL+Dt7a3yfERERERENUGpfeSrY7tGQRCQl5endP+CggJMnjwZO3bsAFD+1pOK+sePH48tW7Zw+8kycB95ItJk3EeeiGSjLfvIA5Xfn12dZs+eje3bt0vPW7RogX79+uG1116DqakpsrKycO/ePQQFBeHBgwcAgO3bt8PMzAyrVq2qqbKJiIiIiJRS5SC/ZcuW6qhDJdeuXcPPP/8MQRBQt25d/Pzzzxg1alSZ7X/77TdMnz4dKSkp+PnnnzF58mQ4OzursWIiIiIiItVUOchPnDixOupQyYYNGyCKIgwMDHDq1Cl06NCh3PYjR45Ey5Yt0bVrV+Tm5mLDhg34+eef1VQtEREREZHqtGL7ybNnz0IQBIwbN67CEK/QoUMHjB8/HqIoIiQkpHoLJCIiIiKSmVYE+SdPngAAevbsWaV+PXr0AAA8ffpU9pqIiIiIiKqTVgR5xW43BgYGVeqnaK/KbjlERERERDVBK4K8tbU1AODmzZtV6hcREQEAaNCggew1ERERERFVp2oJ8n/88Qf8/Pzg5OQES0tL6OvrQ1dXt9yHnp7yN5l1cXGBKIrYsmULUlNTK9UnJSUFmzZtgiAIcHV1VXpuIiIiIqKaIGuQf/HiBXx8fNCjRw9s3LgRkZGRSE1NRX5+PkRRrPChrHfeeQcAkJiYiP79++Px48flto+NjcWAAQOQmJgIAPDx8VF6biIiIiKimqD8afBSjB07FocOHYIoijAxMUG7du0QHh4OQRDg6OgIY2NjxMTEICkpCUDhHVY7duwIExMTleYdNmwYunfvjvPnz+PKlSto06YN3nnnHemGUCYmJnj+/Dnu37+PwMBABAQE4MWLFxAEAd27d8fQoUNl+PREREREROojiDLdpvXUqVPo168fBEHAkCFDsHXrVpiZmUFHRweCIODAgQN4++23AQCXL1/GggULcOLECbRr1w6HDh2Cvb29SvMnJSWhR48euHv3LoDCLwllUXzk119/HefOnYOlpaVKc2urqtwiWC4O846qZR4i0n4xRmNqugQi0hYL09U2VVXyl2xLa7Zt2wYAaNSoEXbu3AkzM7My23bu3BnHjh3DBx98gMjISAwdOhSvXr1SaX4rKytcuXIF06dPh5GRUblLeIyMjDBz5kxcvnyZIZ6IiIiINJJsS2sUS2jeeecdGBkZlXi/tBP/y5cvx8mTJxEREYHNmzdj2rRpKtVgYmKCNWvWYNGiRTh27BguXryIuLg4ZGZmwszMDI0aNYKLiwsGDhzIAE9EREREGk22IB8fHw8AePPNN4u9rljikpOTU6KPjo4Oxo0bh6+++gp79uypMMiHhoYCAN544w3Ur1+/zHZWVlaYMGECJkyYUKXPQERERESkKWRbWpOdnQ0AJdbyKC5kLWtbyJYtWwKAtLa9PB4eHujVqxfOnz9f7PXFixdj8eLFuHfvXpXrJiIiIiLSRLKdka9bty6Sk5Px4sWLYq9bWlri+fPnePDgQan9FAE/OTlZ6bkXLlwIQRDQvn17vPbaa0qPQ0RERESkKWQ7I9+qVSsAwF9//VXs9TfeeAOiKOLUqVOl9jt79iyAkmfyS6OrqwsAyM3NVaVUIiIiIiKNJ1uQ79SpE0RRxPXr14u93r9/fwBAREQE1q1bV+y9/fv3Y/fu3RAEAZ06dapwjrp16wIAoqOj5SmaiIiIiEhDyRbk+/TpAwA4c+YM8vPzpdfHjh0r7RAzY8YMdOnSBWPGjEGXLl0wcuRIaTebqVOnVjiH4uz+6tWrcfHixRJbVpa3dzwRERERkTaRLch7enrCwcEBBgYGxZbR1K1bFxs3boSuri5EUcTVq1exe/duXL16VQrxU6ZMqdTdVX18fAAAsbGxcHNzg7GxsbTcRhRFDB06FLq6ulV+6OnJeoNbIiIiIqJqJ1uQNzQ0xKNHjxAXFwdPT89i7w0ZMgRnz55Fnz59pEAviiJee+01rF27Fhs2bKjUHL6+vhg8eHCJGzwplHcTqIoeRERERESaRG2nort27YqgoCDk5eUhKSkJJiYm5d79tTQ6Ojr4/fffcfDgQRw9ehSxsbHIycnB2bNnIQgCHB0dYWVlVU2fgIiIiIio9lD7mhI9PT3Y2Ngo3V8QBAwdOrTYUhwdncI/LHz77bd4++23VS2RiIiIiKjWk21pDRERERERqY9WXOW5ZcsWAICzs3MNV0JEREREpB7VFuSfPHmC27dvIzU1FdnZ2ZXqM2HCBKXm2rp1KwDg4cOHWLx4sVJjEBERERFpEtmDvL+/P1asWIFbt25VqZ8gCEoHecXdYb29vZXqT0RERESkaWQN8uPGjcOuXbsAQK1bOlpZWSEpKUmli2iJiIiIiDSJbEF+06ZN2Llzp/S8T58+6NGjB2xsbGBoaCjXNKVq3rw5kpKSkJCQUK3zEBERERHVFrIF+Y0bNwIA6tSpg8OHD6NXr15yDV2hoUOH4uLFizhy5AhmzpyptnmJiIiIiGqKbNtP3rp1C4IgYPr06WoN8QAwbdo02NnZITAwEAEBAWqdm4iIiIioJsgW5BU3ZercubNcQ1aahYUFDh48CFtbW0yYMAEff/wxYmJi1F4HEREREZG6yLa0xsHBAZGRkXj58qVcQ1Za7969ARQG+tjYWPz000/46aef0LhxY9ja2sLY2Ljc/oIg4PTp0+oolYiIiIhIFrIF+aFDhyIiIgLnzp3DxIkT5Rq2UkJCQiAIAgBI/xVFEU+fPsXTp0/L7SuKotSHiIiIiEhTyLa0ZsaMGbC2tsaOHTsQEREh17CVJopisUdpr5X2ICIiIiLSRLIFeWtra/z+++8wNDRE3759sXfvXrmGrlBBQYFKj/z8fLXVSkREREQkB1lvCOXq6oqIiAgMGzYM77zzDho2bIiOHTvC0tJSuhi2LIIgYNOmTXKWQ0RERESktWQN8hkZGVi0aBGioqIAAPHx8Th27Fil+zPIExERERFVjmxB/sWLF3jrrbdw5coVACi2Tr0yeMEpEREREVHlyRbk16xZg8uXLwMAbGxsMGvWLHTv3h02NjYwNDSUa5pKiYuLw+nTp3H79m2kpKQgNzeXZ/uJiIiISKvIFuR//fVXAICdnR0uX74Ma2truYautOTkZHz00UcICAiQLmBVbC/5zyA/bdo0bN68GXZ2dnj48KHaayUiIiIiUoVsu9Y8fPgQgiBg5syZNRLio6Oj0aFDB/z666/Iy8urcHvJ6dOnIy8vDzExMQgNDVVjpUREREREqpMtyCvuntq8eXO5hqy0/Px8vP3223j8+DFEUcSYMWNw8uRJrF69usw+Tk5OeO211wAAJ0+eVFepRERERESykG1pTatWrZCcnIykpCS5hqy0bdu24datWxAEAT/99BPef/99AIUX4JanV69euHfvHi5evKiOMomIiIiIZCPbGfnRo0dDFEUcPHhQriErbd++fQCA7t27SyG+Mt544w0AwL1796qlLiIiIiKi6iJbkPfz84OzszMCAwOxdetWuYatlOvXr0MQBHh7e1epn5WVFYDCi2SJiIiIiDSJbEFeX18fR48eRdeuXfHuu+9i+vTpuHXrllzDl0sRxG1tbavUT3G32YKCAtlrIiIiIiKqTrKtkVdc5Jqbm4uCggKsX78e69evh4mJCerXry+F5rIIgqD0NpAmJiZIS0urcE38P8XFxQEA6tevr9S8REREREQ1RbYgHxMTI92dVRAEaevHrKwsZGVlVdhflTu72tnZIS0tDREREVXqd+7cOQCFF+oSEREREWkS2YJ806ZNVQrjqvDw8EBERAQCAgLw7bffwsDAoMI+0dHROHToEARBQK9evdRQJRERERGRfGQ9I19TJk+ejFWrVuHp06f44IMP8PPPP5fbPiUlBSNHjkRubi709fUxZcoUNVVKRERERCQP2S52rUlOTk6YNGkSRFHE+vXr0b9/f5w6dQqZmZlSmxcvXuDPP//Ef//7Xzg5OUk73XzwwQews7OrweqJiIiIiKpOtjPyvXv3BlC4l/vixYvlGrbSfv75Zzx69Ahnz55FUFAQgoKCAPzf2nszMzOprWL9vqenJ77//nu110pEREREpCrZzsifPXsWZ8+ehbW1tVxDVomBgQFOnTqFzz77DIaGhhBFUXoAKPbc0NAQn3zyCY4cOQJdXV1Z6zh06BBGjhwJBwcHGBkZwdraGm5ubvjxxx+RkZEh61xlmTRpEgRBkB4LFy5Uy7xEREREpD6ynZG3srJCUlISbGxs5BqyynR1dfH9999jzpw52L17N0JDQxETE4O0tDSYmprC1tYW7u7u8PHxQZMmTWSdOysrC2PHjsWhQ4eKvZ6YmIjExERcuHABq1atwp49e+Dq6irr3EUdP35c7TfkIiIiIiL1k3Uf+aSkJCQkJMg1pNIaNGiAWbNmYdasWWqZLz8/HyNHjsSJEycAAA0bNoSvry8cHR2RkpKCXbt2ISwsDLGxsfDy8kJYWBjatGkjex0ZGRnw8/MDULi3/vPnz2Wfg4iIiIhqB9mW1gwdOhSiKOLIkSNyDakxNm7cKIV4R0dH3Lx5E9988w1Gjx6NmTNn4vz58/j4448BAKmpqVLYltsnn3yC2NhY2NnZVdscRERERFQ7yBbkp02bBjs7OwQGBiIgIECuYWu9/Px8LFq0SHq+fft2NGzYsES7pUuXon379gAKb0QVGBgoax1nzpzBhg0bAABr164tdnEvEREREWkf2YK8hYUFDh48CFtbW0yYMAEff/xxje0tHxUVhYULF8LLywvt27dHixYt0L59e3h5eWHRokW4deuWbHOFhoYiLi4OAODu7g5nZ+dS2+nq6mL27NnS8127dslWw4sXL+Dr6wtRFPHOO+9g0KBBso1NRERERLWT7NtPWlhYIDY2Fj/99BN++uknNG7cGLa2tjA2Ni63vyAIOH36tEo1xMbGws/PDydPniz1/cjISJw8eRKLFy/GgAED8PPPP6u8h/zx48elYy8vr3LbDhgwoNR+qvr888/x6NEj1K9fH//73/9kG5eIiIiIai/ZgnxISIi0Z7viv6Io4unTp3j69Gm5fUVRlPoo68aNG3jrrbeQkpIibTlZnuPHj6Njx44ICgqCk5OT0vNGRkZKx507dy63rY2NDezs7BAbG4uEhAQkJiaiQYMGSs8NAH/88QdWr14NAFi2bFmpy3qIiIiISPvIemfXonu1l7Z/e1kPVWVmZmLQoEFITk6GKIqws7PD999/j8uXLyMtLQ25ublIS0vDlStX8MMPP6Bp06YQRRFJSUkYNGhQsTvAVtXdu3el42bNmlXYvmibon2VkZ2djSlTpqCgoAB9+vTB5MmTVRqPiIiIiDSHbEG+oKBApUd+fr7Sc//00094+vQpBEGAt7c37ty5g88++wwdO3aEubk5dHV1YW5uDmdnZ3z66ae4c+cOhg8fDgB4+vSpSstR0tLSpGMrK6sK21taWpbaVxnz58/H3bt3YWxsjHXr1qk0FhERERFpFlnPyNeUAwcOAABatmyJnTt3ok6dOuW2NzY2xq+//opWrVpBFEXs27dP6bmzsrKkYyMjowrbF71WQJW/BFy+fBkrVqwAACxatAgtWrRQeiyFnJwcZGRkFHsQERERUe2kFUH+4cOHEAQBEydOhIGBQaX6GBgYYNKkSQCAR48eVWN18nv16hWmTJmC/Px8ODs7Y86cObKM+/3338PCwkJ6qHohMBERERFVH60I8gqtWrWqUvuWLVuqPKepqal0nJ2dXWH7ly9fSsfK7vW+ZMkSREVFQVdXFxs2bICurq5S4/zT559/jvT0dOkRGxsry7hEREREJD/Zdq35p7y8PISFhSE8PBxxcXHIzMyEmZkZGjduDBcXF3Tr1g16evJM37RpU9y+fbvKa84V7Zs2bar03HXr1kVqaioAICkpqViwL01ycnKxvlV18+ZN/PDDDwCAOXPmlLlvvTIMDQ1haGgo23hEREREVH1kD/KiKGL58uVYsWIFEhISymxnY2ODjz/+GB999JHKW08OGjQIt27dwtGjR+Hr61vpfkePHoUgCBg8eLDSc7du3RrR0dEAgOjoaDg4OJTbXtFW0beq/P39kZubCx0dHejr62PJkiWltgsNDS12rGjXunVrjBw5ssrzEhEREVHtImuQf/nyJQYNGoSQkBAAKHdrybi4OHzyySc4duwYjhw5UqkLRcvy/vvvY8OGDTh8+DD27NmDUaNGVdjnt99+w6FDh2BlZYX3339f6bnbtWuHEydOACi8ALVXr15ltk1ISJCWq1hbWyu1h7ziZ1pQUIDvvvuuUn2Cg4MRHBwMABgyZAiDPBEREZEWkHWN/Pjx4xEcHCyFzd69e2Pp0qU4cOAAgoKCcODAASxduhS9e/eGIAgQRRHBwcEYP368SvM2btwY+/btQ7169TBu3Dh8+umniI+PL7VtfHw8PvvsM4wdOxaWlpbYt28fGjVqpPTc/fv3l44rulvrsWPHpOOK7gJLRERERFQeQZTjjkwAzpw5g759+0IQBDRt2hQBAQFwcXEps/2lS5cwevRoREdHQxAEnDp1qtyz2eWZMmUKAOCvv/5CcHAwBEGAjo4OHB0d0apVK5iYmOD58+d48OABbt26hYKCAgCAh4cH7O3tyxxXEARs2rSp3Lnz8/Nha2srfXG4evVqqevW8/Pz0alTJ9y4cQMAcOLECXh6eirzcStl4cKFWLRoEQBgwYIFWLhwYZXHyMjIgIWFBdLT02Fubi5zhaVzmHdULfMQkfaLMRpT0yUQkbZYmK62qaqSv2RbWrNt2zYAhbu4hISElBuQAaBLly44ffo0nJyckJWVha1btyod5P39/aV19or/5ufnIyoqClFRUcXaiqIotVEsASpPRUFeV1cX8+fPx4wZMwAAEyZMwJkzZ2BtbV2s3bx586QQ361btzJDvL+/v3SHVnd390rVSERERET/PrIF+bCwMAiCgMmTJ1cY4hUcHBwwefJkrFy5EufPn1dp/tL+sFDWHxsq+0eIyl6E6+vrKy0funXrFpycnODr6wtHR0ekpKRg165d0uerW7cu78JKRERERCqTLcjHxcUBKDzTXhWK9uXtcFORojvB1AQ9PT3s27cPY8aMwZEjRxAfH49vvvmmRDtbW1vs3r0bbdu2rYEqiYiIiEibyBbkFWevFevPK0uOJfqV/QtAdTIzM8Phw4dx8OBBbNu2DZcvX8azZ89gZmaGFi1awNvbG35+frCwsKjpUomIiIhIC8h2sWurVq3w6NEjzJo1C//73/8q3e+DDz7AqlWr0KJFC9y/f1+OUkgmvNiViDQZL3YlItnU0otdZdt+snv37hBFEf7+/vj7778r1eevv/7Cli1bIAgCunfvLlcpRERERERaT7YgP2HCBABAVlYWevXqhStXrpTb/sqVK+jTpw+ysrIAABMnTpSrFCIiIiIirSfbGvlevXph2LBhOHDgAGJiYuDq6goPDw/069cPr732mrSX+/379xEYGCjdOEoQBAwbNgweHh5ylUJEREREpPVkC/IAsGPHDgwYMAChoaHSXVuDg4NLbatYmu/u7o7t27fLWQYRERERkdaTbWkNABgbG+PMmTNYunQpbGxsIIpimY9GjRrhxx9/xOnTp2FsbCxnGUREREREWk/WM/IAoKOjg08++QQfffQR/vjjD1y8eBFxcXHIzMyEmZkZGjVqBBcXF7i5uUFPT/bpiYiIiIj+FaotSevp6aFnz57o2bNndU1BRERERPSvJevSGiIiIiIiUg8GeSIiIiIiDcQgT0RERESkgZRaI18d694FQcDZs2dlH7cyYmNjYWdnVyNzExEREREpQ6kgf/78eQiCIFsRihtDKatfv37YsWMHrK2tq9x33759mDp1KpKTk5Wen4iIiIhI3ZReWlPeHvFVecjh1KlTcHJyQmBgYKX7ZGdnw8/PD6NGjUJaWposdRARERERqYtSZ+Sjo6NVmjQ/Px/+/v7473//ixcvXqg0lsKzZ8/g5eWFOXPm4Lvvvit3j/qoqCj4+Pjgzp07EEURRkZGstRARERERKQuSgV5e3t7pSc8ePAgvvzyS9y5cwdA4Zn9OnXq4MMPP1R6zCVLlmDhwoXIy8vD8uXLERISgoCAADRv3rxE2zVr1uCTTz5BTk4ORFFE69atERAQoPTcREREREQ1QW271pw/fx7du3eHt7e3dCZcV1cX06ZNw4MHD7BkyRKlx/7iiy9w9uxZ2NvbQxRFXL16FR06dMDOnTulNqmpqRg6dChmz56N7OxsiKKIyZMn4+rVq3BycpLjIxIRERERqU21B/moqCgMHjwY7u7uuHDhgrQuftSoUbh9+zbWrl0LGxsblefp2rUrbty4geHDh0MURWRmZmL8+PGYPHkyjh07hjfffBOHDx+GKIowNzfHzp07sWnTJtSpU0fluYmIiIiI1K3agvzff/+NiRMnokOHDjh27Jh0cWvfvn1x+fJlBAQEoGXLlrLOaWFhgd9++w0///wzjI2NIYoitm3bhsGDB+PJkycQRRFdunTB9evX4ePjI+vcRERERETqJHuQT05OxkcffYTWrVtjx44dyM/PhyiKcHZ2RmBgIAIDA+Hs7Cz3tMX4+flh//790nPF9pbDhg1DWFgYmjVrVq3zExERERFVN9mC/IsXL7BkyRK0aNECK1eulC4mbdmyJQICAnDlyhX07dtXrunKdfjwYYwbNw6CIBTb4vLw4cNYunSpWmogIiIiIqpOKgf5/Px8rF27Fi1atMCCBQuQkZEBURRhY2ODtWvX4vbt2xg1apQctVbo1atXeP/99zF06FCkpKRAFEV4eXlJO+Lk5+fj66+/Rt++fREXF6eWmoiIiIiIqoNKQT4gIACvv/463n//fTx79ky6kHTJkiV48OABpk2bVu5+7nK6e/cuXFxcsHbtWoiiCH19faxYsQJHjhyR/mtlZQVRFBEcHAwnJyccPXpULbUREREREclNqSAfGBiIjh07YuzYsXj48CFEUYSBgQE++ugjPHr0CF988YVad4PZtGkTOnXqhIiICIiiiFatWuHChQvF9qYfMGAAIiIi0KdPH4iiiKSkJLz99tv48MMPkZubq7ZaiYiIiIjkoFSQ79+/P27cuCHtBT9p0iTcv38fy5cvR/369eWusUK+vr548eIFRFHE+PHjce3aNXTo0KFEu4YNGyIoKEi686soili1ahVcXFzUXjMRERERkSpUWvciCAIMDAxw6tQpnDp1SqVCBEHAX3/9pXR/ExMTrF27FuPGjauw7bx589CrVy+MGTMG0dHRuHnzptLzEhERERHVBJUXsGdnZ+PJkycqjaHYHlJZzs7OVd6X3sXFBTdu3ICvry9+++03pecmIiIiIqoJSgf5ots61rQLFy5AX1+/yv3MzMwQEBCAfv36VUNVRERERETVR6kgX1BQIHcdKlEmxBc1ZcoUmSohIiIiIlIP2e/sSkRERERE1Y9BnoiIiIhIA6nnbk1qduHCBYSHh+Px48fIyMhAfn5+ue0FQcCmTZvUVB0RERERkeq0KsgfO3YMc+bMwf3796vcl0GeiIiIiDSJ1gT5zZs3Y+rUqRBFscIddQRBKNZGla0viYiIiIhqglaskX/8+DFmzJiBgoICNGjQAJs2bcKdO3cAFIb09evXIyoqCocPH8aMGTNQp04dCIKAyZMn49GjR3j06FENfwIiIiIioqrRijPya9euxatXr6Cvr4/AwEC8+eabxd63traGo6MjHB0dMXDgQMyZMweDBw+Gv78/zM3N8d///reGKiciIiIiUo5WnJE/c+YMBEHA0KFDS4T40jRv3hzHjx9HnTp1sHLlSoSGhqqhSiIiIiIi+WhFkH/48CEAoGfPnqW+n5ubW+K1pk2bYty4cRBFEZs3b67W+oiIiIiI5KYVQT49PR0AYGNjU+x1Q0NDAMDz589L7de1a1cAQFhYWDVWR0REREQkP60I8kZGRgCAvLy8Yq+bm5sDKLwYtjS6uroAgPj4+GqsjoiIiIhIfloR5O3s7AAASUlJxV5v1aoVAODSpUul9lPsbENEREREpGm0IsgrLnC9fft2sde7du0KURRx4sSJEltMpqSkYP369RAEAS1btlRbrUREREREctCKIN+zZ0+IooiQkJBir48bNw6CICA3NxceHh5Yu3YtAgMDsXbtWnTq1Ek6g+/t7V0DVRMRERERKU8QK7oNqgZ48uQJ7OzsIAgCbty4gXbt2knv+fn5YcOGDaXevVUURTRv3hzXr1+HmZmZOkvWCBkZGbCwsEB6erp0vUF1c5h3VC3zEJH2izEaU9MlEJG2WJiutqmqkr+04oZQTZo0QXBwMLKzs1G3bt1i761duxZ6enpYt24dCgoKir3n4uKCgIAAhngiIiIi0jhaEeQBwN3dvdTXdXV1sWbNGnz++ec4deoU4uPjYWJigs6dO8PV1VXNVRIRERERyUNrgnxFbG1tMWnSpJoug4iIiIhIFlpxsSsRERER0b8NgzwRERERkQZikCciIiIi0kBatUY+JSUFW7ZswYkTJ3D79m2kpqYiJyenwn6CICAvL08NFRIRERERyUNrgvzRo0cxadIkpKSkACjcI56IiIiISFtpRZCPiIiAt7c38vLyIIoiBEGAg4MDbGxsYGhoWNPlERERERHJTiuC/JIlS5CbmwtBEDBhwgQsWbIEtra2NV0WEREREVG10YogHxoaCkEQ0K9fP/j7+9d0OURERERE1U4rdq1JT08HAIwaNaqGKyEiIiIiUg+tCPJNmjQBAJiYmNRwJURERERE6qEVQb5Lly4AgD///LOGKyEiIiIiUg+tCPKzZs2CKIrYunVrpfaNJyIiIiLSdFoR5N3c3PD1118jOjoao0aNQlZWVk2XRERERERUrbRi1xoAWLRoESwsLPDll1+iVatWmDBhArp06QJLS0vo6FT8faVnz55qqJKIiIiISB5aE+QBoGPHjmjVqhWioqKwbNmySvcTBAF5eXnVWBkRERERkby0YmkNAHz33Xfo3bs3bt26BUEQIIpilR5ERERERJpEK87IBwUF4auvvpKet2rVCt26dYONjQ0MDQ1rsDIiIiIiouqhFUFesYxGX18fGzduxPjx42u4IiIiIiKi6qUVS2siIiIgCAImT57MEE9ERERE/wpaEeQzMzMBAB4eHjVbCBERERGRmmhFkLe1tQUA5Ofn13AlRERERETqoRVB3tPTEwBw+fLlGq6EiIiIiEg9tCLIz549G3Xq1MHmzZvx999/13Q5RERERETVTiuCfIsWLbB9+3bk5eWhd+/euHjxYk2XRERERERUrbRi+8nFixcDAN566y0cPnwYbm5ucHZ2hqurKywtLaGjU/H3lfnz51d3mUREREREshFELbitqY6ODgRBkJ6LoljseWXwQtmSMjIyYGFhgfT0dJibm6tlTod5R9UyDxFpvxijMTVdAhFpi4XpapuqKvlLK87IA4Xhvbzn5alq6CciIiIiqmlaEeSDg4NrugQiIiIiIrXSiiDv7u5e0yVIDh06hO3bt+Py5cuIj4+Hubk5WrZsiWHDhsHPz0+2JSqZmZkIDAxEcHAwrl27hvv37yMtLQ3GxsZo3LgxunTpgjFjxsDT05N/cSAiIiLSQlqxRr42yMrKwtixY3Ho0KEy29jZ2WHPnj1wdXVVaa4VK1bgyy+/RHZ2doVte/TogR07dqBp06ZVnodr5IlIk3GNPBHJhmvktVd+fj5GjhyJEydOAAAaNmwIX19fODo6IiUlBbt27UJYWBhiY2Ph5eWFsLAwtGnTRun57t27J4X4Jk2aoG/fvujYsSOsra2RnZ2N8PBw7NixA1lZWTh37hw8PDwQHh4Oa2trWT4vEREREdU8npGXwbp16zBt2jQAgKOjI86cOYOGDRsWazN37lwsX74cQOFZ8tDQUKXnmz59Oh49eoS5c+eiT58+pW6v+ddff8HT0xN3794FAEyePBmbN2+u0jw8I09Emoxn5IlINrX0jDyDvIry8/NhZ2eHuLg4AMDVq1fh7OxcartOnTrhxo0bAICTJ0+iX79+Ss2ZkpKC+vXrV9ju5s2baN++PQCgTp06SExMRJ06dSo9D4M8EWkyBnkikk0tDfJacWfXmhQaGiqFeHd391JDPADo6upi9uzZ0vNdu3YpPWdlQjwAODk5oXXr1gCAFy9e4MGDB0rPSURERES1C4O8io4fPy4de3l5ldt2wIABpfarTkW/yb18+VItcxIRERFR9WOQV1FkZKR03Llz53Lb2tjYwM7ODgCQkJCAxMTEaq3t1atXuHfvnvTc3t6+WucjIiIiIvVhkFeR4mJSAGjWrFmF7Yu2Kdq3OuzcuRPp6YVrupydnWFjY1Ot8xERERGR+jDIqygtLU06trKyqrC9paVlqX3llpiYiM8++0x6/tVXX1XbXERERESkftxHXkVZWVnSsZGRUYXtjY2NpePMzMxqqenVq1cYPnw4nj17BgAYOnQohg0bVmG/nJwc5OTkSM8zMjKqpT4iIiIiUh3PyGuZgoICTJkyBefOnQMAtGjRotL7x3///fewsLCQHor1/ERERERU+zDIq8jU1FQ6VtxttTxFd44xMzOTtRZRFDFt2jT8+uuvAICmTZvi1KlTqFevXqX6f/7550hPT5cesbGxstZHRERERPLh0hoV1a1bF6mpqQCApKSkYsG+NMnJycX6ykUURcyYMQMbNmwAANja2uLMmTNwcHCo9BiGhoYwNDSUrSYiIiIiqj48I68ixQ2XACA6OrrC9kXbFO2rClEUMXPmTPzyyy8AgCZNmiA4OBgtWrSQZXwiIiIiqn0Y5FXUrl076fjy5cvltk1ISJCWq1hbW6NBgwYqz68I8T///DMAoHHjxggODkbLli1VHpuIiIiIai8GeRX1799fOq7obq3Hjh2Tjiu6C2xl/DPEN2rUCMHBwWjVqpXKYxMRERFR7cYgryJ3d3fpRkshISG4du1aqe3y8/OxcuVK6bmPj4/Kc8+aNUsK8TY2NggODsZrr72m8rhEREREVPsxyKtIV1cX8+fPl55PmDBB2r+9qHnz5uHGjRsAgG7dusHT07PU8fz9/SEIAgRBgIeHR5nzvv/++1i7di2AwhAfEhIi25p7IiIiIqr9uGuNDHx9fXHgwAEEBQXh1q1bcHJygq+vLxwdHZGSkoJdu3bh/PnzAAp3qlm3bp1K83311VdYvXo1AEAQBHzwwQe4c+cO7ty5U24/Z2dnNG3aVKW5iYiIiKh2YJCXgZ6eHvbt24cxY8bgyJEjiI+PxzfffFOina2tLXbv3o22bduqNJ/iSwFQuE7+888/r1S/LVu2YNKkSSrNTURERES1A5fWyMTMzAyHDx/G77//Dm9vb9jZ2cHQ0BBWVlZwcXHB0qVLERUVBTc3t5oulYiIiIi0gCCKoljTRVDtlJGRAQsLC6Snp8Pc3FwtczrMO6qWeYhI+8UYjanpEohIWyxMV9tUVclfPCNPRERERKSBGOSJiIiIiDQQgzwRERERkQZikCciIiIi0kAM8kREREREGohBnoiIiIhIAzHIExERERFpIAZ5IiIiIiINxCBPRERERKSBGOSJiIiIiDQQgzwRERERkQZikCciIiIi0kAM8kREREREGohBnoiIiIhIAzHIExERERFpIAZ5IiIiIiINxCBPRERERKSBGOSJiIiIiDQQgzwRERERkQZikCciIiIi0kAM8kREREREGohBnoiIiIhIAzHIExERERFpIAZ5IiIiIiINxCBPRERERKSBGOSJiIiIiDQQgzwRERERkQZikCciIiIi0kAM8kREREREGohBnoiIiIhIAzHIExERERFpIAZ5IiIiIiINxCBPRERERKSBGOSJiIiIiDQQgzwRERERkQZikCciIiIi0kAM8kREREREGohBnoiIiIhIAzHIExERERFpIAZ5IiIiIiINxCBPRERERKSBGOSJiIiIiDQQgzwRERERkQZikCciIiIi0kAM8kREREREGohBnoiIiIhIAzHIExERERFpIAZ5IiIiIiINxCBPRERERKSBGOSJiIiIiDQQgzwRERERkQZikCciIiIi0kAM8kREREREGohBnoiIiIhIAzHIExERERFpIAZ5IiIiIiINxCBPRERERKSBGOSJiIiIiDQQgzwRERERkQZikCciIiIi0kAM8kREREREGohBnoiIiIhIAzHIExERERFpIAZ5IiIiIiINxCBPRERERKSBGOSJiIiIiDQQgzwRERERkQZikCciIiIi0kAM8kREREREGohBnoiIiIhIAzHIExERERFpIAZ5IiIiIiINxCBPRERERKSBGOSJiIiIiDQQg7zMDh06hJEjR8LBwQFGRkawtraGm5sbfvzxR2RkZGjNnERERERUswRRFMWaLkIbZGVlYezYsTh06FCZbezs7LBnzx64urpqxJwZGRmwsLBAeno6zM3NVSm10hzmHVXLPESk/WKMxtR0CUSkLRamq22qquQvPTXVpNXy8/MxcuRInDhxAgDQsGFD+Pr6wtHRESkpKdi1axfCwsIQGxsLLy8vhIWFoU2bNho3JxERERHVHgzyMti4caMUqB0dHXHmzBk0bNhQen/mzJmYO3culi9fjtTUVPj5+SE0NFTj5iQiIiKi2oNr5FWUn5+PRYsWSc+3b99eLFArLF26FO3btwcAnDt3DoGBgRo1JxERERHVLgzyKgoNDUVcXBwAwN3dHc7OzqW209XVxezZs6Xnu3bt0qg5iYiIiKh2YZBX0fHjx6VjLy+vctsOGDCg1H6aMCcRERER1S4M8iqKjIyUjjt37lxuWxsbG9jZ2QEAEhISkJiYqDFzEhEREVHtwiCvort370rHzZo1q7B90TZF+9b2OYmIiIioduGuNSpKS0uTjq2srCpsb2lpWWrf2jBnTk4OcnJypOfp6YV7pqrzplIFOS/UNhcRabcMgbdJISKZqDELKXJXZW71xCCvoqysLOnYyMiowvbGxsbScWZmZq2a8/vvvy+2G46CYmkOEZEmsajpAohIe/yg/v9FyczMhIVF+fMyyJPk888/x5w5c6TnBQUFSElJgaWlJQRBqMHKiIiUl5GRATs7O8TGxqrtLtVERMoSRRGZmZlo3LhxhW0Z5FVkamqK1NRUAEB2djZMTU3Lbf/y5Uvp2MzMrFbNaWhoCENDw2Kv1a1bV6kaiYhqG3NzcwZ5ItIIFZ2JV+DFrioqGnSTkpIqbJ+cnFxq39o+JxERERHVLgzyKmrdurV0HB0dXWH7om2K9q3tcxIRERFR7cIgr6J27dpJx5cvXy63bUJCAmJjYwEA1tbWaNCggcbMSUSkqQwNDbFgwYISSweJiDQdg7yK+vfvLx1XdOfUY8eOSccV3ZG1ts1JRKSpDA0NsXDhQgZ5ItI6DPIqcnd3h42NDQAgJCQE165dK7Vdfn4+Vq5cKT338fHRqDmJiIiIqHZhkFeRrq4u5s+fLz2fMGECnj17VqLdvHnzcOPGDQBAt27d4OnpWep4/v7+EAQBgiDAw8NDLXMSERERkeYRxMrcNorKlZeXBy8vLwQFBQEAbGxs4OvrC0dHR6SkpGDXrl04f/48gMJdY86fP4+2bduWOpa/vz8mT54MoPDMe0hISLXPSURERESah0FeJpmZmRgzZgyOHDlSZhtbW1vs3r0bbm5uZbapbJCXc04iotpo4cKF0t2mg4ODy/wrJRHRvxWX1sjEzMwMhw8fxu+//w5vb2/Y2dnB0NAQVlZWcHFxwdKlSxEVFSVroK6JOYmIyhITEyMtDVT1MWnSpJr+OEREtR7PyBMRkSxiYmLQrFkzWcaaOHEiHBwceEaeiKgcDPJERCSLFy9eIDAwsMz3o6Ki8PXXXwMA2rZtiyVLlpTZtmnTpnB2dpa9RiIibaJX0wUQEZF2qFOnDoYOHVrm+3Xr1pWOraysym1LREQV4xp5IiIiIiINxCBPRES10sKFC6WLX8vbwQsAzp8/j9GjR8PW1hZGRkZo0qQJvLy8sG/fPgDFL8StzIW0oaGhmDp1Ktq0aYO6devCyMgIdnZ2GD58OPbt24fyVqWWNtfTp08xf/58dOjQAZaWlrygl4hkwaU1RESk0T799FMsW7asWLh++vQpnj59iuPHj8PHxwfffPNNpcZKS0vD+PHjS93W9/Hjx3j8+DH279+Pnj17Yt++fbCysqpwzKCgIPj4+CAlJaXyH4qIqBIY5ImISGMtWbIEP/74IwBAEAR4e3ujf//+MDU1xb1797B582YEBASgoKCgwrEyMjLQrVs33L59GwDQqlUrjBw5Em3atIGBgQEePXqEXbt2ISIiAqGhoejbty/Cw8NhZGRU5pgPHjzAiBEjkJmZieHDh6Nv376oV68e/v77b+jp8f8FE5Fq+L8iRESkke7du4fFixcDAPT19bF37168/fbbxdrMnTsXQ4cOxZ49eyocz8/PTwrxCxcuxFdffQVdXd1ibT799FN8+umnWL58OW7evIklS5aUu/tOWFgYTExMEBQUhD59+lT1IxIRlYtr5ImISCOtXr0aubm5AAoD+z9DPFC4k87OnTuL7ZhTmoiICAQEBAAA3n33XSxYsKBEiAcAHR0dLFu2DN26dZNqyMnJKXfsJUuWMMQTUbVgkCciIo30+++/AygM17Nnzy6znZWVFcaPH1/uWFu3bpWOP/300wrnnjBhAgAgPT0dFy9eLLOdsbEx3nvvvQrHIyJSBpfWEBGRxklISEBsbCwAoE2bNrCxsSm3fa9evbBq1aoy3w8NDQUAGBkZ4fbt29ISm7I8efJEOr59+zZ69uxZarsOHTrA1NS03LGIiJTFIE9ERBrn6dOn0nGLFi0qbN+8efNy34+JiQEAZGdnY9iwYVWqpbzdaGxtbas0FhFRVXBpDRERaZznz59Lx3Xq1KmwvYmJSbnvp6WlKV3Lq1evynzP2NhY6XGJiCrCM/JERKRxigbzFy9eVNi+aPAvjampKdLS0lC/fn0kJyerXB8RkTrwjDwREWmcxo0bS8cPHz6ssP2jR4/KfV+xBCYtLQ1ZWVmqFUdEpCYM8kREpHEaNmwIOzs7AMCdO3cQHx9fbvvg4OBy33d3dwcAFBQU4OTJk/IUSURUzRjkiYhIIw0ZMgRAYfheuXJlme2SkpKwffv2csdSbCcJAIsXL0Z2drY8RRIRVSMGeSIi0kizZs2Cvr4+AGDZsmU4dOhQiTYvXrzAmDFjKryYtUuXLhg5ciSAwptDDRkyBImJiWW2F0URYWFhmDt3rvIfgIhIRbzYlYiINFLr1q0xf/58fP3118jNzcXQoUPh7e2N/v37w8zMDHfv3sWWLVsQExODUaNGYc+ePQAKbyBVmk2bNuHevXu4efMmAgMD4eDggOHDh8PV1RUNGjRAbm4uEhISEBERgVOnTuHx48do0aIFli1bps6PTUQkYZAnIiKN9dVXXyE9PR3Lly+HKIrYt28f9u3bV6yNj48PFixYIAV5MzOzUscyMzPD+fPnMWPGDOzYsQMvXrzA9u3by12Ww33iiagmcWkNERFptB9//BFnz57FqFGj0LhxYxgYGKBRo0bo378/9u7di127diE9PV1qX79+/TLHMjU1xbZt2xAVFYVPPvkEXbp0QYMGDaCnp4c6derA3t4e/fr1w8KFC3Hx4kWEhISo4RMSEZVOEEVRrOkiiIiIqtOqVaswe/ZsAMCBAwcwdOjQmi2IiEgGDPJERKTVcnNz0aFDB9y6dQv6+vp48uQJGjRoUNNlERGpjEtriIhIYz179gy3b98u8/3s7GxMmTIFt27dAgCMGDGCIZ6ItAbPyBMRkca6cuUKOnfujE6dOqFPnz5o3bo1zM3NkZmZiYiICAQEBCAuLg4AYGlpiYiIiGJ3hSUi0mTctYaIiDTelStXcOXKlTLfb9asGQ4ePMgQT0RahWfkiYhIY+Xk5ODAgQM4ceIEbt68icTERCQnJwMArKys0KFDBwwePBgTJ06EgYFBDVdLRCQvBnkiIiIiIg3Ei12JiIiIiDQQgzwRERERkQZikCciIiIi0kAM8kREREREGohBnoiIiIhIAzHIExERERFpIAZ5IiIiIiINxCBPRERERKSBGOSJiIiIiDTQ/wMzEWJMepjHnQAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# filtered plots, where we only show n_mem_state = 2\n", - "def maybe_spec_map(id: str):\n", - " spec_map = {\n", - " '4x3.95': '4x3',\n", - " 'cheese.95': 'Cheese\\nMaze',\n", - " 'paint.95': 'Paint',\n", - " 'shuttle.95': 'Shuttle',\n", - " 'example_7': 'ex. 7',\n", - " 'network': 'Network',\n", - " 'tmaze_5_two_thirds_up': 'T-maze',\n", - " 'tiger-alt-start': 'Tiger'\n", - " }\n", - " if id not in spec_map:\n", - " return id\n", - " return spec_map[id]\n", - "\n", - "def label_map(label_str: str):\n", - " if 'mem' in label_str:\n", - " n_mem_states = int(label_str.split('_')[-1])\n", - " return f\"{n_mem_states} Memory States\"\n", - " elif label_str == 'init_improvement':\n", - " return \"Memoryless\"\n", - "\n", - "filtered_ordered_plot = []\n", - " \n", - "for (label, v) in ordered_plot:\n", - " if 'mem' in label:\n", - " if int(label.split('_')[-1]) > 2:\n", - " continue\n", - "\n", - " filtered_ordered_plot.append((label, v)) \n", - " \n", - "group_width = 1\n", - "bar_width = group_width / (len(filtered_ordered_plot) + 2)\n", - "fig, ax = plt.subplots(figsize=(8, 6))\n", - "\n", - "x = np.array(all_plot_results['x'])\n", - "xlabels = [maybe_spec_map(l) for l in all_plot_results['xlabels']]\n", - "bars = []\n", - "bar_labels = []\n", - "\n", - "for i, (label, plot_dict) in enumerate(filtered_ordered_plot):\n", - " bars += ax.bar(x + (i + 1) * bar_width,\n", - " plot_dict['mean'],\n", - " bar_width,\n", - "# yerr=plot_dict['std_err'],\n", - " label=label_map(label))\n", - " bar_labels.append(label_map(label))\n", - " \n", - "for l, rect in zip(bar_labels, bars):\n", - " height = rect.get_height()\n", - " plt.text(rect.get_x() + rect.get_width() / 2.0, height, l, ha='center', va='bottom')\n", - "\n", - "ax.set_ylim([0, 1])\n", - "# ax.set_ylabel(f'Normalized Performance\\n (w.r.t. optimal {compare_to} & random initial policy)')\n", - "ax.set_ylabel(f'Normalized\\nmax performance')\n", - "\n", - "ax.set_xticks(x + group_width / 2.65)\n", - "ax.set_xticklabels(xlabels)\n", - "fig.tight_layout()\n", - "# ax.legend(framealpha=0.95)\n", - "# ax.legend(loc='upper left', bbox_to_anchor=(0.3, 0.5), framealpha=0.95)\n", - "\n", - "\n", - "# ax.set_title(\"Performance of Memory Iteration in POMDPs\")\n", - "\n", - "downloads = Path().home() / 'Downloads'\n", - "fig_path = downloads / f\"{results_dir.stem}_max_2_mem.pdf\"\n", - "fig.savefig(fig_path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d7ef59c6", - "metadata": {}, - "outputs": [], - "source": [ - "# unfiltered, all n_mem_state\n", - "group_width = 1\n", - "bar_width = group_width / (len(ordered_plot) + 2)\n", - "fig, ax = plt.subplots(figsize=(12, 6))\n", - "\n", - "x = np.array(all_plot_results['x'])\n", - "xlabels = [maybe_spec_map(l) for l in all_plot_results['xlabels']]\n", - "\n", - "for i, (label, plot_dict) in enumerate(ordered_plot):\n", - " ax.bar(x + (i + 1) * bar_width,\n", - " plot_dict['mean'],\n", - " bar_width,\n", - " yerr=plot_dict['std_err'],\n", - " label=label_map(label))\n", - "ax.set_ylim([0, 1])\n", - "# ax.set_ylabel(f'Normalized Performance\\n (w.r.t. optimal {compare_to} & random initial policy)')\n", - "ax.set_ylabel(f'Normalized performance \\n (10 runs)')\n", - "\n", - "ax.set_xticks(x + group_width / 2.4)\n", - "ax.set_xticklabels(xlabels)\n", - "ax.legend(bbox_to_anchor=(0.605, 1.02), framealpha=0.95)\n", - "fig.tight_layout()\n", - "# ax.set_title(\"Performance of Memory Iteration in POMDPs\")\n", - "\n", - "downloads = Path().home() / 'Downloads'\n", - "fig_path = downloads / f\"{results_dir.stem}_all.pdf\"\n", - "fig.savefig(fig_path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d6db67eb", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.6" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/plotting/tmaze_alpha_sweep_viz.ipynb b/scripts/plotting/tmaze_alpha_sweep_viz.ipynb deleted file mode 100644 index d4266032..00000000 --- a/scripts/plotting/tmaze_alpha_sweep_viz.ipynb +++ /dev/null @@ -1,382 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "id": "595e26fd", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import seaborn as sns\n", - "import jax.numpy as jnp\n", - "import numpy as np\n", - "\n", - "from jax.config import config\n", - "from jax.nn import softmax\n", - "from pathlib import Path\n", - "config.update('jax_platform_name', 'cpu')\n", - "\n", - "from grl.utils import load_info\n", - "from grl.utils.mdp import get_perf\n", - "from definitions import ROOT_DIR" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "5cbfea55", - "metadata": {}, - "outputs": [], - "source": [ - "results_dir = Path(ROOT_DIR, 'results', 'tmaze_sweep_alpha')\n", - "\n", - "args_to_extract = ['spec', 'algo', 'n_mem_states', 'alpha', 'seed']\n", - "group_by_args = [arg for arg in args_to_extract if arg != 'seed']" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "8239d26a", - "metadata": {}, - "outputs": [], - "source": [ - "results_path = list(results_dir.iterdir())[40]\n", - "\n", - "\n", - "info = load_info(results_path)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "id": "1c98ab50", - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "def test_mem_matrix(mem_params: jnp.ndarray):\n", - " \"\"\"\n", - " Tests the memory matrix for t-maze.\n", - " our tolerance is set to 1e-1, which seems high, but should be fine for\n", - " stochastic matrices that we have.\n", - " \"\"\"\n", - " RIGHT_ACTION = 2\n", - " UP_START_OBS = 0\n", - " DOWN_START_OBS = 1\n", - " CORRIDOR_OBS = 2\n", - "\n", - " mem_func = softmax(mem_params, axis=-1)\n", - " right_mem_func = mem_func[RIGHT_ACTION]\n", - "\n", - " # we index by zero here, since memory starts at 0\n", - " right_up_start = right_mem_func[UP_START_OBS, 0]\n", - " right_down_start = right_mem_func[DOWN_START_OBS, 0]\n", - "\n", - " # we test whether start bits set to different memory states\n", - " def test_start_bits_set(right_up_start: np.ndarray, right_down_start: np.ndarray):\n", - " return (np.abs(right_up_start - right_down_start).sum() / 2 - 1)**2\n", - "\n", - " diff_start_bits_set = test_start_bits_set(right_up_start, right_down_start)\n", - "\n", - " # now we test whether the right corridor memory function is all set or reset\n", - " right_corridor = right_mem_func[CORRIDOR_OBS]\n", - "\n", - " def test_corridor_hold_or_toggle(right_corridor: np.ndarray):\n", - " is_toggle = ((right_corridor - np.eye(2)[:, ::-1])**2).mean()\n", - " is_hold = ((right_corridor - np.eye(2))**2).mean()\n", - " return is_toggle, is_hold\n", - "\n", - " is_toggle, is_hold = test_corridor_hold_or_toggle(right_corridor)\n", - " return diff_start_bits_set.item(), is_toggle.item(), is_hold.item()\n", - "\n", - "list_for_df = []\n", - "\n", - "for results_path in list(results_dir.iterdir()):\n", - " if results_path.is_dir() or results_path.suffix != '.npy':\n", - " continue\n", - "\n", - " info = load_info(results_path)\n", - " agent_path = Path(results_path.parent, 'agents', f\"{results_path.stem}.pkl.npy\")\n", - " agent = load_info(agent_path)\n", - "\n", - " args = info['args']\n", - "\n", - " # agent = info['agent']\n", - " init_policy_info = info['logs']['initial_policy_stats']\n", - " final_mem_info = info['logs']['greedy_final_mem_stats']\n", - "\n", - " diff_start_bits_set, is_toggle, is_hold = test_mem_matrix(agent.mem_params)\n", - "\n", - " single_row = {k: args[k] for k in args_to_extract}\n", - "\n", - " single_row.update({\n", - " 'init_policy_perf': init_policy_info['discrep'].item(),\n", - " 'final_mem_perf': final_mem_info['discrep'].item(),\n", - " 'diff_start_bits_set': diff_start_bits_set,\n", - " 'is_toggle': is_toggle,\n", - " 'is_hold': is_hold,\n", - " 'is_optimal': 0.5 * diff_start_bits_set + 0.5 * min(is_toggle, is_hold),\n", - " # 'final_mem': np.array(agent.memory),\n", - " # 'final_policy': np.array(agent.policy)\n", - " })\n", - " list_for_df.append(single_row)\n", - "\n", - "df = pd.DataFrame(list_for_df)" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "id": "f487d10b", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
seedinit_policy_perffinal_mem_perfdiff_start_bits_setis_toggleis_holdis_optimal
specalgon_mem_statesalpha
tmaze_hyperparamsmi20.0000002024.50.0748250.2394940.9950070.6270471.228688e-010.558938
0.1111112024.50.0738110.2395860.9906520.6265861.233155e-010.556984
0.2222222024.50.0727970.2395620.9723280.6264851.234075e-010.547868
0.3333332024.50.0717830.2391470.8707210.6290251.209233e-010.495822
0.4444442024.50.0707690.2393210.7317540.6325441.175679e-010.424661
0.5555562024.50.0697550.2404280.5862860.6365851.138281e-010.350057
0.6666672024.50.0687410.2425790.4369860.6428121.082834e-010.272635
0.7777782024.50.0677270.2459110.2901520.6556249.765236e-020.193902
0.8888892024.50.0667130.2510770.1463390.6911457.279920e-020.109569
1.0000002024.50.0656990.0282470.0033050.9986498.611360e-070.001653
\n", - "
" - ], - "text/plain": [ - " seed init_policy_perf \\\n", - "spec algo n_mem_states alpha \n", - "tmaze_hyperparams mi 2 0.000000 2024.5 0.074825 \n", - " 0.111111 2024.5 0.073811 \n", - " 0.222222 2024.5 0.072797 \n", - " 0.333333 2024.5 0.071783 \n", - " 0.444444 2024.5 0.070769 \n", - " 0.555556 2024.5 0.069755 \n", - " 0.666667 2024.5 0.068741 \n", - " 0.777778 2024.5 0.067727 \n", - " 0.888889 2024.5 0.066713 \n", - " 1.000000 2024.5 0.065699 \n", - "\n", - " final_mem_perf \\\n", - "spec algo n_mem_states alpha \n", - "tmaze_hyperparams mi 2 0.000000 0.239494 \n", - " 0.111111 0.239586 \n", - " 0.222222 0.239562 \n", - " 0.333333 0.239147 \n", - " 0.444444 0.239321 \n", - " 0.555556 0.240428 \n", - " 0.666667 0.242579 \n", - " 0.777778 0.245911 \n", - " 0.888889 0.251077 \n", - " 1.000000 0.028247 \n", - "\n", - " diff_start_bits_set is_toggle \\\n", - "spec algo n_mem_states alpha \n", - "tmaze_hyperparams mi 2 0.000000 0.995007 0.627047 \n", - " 0.111111 0.990652 0.626586 \n", - " 0.222222 0.972328 0.626485 \n", - " 0.333333 0.870721 0.629025 \n", - " 0.444444 0.731754 0.632544 \n", - " 0.555556 0.586286 0.636585 \n", - " 0.666667 0.436986 0.642812 \n", - " 0.777778 0.290152 0.655624 \n", - " 0.888889 0.146339 0.691145 \n", - " 1.000000 0.003305 0.998649 \n", - "\n", - " is_hold is_optimal \n", - "spec algo n_mem_states alpha \n", - "tmaze_hyperparams mi 2 0.000000 1.228688e-01 0.558938 \n", - " 0.111111 1.233155e-01 0.556984 \n", - " 0.222222 1.234075e-01 0.547868 \n", - " 0.333333 1.209233e-01 0.495822 \n", - " 0.444444 1.175679e-01 0.424661 \n", - " 0.555556 1.138281e-01 0.350057 \n", - " 0.666667 1.082834e-01 0.272635 \n", - " 0.777778 9.765236e-02 0.193902 \n", - " 0.888889 7.279920e-02 0.109569 \n", - " 1.000000 8.611360e-07 0.001653 " - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.groupby(group_by_args).mean()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1adf4297", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.6" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/plotting/tmaze_alpha_sweep_viz.py b/scripts/plotting/tmaze_alpha_sweep_viz.py new file mode 100644 index 00000000..6006d50d --- /dev/null +++ b/scripts/plotting/tmaze_alpha_sweep_viz.py @@ -0,0 +1,98 @@ +# %% codecell +import pandas as pd +import seaborn as sns +import jax.numpy as jnp +import numpy as np + +from jax.config import config +from jax.nn import softmax +from pathlib import Path +config.update('jax_platform_name', 'cpu') + +from grl.utils import load_info +from grl.utils.mdp import get_perf +from definitions import ROOT_DIR +# %% codecell +results_dir = Path(ROOT_DIR, 'results', 'tmaze_sweep_alpha') + +args_to_extract = ['spec', 'algo', 'n_mem_states', 'alpha', 'seed'] +group_by_args = [arg for arg in args_to_extract if arg != 'seed'] +# %% codecell +results_path = list(results_dir.iterdir())[40] + + +info = load_info(results_path) + +# %% codecell + +def test_mem_matrix(mem_params: jnp.ndarray): + """ + Tests the memory matrix for t-maze. + our tolerance is set to 1e-1, which seems high, but should be fine for + stochastic matrices that we have. + """ + RIGHT_ACTION = 2 + UP_START_OBS = 0 + DOWN_START_OBS = 1 + CORRIDOR_OBS = 2 + + mem_func = softmax(mem_params, axis=-1) + right_mem_func = mem_func[RIGHT_ACTION] + + # we index by zero here, since memory starts at 0 + right_up_start = right_mem_func[UP_START_OBS, 0] + right_down_start = right_mem_func[DOWN_START_OBS, 0] + + # we test whether start bits set to different memory states + def test_start_bits_set(right_up_start: np.ndarray, right_down_start: np.ndarray): + return (np.abs(right_up_start - right_down_start).sum() / 2 - 1)**2 + + diff_start_bits_set = test_start_bits_set(right_up_start, right_down_start) + + # now we test whether the right corridor memory function is all set or reset + right_corridor = right_mem_func[CORRIDOR_OBS] + + def test_corridor_hold_or_toggle(right_corridor: np.ndarray): + is_toggle = ((right_corridor - np.eye(2)[:, ::-1])**2).mean() + is_hold = ((right_corridor - np.eye(2))**2).mean() + return is_toggle, is_hold + + is_toggle, is_hold = test_corridor_hold_or_toggle(right_corridor) + return diff_start_bits_set.item(), is_toggle.item(), is_hold.item() + +list_for_df = [] + +for results_path in list(results_dir.iterdir()): + if results_path.is_dir() or results_path.suffix != '.npy': + continue + + info = load_info(results_path) + agent_path = Path(results_path.parent, 'agents', f"{results_path.stem}.pkl.npy") + agent = load_info(agent_path) + + args = info['args'] + + # agent = info['agent'] + init_policy_info = info['logs']['initial_policy_stats'] + final_mem_info = info['logs']['greedy_final_mem_stats'] + + diff_start_bits_set, is_toggle, is_hold = test_mem_matrix(agent.mem_params) + + single_row = {k: args[k] for k in args_to_extract} + + single_row.update({ + 'init_policy_perf': init_policy_info['discrep'].item(), + 'final_mem_perf': final_mem_info['discrep'].item(), + 'diff_start_bits_set': diff_start_bits_set, + 'is_toggle': is_toggle, + 'is_hold': is_hold, + 'is_optimal': 0.5 * diff_start_bits_set + 0.5 * min(is_toggle, is_hold), + # 'final_mem': np.array(agent.memory), + # 'final_policy': np.array(agent.policy) + }) + list_for_df.append(single_row) + +df = pd.DataFrame(list_for_df) +# %% codecell +df.groupby(group_by_args).mean() +# %% codecell diff --git a/scripts/reduce_online_size.py b/scripts/reduce_online_size.py new file mode 100644 index 00000000..5e01a8fa --- /dev/null +++ b/scripts/reduce_online_size.py @@ -0,0 +1,60 @@ +from pathlib import Path +import argparse +from argparse import Namespace + +import numpy as np +from tqdm import tqdm + +from grl.environment import get_env +from grl.utils.data import uncompress_episode_rewards +from grl.utils.file_system import load_info, numpyify_and_save +from grl.utils.loss import mse +from grl.utils.mdp import all_t_discounted_returns + +from definitions import ROOT_DIR + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument('--dir_name', default='popgym_sweep_mc', type=str, + help='Results directory in the results folder to parse.') + + args = parser.parse_args() + + results_dir = Path(ROOT_DIR, 'results', args.dir_name) + + all_results_dir = [res_path for res_path in results_dir.iterdir() if res_path.suffix == '.npy'] + + for res_path in tqdm(all_results_dir): + info = load_info(res_path) + args = Namespace(**info['args']) + oi = info['episodes_info']['online_info'] + if args.gamma is None: + env = get_env(args) + args.gamma = env.gamma + raise NotImplementedError('Hyperparam gamma is None. Load the environment?') + + returns = [] + discounted_returns = [] + + for ep_len, ep_common_reward, ep_compressed_rewards in \ + zip(oi['episode_length'], oi['most_common_reward'], oi['compressed_rewards']): + ep_rewards = np.array(uncompress_episode_rewards(ep_len, ep_common_reward, ep_compressed_rewards)) + returns.append(ep_rewards.sum()) + discounted_returns.append(np.dot(ep_rewards, args.gamma ** np.arange(len(ep_rewards)))) + + returns = np.array(returns, dtype=np.float16) + discounted_returns = np.array(discounted_returns, dtype=np.float16) + + del oi['episode_length'] + del oi['most_common_reward'] + del oi['compressed_rewards'] + + oi['episode_returns'] = returns + oi['discounted_returns'] = discounted_returns + oi['total_episode_loss'] = np.array(oi['total_episode_loss'], dtype=np.float16) + # res_path.unlink(missing_ok=True) + np.save(res_path, info) + + + diff --git a/scripts/run_vi.py b/scripts/run_vi.py index a49cde3c..f1023b71 100644 --- a/scripts/run_vi.py +++ b/scripts/run_vi.py @@ -1,6 +1,8 @@ import argparse from pathlib import Path +import numpy as np + from grl.environment import load_spec from grl.mdp import MDP from grl.utils.file_system import numpyify_and_save diff --git a/scripts/tmaze_mem_interpolation.py b/scripts/tmaze_mem_interpolation.py index 9599cb5d..bf7d3fbb 100644 --- a/scripts/tmaze_mem_interpolation.py +++ b/scripts/tmaze_mem_interpolation.py @@ -12,7 +12,7 @@ from grl.utils.math import reverse_softmax from grl.utils.lambda_discrep import lambda_discrep_measures from grl.utils.file_system import numpyify_and_save -from grl.utils.loss import discrep_loss, magnitude_td_loss +from grl.utils.loss import discrep_loss, bellman_loss from definitions import ROOT_DIR if __name__ == "__main__": diff --git a/scripts/write_jobs.py b/scripts/write_jobs.py index 2e421624..8862db5e 100644 --- a/scripts/write_jobs.py +++ b/scripts/write_jobs.py @@ -4,19 +4,14 @@ """ import argparse import numpy as np -import importlib.util from typing import List, Iterable from pathlib import Path from itertools import product +from grl.utils.file_system import import_module_to_var + from definitions import ROOT_DIR -def import_module_to_hparam(hparam_path: Path) -> dict: - spec = importlib.util.spec_from_file_location("hparam", hparam_path) - hparam_module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(hparam_module) - hparams = hparam_module.hparams - return hparams def generate_runs(run_dicts: List[dict], runs_dir: Path, @@ -74,6 +69,8 @@ def generate_runs(run_dicts: List[dict], elif v is False or v is None: continue else: + if isinstance(v, list): + v = ' '.join(v) run_string += f" --{k} {v}" if experiment_name is not None and 'study_name' not in run_dict: @@ -87,14 +84,14 @@ def generate_runs(run_dicts: List[dict], if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--hparam', default='', type=str) + parser.add_argument('hyperparam_file', type=str) parser.add_argument('--local', action='store_true') args = parser.parse_args() runs_dir = Path(ROOT_DIR, 'scripts', 'runs') - hparam_path = Path(ROOT_DIR, 'scripts', 'hyperparams', args.hparam + ".py") - hparams = import_module_to_hparam(hparam_path) + hparam_path = Path(args.hyperparam_file).resolve() + hparams = import_module_to_var(hparam_path, 'hparams') results_dir = Path(ROOT_DIR, 'results') # if not args.local: @@ -116,7 +113,7 @@ def generate_runs(run_dicts: List[dict], runs_dir, runs_fname=hparams['file_name'], main_fname=main_fname, - experiment_name=args.hparam, + experiment_name=hparam_path.stem, exclude_dict=exclude_dict) print(f"Runs wrote to {runs_dir / hparams['file_name']}") diff --git a/scripts/write_pomdp.py b/scripts/write_pomdp.py index 38c5f3e5..196cf891 100644 --- a/scripts/write_pomdp.py +++ b/scripts/write_pomdp.py @@ -14,11 +14,12 @@ def convert_arr_to_lines(arr: np.ndarray) -> Union[str, list]: raise NotImplementedError if __name__ == '__main__': - # spec_name = 'tmaze_hyperparams' - spec_name = 'slippery_tmaze_5_random' - spec = load_spec(spec_name) + spec_name = 'tmaze_hyperparams' + # spec_name = 'paint.95' + spec = load_spec(spec_name, corridor_length=2) pomdp_files_dir = Path(ROOT_DIR, 'grl', 'environment', 'pomdp_files') - pomdp_path = pomdp_files_dir / 'tmaze5.POMDP' + pomdp_path = pomdp_files_dir / 'tmaze2.POMDP' + # pomdp_path = pomdp_files_dir / 'paint.95-action-cross.POMDP' lines = [f"# Converted POMDP file for {spec_name}"] lines.append('') @@ -56,4 +57,4 @@ def convert_arr_to_lines(arr: np.ndarray) -> Union[str, list]: with open(pomdp_path, 'w') as f: f.writelines([l + "\n" for l in lines]) - print(f"Written spec {spec_name} to POMDP file {pomdp_path}") \ No newline at end of file + print(f"Written spec {spec_name} to POMDP file {pomdp_path}") diff --git a/tests/maybe_test_count.py b/tests/maybe_test_count.py index 3695e856..066f9678 100644 --- a/tests/maybe_test_count.py +++ b/tests/maybe_test_count.py @@ -130,7 +130,7 @@ def test_count(): print("Calculating analytical occupancy") c_s = functional_get_occupancy(pi_ground, mem_aug_mdp) - c_s = c_s.at[-2:].set(0) + c_s = c_s * (1 - pomdp.terminal_mask) analytical_count_dist = c_s / c_s.sum(axis=-1, keepdims=True) state_counts = np.zeros(mem_aug_mdp.state_space.n, dtype=int) diff --git a/tests/maybe_test_popgym.py b/tests/maybe_test_popgym.py new file mode 100644 index 00000000..11cd4bbe --- /dev/null +++ b/tests/maybe_test_popgym.py @@ -0,0 +1,123 @@ +from argparse import Namespace +from typing import Tuple + +import jax +import numpy as np +import gymnasium as gym + +from grl.agent import get_agent, RNNAgent +from grl.environment import get_env +from grl.evaluation import eval_episodes +from grl.model import get_network +from grl.run_sample_based import parse_arguments +from grl.sample_trainer import Trainer +from grl.utils.data import uncompress_episode_rewards +from grl.utils.optimizer import get_optimizer + +def train_agent(rand_key: jax.random.PRNGKey, args: Namespace, env: gym.Env) \ + -> Tuple[RNNAgent, dict, jax.random.PRNGKey]: + network = get_network(args, env.action_space.n) + + optimizer = get_optimizer(args.optimizer, step_size=args.lr) + + agent = get_agent(network, optimizer, env.observation_space.shape, env, args) + + trainer_key, rand_key = jax.random.split(rand_key) + trainer = Trainer(env, agent, trainer_key, args) + + final_network_params, final_optimizer_params, episodes_info = trainer.train() + return agent, final_network_params, rand_key + +def test_popgym_integration_discrete(): + args = parse_arguments(return_defaults=True) + args.seed = 2022 + + args.trunc = 5 + args.replay_size = 2500 + args.batch_size = 16 + + args.total_steps = 20000 + args.no_gamma_terminal = True + args.spec = 'popgym-RepeatPreviousEasy-v0' + args.feature_encoding = 'one_hot' + args.action_cond = 'none' + args.algo = 'multihead_rnn' + args.gamma = 0.9 + args.multihead_loss_mode = 'td' + args.multihead_action_mode = 'td' + args.lr = 0.001 + + env = get_env(args) + + np.random.seed(args.seed) + rand_key = jax.random.PRNGKey(args.seed) + + agent, network_params, rand_key = train_agent(rand_key, args, env) + + final_eval_info, rand_key = eval_episodes(agent, + network_params, + env, + rand_key, + n_episodes=5, + test_eps=0., + max_episode_steps=args.max_episode_steps) + + final_eval_rewards_compressed = final_eval_info['episode_rewards'] + + for compressed_ep_rewards in final_eval_rewards_compressed: + ep_rewards = uncompress_episode_rewards(compressed_ep_rewards['episode_length'], + compressed_ep_rewards['most_common_reward'], + compressed_ep_rewards['compressed_rewards']) + + assert sum(ep_rewards) > 0., f"We're (at least) breaking even" + +def test_popgym_integration_continuous(): + args = parse_arguments(return_defaults=True) + args.max_episode_steps = 200 + args.seed = 2022 + + # args.trunc = 5 + args.trunc = -1 + args.replay_size = 5000 + # args.batch_size = 16 + + args.total_steps = 100000 + args.no_gamma_terminal = True + args.spec = 'popgym-StatelessCartPoleEasy-v0' + args.algo = 'multihead_rnn' + args.gamma = 0.99 + args.feature_encoding = 'none' + args.hidden_size = 256 + args.action_cond = 'cat' + args.multihead_loss_mode = 'td' + args.multihead_action_mode = 'td' + args.lr = 1e-4 + + # args.residual_obs_val_input = True + env = get_env(args) + + np.random.seed(args.seed) + rand_key = jax.random.PRNGKey(args.seed) + + agent, network_params, rand_key = train_agent(rand_key, args, env) + + final_eval_info, rand_key = eval_episodes(agent, + network_params, + env, + rand_key, + n_episodes=5, + test_eps=0., + max_episode_steps=args.max_episode_steps) + + final_eval_rewards_compressed = final_eval_info['episode_rewards'] + + for compressed_ep_rewards in final_eval_rewards_compressed: + ep_rewards = uncompress_episode_rewards(compressed_ep_rewards['episode_length'], + compressed_ep_rewards['most_common_reward'], + compressed_ep_rewards['compressed_rewards']) + + assert sum(ep_rewards) > 0.5, "Could not keep up the pole for more than half the time" + +if __name__ == "__main__": + # test_popgym_integration_discrete() + test_popgym_integration_continuous() diff --git a/tests/test_rnn.py b/tests/test_rnn.py index 6bc6d249..1c672b69 100644 --- a/tests/test_rnn.py +++ b/tests/test_rnn.py @@ -1,20 +1,21 @@ from argparse import Namespace -from typing import Union, Tuple +from typing import Tuple import jax import numpy as np +import gymnasium as gym from grl.agent import get_agent, RNNAgent from grl.environment import get_env from grl.evaluation import eval_episodes -from grl.mdp import MDP, POMDP from grl.model import get_network from grl.run_sample_based import parse_arguments from grl.sample_trainer import Trainer from grl.utils.data import uncompress_episode_rewards from grl.utils.optimizer import get_optimizer +from grl.environment.wrappers import GammaTerminalWrapper -def train_agent(rand_key: jax.random.PRNGKey, args: Namespace, env: Union[MDP, POMDP]) \ +def train_agent(rand_key: jax.random.PRNGKey, args: Namespace, env: gym.Env) \ -> Tuple[RNNAgent, dict, jax.random.PRNGKey]: network = get_network(args, env.action_space.n) @@ -32,6 +33,7 @@ def test_both_values(): args = parse_arguments(return_defaults=True) chain_length = 5 args.spec = 'po_simple_chain' + args.feature_encoding = 'one_hot' args.max_episode_steps = chain_length args.seed = 2020 @@ -63,7 +65,7 @@ def test_both_values(): all_td_qs = [q_td.item()] all_mc_qs = [q_mc.item()] for t in range(args.max_episode_steps): - next_obs, reward, done, _, info = env.step(action, gamma_terminal=False) + next_obs, reward, done, _, info = env.step(action) next_action, rand_key, next_hs, _ = agent.act(network_params, next_obs, hs, rand_key) new_carry, q_td, q_mc = agent.Qs(network_params, @@ -102,11 +104,13 @@ def test_gamma_terminal(): args.total_steps = 10000 args.algo = 'multihead_rnn' + args.feature_encoding = 'one_hot' np.random.seed(args.seed) rand_key = jax.random.PRNGKey(args.seed) - env = get_env(args, n=chain_length) + env = GammaTerminalWrapper(get_env(args, n=chain_length)) + test_env = get_env(args, n=chain_length) args.multihead_loss_mode = 'td' args.multihead_action_mode = 'td' @@ -117,14 +121,15 @@ def test_gamma_terminal(): final_eval_info, rand_key = eval_episodes(agent, network_params, - env, + test_env, rand_key, n_episodes=1, test_eps=0., max_episode_steps=args.max_episode_steps) all_qs = np.array([q.item() for q in final_eval_info['episode_qs'][0]])[:-1] - ground_truth_vals = env.gamma**(np.arange(chain_length - 1)[::-1]) + ground_truth_vals = env.env.gamma**(np.arange(chain_length - 1)[::-1]) + print(ground_truth_vals) mse = ((ground_truth_vals - all_qs)**2).mean() # we set a higher tolerance here b/c gamma termination introduces @@ -136,6 +141,7 @@ def test_td_mc_values(): args = parse_arguments(return_defaults=True) chain_length = 5 args.spec = 'po_simple_chain' + args.feature_encoding = 'one_hot' args.max_episode_steps = chain_length args.seed = 2020 @@ -181,6 +187,7 @@ def test_actions(): args.no_gamma_terminal = True args.spec = 'tmaze_hyperparams' args.algo = 'multihead_rnn' + args.feature_encoding = 'one_hot' # args.residual_obs_val_input = True @@ -218,5 +225,5 @@ def test_actions(): if __name__ == "__main__": # test_gamma_terminal() # test_td_mc_values() - # test_both_values() - test_actions() + test_both_values() + # test_actions()