From c0550e95bc060ab5bfc3f528a82db84bfb94fa24 Mon Sep 17 00:00:00 2001 From: Romain Pignolet Date: Sun, 7 Jul 2024 01:02:26 +0200 Subject: [PATCH 1/6] feat(kerberos): support multiple principals for krb_* modules - "krb_principal" can take a list of principals to create or delete. - "krb_keytab" can take a list of principals to put inside one keytab. - Move "try_kinit" in module_utils to be used by other modules. --- plugins/module_utils/kerberos.py | 23 ++++++++++++++++ plugins/modules/krb_keytab.py | 37 ++++++------------------- plugins/modules/krb_principal.py | 47 ++++++++++++++++++-------------- 3 files changed, 58 insertions(+), 49 deletions(-) diff --git a/plugins/module_utils/kerberos.py b/plugins/module_utils/kerberos.py index fcaa433c..a14f9678 100644 --- a/plugins/module_utils/kerberos.py +++ b/plugins/module_utils/kerberos.py @@ -1,6 +1,9 @@ # Copyright 2022 TOSIT.IO # SPDX-License-Identifier: Apache-2.0 +import os +import tempfile +import shutil kerberos_spec = dict( kerberos=dict(type='bool', default=False), @@ -42,6 +45,26 @@ def kinit(module): module.run_command(kinit_cmd, check_rc=True) +def try_kinit(module, kinit_bin, kdestroy_bin, principals, keytab_path): + """Try kinit, return True if success, False or exception otherwise""" + # Create a tmp dir to store the krb cache in order to not override + # an existing cache in default location + for principal in principals: + tmp_dir = tempfile.mkdtemp(suffix='_ansible_module_utils_kerberos') + try: + ccache = os.path.join(tmp_dir, "krb5cc") + kinit_cmd = get_kinit_cmd(kinit_bin, principal, keytab_path, ccache) + rc, stdout, stderr = module.run_command(kinit_cmd) + if rc == 0: + kdestroy_cmd = get_kdestroy_cmd(kdestroy_bin, ccache) + module.run_command(kdestroy_cmd) + else: + return False + finally: + shutil.rmtree(tmp_dir) + return True + + def get_kdestroy_cmd(kdestroy_bin, ccache=None): kdestroy_cmd = [kdestroy_bin] if ccache: diff --git a/plugins/modules/krb_keytab.py b/plugins/modules/krb_keytab.py index 6b5b9e19..d9a6e965 100644 --- a/plugins/modules/krb_keytab.py +++ b/plugins/modules/krb_keytab.py @@ -9,36 +9,16 @@ __metaclass__ = type import os -import tempfile -import shutil from ansible.module_utils.basic import AnsibleModule from ansible.module_utils._text import to_native -from ansible_collections.tosit.tdp.plugins.module_utils.kerberos import get_kinit_cmd, get_kdestroy_cmd +from ansible_collections.tosit.tdp.plugins.module_utils.kerberos import try_kinit from ansible_collections.tosit.tdp.plugins.module_utils.kerberos_admin import kerberos_admin_spec, kadmin -def try_kinit(module, kinit_bin, kdestroy_bin, principal, keytab_path): - """Try kinit, return True if success, False or exception otherwise""" - # Create a tmp dir to store the krb cache in order to not override - # an existing cache in default location - tmp_dir = tempfile.mkdtemp(suffix='_ansible_module_krb_keytab') - try: - ccache = os.path.join(tmp_dir, "krb5cc") - kinit_cmd = get_kinit_cmd(kinit_bin, principal, keytab_path, ccache) - rc, stdout, stderr = module.run_command(kinit_cmd) - if rc == 0: - kdestroy_cmd = get_kdestroy_cmd(kdestroy_bin, ccache) - module.run_command(kdestroy_cmd) - return True - else: - return False - finally: - shutil.rmtree(tmp_dir) - def main(): argument_spec = dict( kinit_bin=dict(type='path', default='kinit'), kdestroy_bin=dict(type='path', default='kdestroy'), - principal=dict(type='str', required=True), + principal=dict(type='list', elements='str', required=True), path=dict(type='path', required=True), state=dict(type='str', choices=['present', 'absent'], default='present'), **kerberos_admin_spec @@ -52,7 +32,7 @@ def main(): kinit_bin = module.params['kinit_bin'] kdestroy_bin = module.params['kdestroy_bin'] - principal = module.params['principal'] + principals = module.params['principal'] keytab_path = module.params['path'] state = module.params['state'] @@ -83,7 +63,7 @@ def main(): return module.exit_json(**results) # Case when keytab exists, try kinit to verify if the keytab is working. - if try_kinit(module, kinit_bin, kdestroy_bin, principal, keytab_path): + if try_kinit(module, kinit_bin, kdestroy_bin, principals, keytab_path): # Update file permissions for existing keytab if needed file_args = module.load_file_common_arguments(module.params) results['changed'] = module.set_fs_attributes_if_different( @@ -95,16 +75,17 @@ def main(): # Either the keytab does not exist or it is not valid, so it must be generated results['changed'] = True if not module.check_mode: - rc, stdout, stderr = kadmin(module, ['-q', 'ktadd -k {} {}'.format(keytab_path, principal)]) + principal_args = ' '.join(principals) + rc, stdout, stderr = kadmin(module, ['-q', 'ktadd -k {} {}'.format(keytab_path, principal_args)]) # rc is 0 when the principal does not exist... if 'Principal' in stderr and 'does not exist' in stderr: - raise RuntimeError("Failed to generate keytab for principal '{}': {}".format(principal, stderr)) + raise RuntimeError("Failed to generate keytab for principal '{}': {}".format(principals, stderr)) # Keytab generated is not guarantee to works so it must be verified, # for example, deleting a principal without deleting the keytab, then create the # same principal will reset the kvno, generate keytab in the same keytab file, # the keytab file will have the old kvno which is greater than the new kvno - if not try_kinit(module, kinit_bin, kdestroy_bin, principal, keytab_path): - raise RuntimeError("Keytab '{}' generated for principal '{}' is not working".format(keytab_path, principal)) + if not try_kinit(module, kinit_bin, kdestroy_bin, principals, keytab_path): + raise RuntimeError("Keytab '{}' generated for principal '{}' is not working".format(keytab_path, principals)) file_args = module.load_file_common_arguments(module.params) diff --git a/plugins/modules/krb_principal.py b/plugins/modules/krb_principal.py index 3e249458..5983e539 100644 --- a/plugins/modules/krb_principal.py +++ b/plugins/modules/krb_principal.py @@ -14,7 +14,7 @@ def main(): argument_spec = dict( - principal=dict(type='str', required=True), + principal=dict(type='list', elements='str', required=True), state=dict(type='str', choices=['present', 'absent'], default='present'), **kerberos_admin_spec ) @@ -24,34 +24,39 @@ def main(): supports_check_mode=True, ) - principal = module.params['principal'] + principals = module.params['principal'] state = module.params['state'] try: results = { 'changed': False, + 'created': [], + 'deleted': [], } current_state = None - rc, stdout, stderr = kadmin(module, ['-q', 'getprinc {}'.format(principal)]) - if 'Principal does not exist' in stderr: - current_state = 'absent' - else: - current_state = 'present' - - # Case when principal does not exist - if current_state == 'absent': - if state == 'absent': - return module.exit_json(**results) - results['changed'] = True - if not module.check_mode: - kadmin(module, ['-q', 'addprinc -randkey {}'.format(principal)]) - - # Case when principal exists and must be remove - if current_state == 'present' and state == 'absent': - results['changed'] = True - if not module.check_mode: - kadmin(module, ['-q', 'delprinc -force {}'.format(principal)]) + for principal in principals: + rc, stdout, stderr = kadmin(module, ['-q', 'getprinc {}'.format(principal)]) + if 'Principal does not exist' in stderr: + current_state = 'absent' + else: + current_state = 'present' + + # Case when principal does not exist + if current_state == 'absent': + if state == 'absent': + return module.exit_json(**results) + results['changed'] = True + results['created'].append(principal) + if not module.check_mode: + kadmin(module, ['-q', 'addprinc -randkey {}'.format(principal)]) + + # Case when principal exists and must be remove + if current_state == 'present' and state == 'absent': + results['changed'] = True + results['deleted'].append(principal) + if not module.check_mode: + kadmin(module, ['-q', 'delprinc -force {}'.format(principal)]) module.exit_json(**results) From 59f44a8a5eeac327b1e24f0ee1566859a369d911 Mon Sep 17 00:00:00 2001 From: Romain Pignolet Date: Sun, 7 Jul 2024 01:16:52 +0200 Subject: [PATCH 2/6] fix(kerberos): checking keytab is now working The return code of grep was not checked so nothing was check. Use a module to check if keytab is working and reuse the same code used by "krb_keytab". The new check use the new "try_kinit" which supports checking multiple principals inside one keytab. --- plugins/modules/krb_check_keytab.py | 49 +++++++++++++++++++ .../kerberos/tasks/check_secure_keytab.yml | 12 ++--- 2 files changed, 54 insertions(+), 7 deletions(-) create mode 100644 plugins/modules/krb_check_keytab.py diff --git a/plugins/modules/krb_check_keytab.py b/plugins/modules/krb_check_keytab.py new file mode 100644 index 00000000..87311508 --- /dev/null +++ b/plugins/modules/krb_check_keytab.py @@ -0,0 +1,49 @@ +#!/usr/bin/python +# Copyright 2022 TOSIT.IO +# SPDX-License-Identifier: Apache-2.0 + +# -*- coding: utf-8 -*- + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils._text import to_native +from ansible_collections.tosit.tdp.plugins.module_utils.kerberos import try_kinit + +def main(): + argument_spec = dict( + kinit_bin=dict(type='path', default='kinit'), + kdestroy_bin=dict(type='path', default='kdestroy'), + principal=dict(type='list', elements='str', required=True), + path=dict(type='path', required=True), + ) + + module = AnsibleModule( + argument_spec=argument_spec, + add_file_common_args=True, + supports_check_mode=True, + ) + + kinit_bin = module.params['kinit_bin'] + kdestroy_bin = module.params['kdestroy_bin'] + principals = module.params['principal'] + keytab_path = module.params['path'] + + try: + results = { + 'changed': False, + } + + if not try_kinit(module, kinit_bin, kdestroy_bin, principals, keytab_path): + raise RuntimeError("Keytab '{}' with principal '{}' is not working".format(keytab_path, principals)) + + module.exit_json(**results) + + except Exception: + import traceback + module.fail_json(msg=to_native(traceback.format_exc())) + +if __name__ == '__main__': + main() diff --git a/roles/utils/kerberos/tasks/check_secure_keytab.yml b/roles/utils/kerberos/tasks/check_secure_keytab.yml index f7f3efe5..429f6af0 100644 --- a/roles/utils/kerberos/tasks/check_secure_keytab.yml +++ b/roles/utils/kerberos/tasks/check_secure_keytab.yml @@ -2,14 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 --- -- name: Check kinit for {{ principal }} - ansible.builtin.shell: | - kinit -kt {{ keytabs_dir }}/{{ keytab }} {{ principal }}@{{ realm }} -c /tmp/check_keytab_cache - klist /tmp/check_keytab_cache | grep "Default principal: {{ principal }}@{{ realm }}" - rm -f /tmp/check_keytab_cache - changed_when: false +- name: Check kinit with keytab {{ keytabs_dir }}/{{ keytab }} + tosit.tdp.krb_check_keytab: + principal: "{{ principal }}" + path: "{{ keytabs_dir }}/{{ keytab }}" -- name: Set keytab permissions and ownership for {{ principal }} +- name: Set keytab permissions and ownership for {{ keytabs_dir }}/{{ keytab }} ansible.builtin.file: path: "{{ keytabs_dir }}/{{ keytab }}" owner: "{{ user | default(omit) }}" From 4f3fabd4f0e71a38ba88ef65acdf961534bd64b6 Mon Sep 17 00:00:00 2001 From: Romain Pignolet Date: Sun, 7 Jul 2024 11:06:23 +0200 Subject: [PATCH 3/6] feat(kerberos): add kerberos service to handle spnego keytabs Spnego keytabs need to be created once per machine. Add support for creating spnego HA keytabs by adding HA addresses (for example a load balancer) inside the spnego keytab. --- playbooks/kerberos_spnego_install.yml | 14 ++++++ roles/kerberos/spnego/tasks/install.yml | 50 +++++++++++++++++++ .../kerberos/kerberos_spnego.yml | 23 +++++++++ 3 files changed, 87 insertions(+) create mode 100644 playbooks/kerberos_spnego_install.yml create mode 100644 roles/kerberos/spnego/tasks/install.yml create mode 100644 tdp_vars_defaults/kerberos/kerberos_spnego.yml diff --git a/playbooks/kerberos_spnego_install.yml b/playbooks/kerberos_spnego_install.yml new file mode 100644 index 00000000..2df9740c --- /dev/null +++ b/playbooks/kerberos_spnego_install.yml @@ -0,0 +1,14 @@ +# Copyright 2022 TOSIT.IO +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Kerberos spnego install + hosts: hdfs_nn:hdfs_jn:hdfs_dn:hdfs_httpfs:yarn_rm:yarn_nm:yarn_ats:mapred_jhs:hive_s2:hbase_rest:phoenix_queryserver_daemon:ranger_admin:ranger_kms:spark_hs:spark3_hs + tasks: + - tosit.tdp.resolve: # noqa unnamed-task + node_name: kerberos_spnego + - name: Install Kerberos spnego + ansible.builtin.import_role: + name: tosit.tdp.kerberos.spnego + tasks_from: install + - ansible.builtin.meta: clear_facts # noqa unnamed-task diff --git a/roles/kerberos/spnego/tasks/install.yml b/roles/kerberos/spnego/tasks/install.yml new file mode 100644 index 00000000..48a1c186 --- /dev/null +++ b/roles/kerberos/spnego/tasks/install.yml @@ -0,0 +1,50 @@ +# Copyright 2022 TOSIT.IO +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Ensure kerberos common installation steps are performed + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: install + +- name: Kerberos spnego keytabs creation + when: krb_create_principals_keytabs + block: + - name: Ensure HTTP spnego's keytab exists + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: create_principal_keytab + vars: + principal: "{{ kerberos_spnego_default_principal }}" + keytab: spnego.service.keytab + user: root + group: "{{ hadoop_group }}" + mode: "0640" + when: kerberos_spnego_generated_ha_service_principals is not defined or + kerberos_spnego_generated_ha_service_principals | length < 1 + + - name: Ensure HTTP spnego's keytab with HA support exists + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: create_headless_principal_keytab + vars: + principal: "{{ kerberos_spnego_ha_principals }}" + keytab: spnego.service.keytab + user: root + group: "{{ hadoop_group }}" + mode: "0640" + when: + - kerberos_spnego_generated_ha_service_principals is defined + - kerberos_spnego_generated_ha_service_principals | length > 0 + +- name: Ensure HTTP spnego keytab works + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: check_secure_keytab + vars: + principal: "{{ kerberos_spnego_default_principal }}" + keytab: spnego.service.keytab + user: root + group: "{{ hadoop_group }}" + mode: "640" + when: not krb_create_principals_keytabs diff --git a/tdp_vars_defaults/kerberos/kerberos_spnego.yml b/tdp_vars_defaults/kerberos/kerberos_spnego.yml new file mode 100644 index 00000000..219dddcc --- /dev/null +++ b/tdp_vars_defaults/kerberos/kerberos_spnego.yml @@ -0,0 +1,23 @@ +# Copyright 2022 TOSIT.IO +# SPDX-License-Identifier: Apache-2.0 + +--- +kerberos_spnego_default_principal: HTTP/{{ ansible_fqdn }} +kerberos_spnego_generated_ha_fqdn: >- + {{ ansible_play_hosts | + map('extract', hostvars, 'ansible_fqdn') | + list + }} +kerberos_spnego_generated_ha_fqdn_principals: >- + {{ ['HTTP/'] | + product(kerberos_spnego_generated_ha_fqdn) | + map('join') | + list + }} +kerberos_spnego_generated_ha_service_principals: >- + [ + {%- if ranger_ha_address is defined -%} + "HTTP/{{ ranger_ha_address | urlsplit("hostname") }}", + {%- endif -%} + ] +kerberos_spnego_ha_principals: "{{ kerberos_spnego_generated_ha_fqdn_principals + kerberos_spnego_generated_ha_service_principals }}" From 8d7d0685f06a6402a1526007d652558b6c6a3dd9 Mon Sep 17 00:00:00 2001 From: Romain Pignolet Date: Sun, 7 Jul 2024 14:34:02 +0200 Subject: [PATCH 4/6] feat(kerberos): use the new kerberos_spnego_install playbook --- playbooks/ranger_kerberos_install.yml | 11 --- .../queryserver/daemon/tasks/kerberos.yml | 67 ++++++------------ roles/hbase/rest/tasks/kerberos.yml | 66 ++++++------------ roles/hdfs/datanode/tasks/kerberos.yml | 66 ++++++------------ roles/hdfs/httpfs/tasks/kerberos.yml | 68 ++++++------------- roles/hdfs/journalnode/tasks/kerberos.yml | 66 ++++++------------ roles/hdfs/namenode/tasks/kerberos.yml | 66 ++++++------------ roles/hive/hiveserver2/tasks/kerberos.yml | 66 ++++++------------ .../ranger/admin/tasks/kerberos-spnego-ha.yml | 27 -------- roles/ranger/admin/tasks/kerberos.yml | 23 ------- .../common/templates/install.properties.j2 | 2 +- roles/ranger/kms/tasks/kerberos.yml | 66 ++++++------------ roles/spark/historyserver/tasks/kerberos.yml | 66 ++++++------------ .../yarn/apptimelineserver/tasks/kerberos.yml | 66 ++++++------------ .../yarn/jobhistoryserver/tasks/kerberos.yml | 66 ++++++------------ roles/yarn/nodemanager/tasks/kerberos.yml | 66 ++++++------------ roles/yarn/resourcemanager/tasks/kerberos.yml | 67 ++++++------------ tdp_lib_dag/hbase.yml | 2 + tdp_lib_dag/hdfs.yml | 1 + tdp_lib_dag/hive.yml | 1 + tdp_lib_dag/kerberos.yml | 27 ++++++++ tdp_lib_dag/ranger.yml | 1 + tdp_lib_dag/spark.yml | 1 + tdp_lib_dag/spark3.yml | 1 + tdp_lib_dag/yarn.yml | 1 + 25 files changed, 300 insertions(+), 660 deletions(-) delete mode 100644 roles/ranger/admin/tasks/kerberos-spnego-ha.yml create mode 100644 tdp_lib_dag/kerberos.yml diff --git a/playbooks/ranger_kerberos_install.yml b/playbooks/ranger_kerberos_install.yml index 590b5a91..3ac85cd7 100644 --- a/playbooks/ranger_kerberos_install.yml +++ b/playbooks/ranger_kerberos_install.yml @@ -13,17 +13,6 @@ name: tosit.tdp.ranger.admin tasks_from: kerberos - ansible.builtin.meta: clear_facts # noqa unnamed-task -- name: Kerberos Ranger Admin HA install - hosts: spnego_ha - strategy: linear - tasks: - - tosit.tdp.resolve: # noqa unnamed-task - node_name: ranger_kerberos - - name: Install Ranger Admin Kerberos - ansible.builtin.import_role: - name: tosit.tdp.ranger.admin - tasks_from: kerberos-spnego-ha - - ansible.builtin.meta: clear_facts # noqa unnamed-task - name: Kerberos Ranger UserSync install hosts: ranger_usersync strategy: linear diff --git a/roles/hbase/phoenix/queryserver/daemon/tasks/kerberos.yml b/roles/hbase/phoenix/queryserver/daemon/tasks/kerberos.yml index 92d0142b..6007d91f 100644 --- a/roles/hbase/phoenix/queryserver/daemon/tasks/kerberos.yml +++ b/roles/hbase/phoenix/queryserver/daemon/tasks/kerberos.yml @@ -7,51 +7,26 @@ name: tosit.tdp.utils.kerberos tasks_from: install -- when: krb_create_principals_keytabs - block: - - name: Ensure phoenix queryserver principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: phoenixqueryserver/{{ ansible_fqdn }} - keytab: phoenixqueryserver.service.keytab - user: "{{ phoenix_queryserver_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" +- name: Ensure phoenix queryserver principal and keytab exist + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: create_principal_keytab + vars: + principal: phoenixqueryserver/{{ ansible_fqdn }} + keytab: phoenixqueryserver.service.keytab + user: "{{ phoenix_queryserver_user }}" + group: "{{ hadoop_group }}" + mode: "0600" + when: krb_create_principals_keytabs -- name: Phoenix QueryServer keytabs check +- name: Ensure phoenix queryserver keytab works + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: check_secure_keytab + vars: + principal: "phoenixqueryserver/{{ ansible_fqdn }}" + keytab: "phoenixqueryserver.service.keytab" + user: "{{ phoenix_queryserver_user }}" + group: "{{ hadoop_group }}" + mode: "0600" when: not krb_create_principals_keytabs - block: - - name: Ensure phoenix queryserver keytab works - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: "phoenixqueryserver/{{ ansible_fqdn }}" - keytab: "phoenixqueryserver.service.keytab" - user: "{{ phoenix_queryserver_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego keytab works - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" diff --git a/roles/hbase/rest/tasks/kerberos.yml b/roles/hbase/rest/tasks/kerberos.yml index 7b90337e..ffa75d8b 100644 --- a/roles/hbase/rest/tasks/kerberos.yml +++ b/roles/hbase/rest/tasks/kerberos.yml @@ -23,52 +23,26 @@ hbase_keytab_file: "{{ hbase_site['hbase.rest.keytab.file'] }}" hbase_principal: "{{ hbase_rest_kerberos_principal }}" -- name: HBase Rest keytabs creation +- name: Ensure hbase principal and keytab exist + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: create_principal_keytab + vars: + principal: hbase/{{ ansible_fqdn }} + keytab: hbase.service.keytab + user: "{{ hbase_user }}" + group: "{{ hadoop_group }}" + mode: "0600" when: krb_create_principals_keytabs - block: - - name: Ensure hbase principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: hbase/{{ ansible_fqdn }} - keytab: hbase.service.keytab - user: "{{ hbase_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: "HTTP/{{ ansible_fqdn }}" - keytab: "spnego.service.keytab" - user: "root" - group: "{{ hadoop_group }}" - mode: "0640" -- name: HBase Rest keytabs check +- name: Ensure hbase keytab works + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: check_secure_keytab + vars: + principal: "hbase/{{ ansible_fqdn }}" + keytab: "hbase.service.keytab" + user: "{{ hbase_user }}" + group: "{{ hadoop_group }}" + mode: "0600" when: not krb_create_principals_keytabs - block: - - name: Ensure hbase keytab works - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: "hbase/{{ ansible_fqdn }}" - keytab: "hbase.service.keytab" - user: "{{ hbase_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego keytab works - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: "HTTP/{{ ansible_fqdn }}" - keytab: "spnego.service.keytab" - user: "root" - group: "{{ hadoop_group }}" - mode: "0640" diff --git a/roles/hdfs/datanode/tasks/kerberos.yml b/roles/hdfs/datanode/tasks/kerberos.yml index f5b8c56c..8abe07c0 100644 --- a/roles/hdfs/datanode/tasks/kerberos.yml +++ b/roles/hdfs/datanode/tasks/kerberos.yml @@ -12,52 +12,26 @@ name: tosit.tdp.hadoop.common tasks_from: kerberos -- name: HDFS Datanode keytabs creation +- name: Ensure hdfs dn user's principal and keytab exist + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: create_principal_keytab + vars: + principal: dn/{{ ansible_fqdn }} + keytab: dn.service.keytab + user: "{{ hdfs_user }}" + group: "{{ hadoop_group }}" + mode: "0600" when: krb_create_principals_keytabs - block: - - name: Ensure hdfs dn user's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: dn/{{ ansible_fqdn }} - keytab: dn.service.keytab - user: "{{ hdfs_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" -- name: HDFS Datanode keytabs check +- name: Ensure hdfs dn user's keytab is working + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: check_secure_keytab + vars: + principal: dn/{{ ansible_fqdn }} + keytab: dn.service.keytab + user: "{{ hdfs_user }}" + group: "{{ hadoop_group }}" + mode: "0600" when: not krb_create_principals_keytabs - block: - - name: Ensure hdfs dn user's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: dn/{{ ansible_fqdn }} - keytab: dn.service.keytab - user: "{{ hdfs_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" diff --git a/roles/hdfs/httpfs/tasks/kerberos.yml b/roles/hdfs/httpfs/tasks/kerberos.yml index 58fc4bad..eb0812dd 100644 --- a/roles/hdfs/httpfs/tasks/kerberos.yml +++ b/roles/hdfs/httpfs/tasks/kerberos.yml @@ -12,50 +12,26 @@ name: tosit.tdp.hadoop.common tasks_from: kerberos -- when: krb_create_principals_keytabs - block: - - name: Ensure httpfs user's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: httpfs/{{ ansible_fqdn }} - keytab: httpfs.service.keytab - user: "{{ hdfs_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: "HTTP/{{ ansible_fqdn }}" - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" - -- when: not krb_create_principals_keytabs - block: - - name: Ensure httpfs user's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: httpfs/{{ ansible_fqdn }} - keytab: httpfs.service.keytab - user: "{{ hdfs_user }}" - group: "{{ hadoop_group }}" - mode: "0600" +- name: Ensure httpfs user's principal and keytab exist + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: create_principal_keytab + vars: + principal: httpfs/{{ ansible_fqdn }} + keytab: httpfs.service.keytab + user: "{{ hdfs_user }}" + group: "{{ hadoop_group }}" + mode: "0600" + when: krb_create_principals_keytabs - - name: Ensure HTTP spnego's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" +- name: Ensure httpfs user's keytab is working + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: check_secure_keytab + vars: + principal: httpfs/{{ ansible_fqdn }} + keytab: httpfs.service.keytab + user: "{{ hdfs_user }}" + group: "{{ hadoop_group }}" + mode: "0600" + when: not krb_create_principals_keytabs diff --git a/roles/hdfs/journalnode/tasks/kerberos.yml b/roles/hdfs/journalnode/tasks/kerberos.yml index 013cc38e..da31b31e 100644 --- a/roles/hdfs/journalnode/tasks/kerberos.yml +++ b/roles/hdfs/journalnode/tasks/kerberos.yml @@ -7,52 +7,26 @@ name: tosit.tdp.utils.kerberos tasks_from: install -- name: HDFS JournalNode keytabs creation +- name: Ensure hdfs jn user's principal and keytab exist + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: create_principal_keytab + vars: + principal: jn/{{ ansible_fqdn }} + keytab: jn.service.keytab + user: "{{ hdfs_user }}" + group: "{{ hadoop_group }}" + mode: "0600" when: krb_create_principals_keytabs - block: - - name: Ensure hdfs jn user's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: jn/{{ ansible_fqdn }} - keytab: jn.service.keytab - user: "{{ hdfs_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: "HTTP/{{ ansible_fqdn }}" - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" -- name: HDFS JournalNode keytabs check +- name: Ensure hdfs jn user's keytab is working + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: check_secure_keytab + vars: + principal: jn/{{ ansible_fqdn }} + keytab: jn.service.keytab + user: "{{ hdfs_user }}" + group: "{{ hadoop_group }}" + mode: "0600" when: not krb_create_principals_keytabs - block: - - name: Ensure hdfs jn user's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: jn/{{ ansible_fqdn }} - keytab: jn.service.keytab - user: "{{ hdfs_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" diff --git a/roles/hdfs/namenode/tasks/kerberos.yml b/roles/hdfs/namenode/tasks/kerberos.yml index 4bc28816..e378194a 100644 --- a/roles/hdfs/namenode/tasks/kerberos.yml +++ b/roles/hdfs/namenode/tasks/kerberos.yml @@ -20,52 +20,26 @@ group: root mode: "644" -- name: HDFS NameNode keytabs creation +- name: Ensure hdfs nn user's principal and keytab exist + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: create_principal_keytab + vars: + principal: nn/{{ ansible_fqdn }} + keytab: nn.service.keytab + user: "{{ hdfs_user }}" + group: "{{ hadoop_group }}" + mode: "0600" when: krb_create_principals_keytabs - block: - - name: Ensure hdfs nn user's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: nn/{{ ansible_fqdn }} - keytab: nn.service.keytab - user: "{{ hdfs_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" -- name: HDFS NameNode keytabs check +- name: Ensure hdfs nn user's keytab is working + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: check_secure_keytab + vars: + principal: nn/{{ ansible_fqdn }} + keytab: nn.service.keytab + user: "{{ hdfs_user }}" + group: "{{ hadoop_group }}" + mode: "0600" when: not krb_create_principals_keytabs - block: - - name: Ensure hdfs nn user's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: nn/{{ ansible_fqdn }} - keytab: nn.service.keytab - user: "{{ hdfs_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" diff --git a/roles/hive/hiveserver2/tasks/kerberos.yml b/roles/hive/hiveserver2/tasks/kerberos.yml index a04fdd26..3dacd915 100644 --- a/roles/hive/hiveserver2/tasks/kerberos.yml +++ b/roles/hive/hiveserver2/tasks/kerberos.yml @@ -7,52 +7,26 @@ name: tosit.tdp.utils.kerberos tasks_from: install -- name: HiveServer2 keytabs creation +- name: Ensure hive user's principal and keytab exist + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: create_principal_keytab + vars: + principal: hive/{{ ansible_fqdn }} + keytab: hive.service.keytab + user: "{{ hive_user }}" + group: "{{ hadoop_group }}" + mode: "0600" when: krb_create_principals_keytabs - block: - - name: Ensure hive user's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: hive/{{ ansible_fqdn }} - keytab: hive.service.keytab - user: "{{ hive_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" -- name: HiveServer2 keytabs check +- name: Ensure hive's keytab is working + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: check_secure_keytab + vars: + principal: hive/{{ ansible_fqdn }} + keytab: hive.service.keytab + user: "{{ hive_user }}" + group: "{{ hadoop_group }}" + mode: "0600" when: not krb_create_principals_keytabs - block: - - name: Ensure hive's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: hive/{{ ansible_fqdn }} - keytab: hive.service.keytab - user: "{{ hive_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" diff --git a/roles/ranger/admin/tasks/kerberos-spnego-ha.yml b/roles/ranger/admin/tasks/kerberos-spnego-ha.yml deleted file mode 100644 index 9f59d3a6..00000000 --- a/roles/ranger/admin/tasks/kerberos-spnego-ha.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2022 TOSIT.IO -# SPDX-License-Identifier: Apache-2.0 - ---- -- name: Ensure HTTP HA spnego user's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_headless_principal_keytab - vars: - principal: HTTP/{{ ranger_ha_address | urlsplit("hostname") }} - keytab: '{{ ranger_ha_address | urlsplit("hostname") }}.service.keytab' - user: root - group: "{{ hadoop_group }}" - mode: "0640" - when: ranger_ha_address is defined - -- name: Ensure HA HTTP spnego's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: HTTP/{{ ranger_ha_address | urlsplit("hostname") }} - keytab: '{{ ranger_ha_address | urlsplit("hostname") }}.service.keytab' - user: root - group: "{{ hadoop_group }}" - mode: "0640" - when: ranger_ha_address is defined diff --git a/roles/ranger/admin/tasks/kerberos.yml b/roles/ranger/admin/tasks/kerberos.yml index a2ba8c15..e253a57f 100644 --- a/roles/ranger/admin/tasks/kerberos.yml +++ b/roles/ranger/admin/tasks/kerberos.yml @@ -32,17 +32,6 @@ group: "{{ hadoop_group }}" mode: "0600" - - name: Ensure HTTP spnego user's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" - - name: Ranger Admin keytabs check when: not krb_create_principals_keytabs block: @@ -67,15 +56,3 @@ user: "{{ ranger_user }}" group: "{{ hadoop_group }}" mode: "0600" - - - name: Ensure HTTP spnego's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" - when: not krb_create_principals_keytabs diff --git a/roles/ranger/common/templates/install.properties.j2 b/roles/ranger/common/templates/install.properties.j2 index 6e4bde4e..0b7f38e2 100644 --- a/roles/ranger/common/templates/install.properties.j2 +++ b/roles/ranger/common/templates/install.properties.j2 @@ -195,7 +195,7 @@ xa_ldap_ad_userSearchFilter= #------------ Kerberos Config ----------------- spnego_principal=HTTP/{% if ranger_ha_address is defined %}{{ ranger_ha_address | urlsplit("hostname") }}{% else %}{{ ansible_fqdn }}{% endif %}@{{ realm }} -spnego_keytab=/etc/security/keytabs/{% if ranger_ha_address is defined %}{{ ranger_ha_address | urlsplit("hostname") }}.service.keytab{% else %}spnego.service.keytab{% endif %} +spnego_keytab=/etc/security/keytabs/spnego.service.keytab token_valid=30 cookie_domain= cookie_path=/ diff --git a/roles/ranger/kms/tasks/kerberos.yml b/roles/ranger/kms/tasks/kerberos.yml index 2cc6e693..24bdbd4f 100644 --- a/roles/ranger/kms/tasks/kerberos.yml +++ b/roles/ranger/kms/tasks/kerberos.yml @@ -7,52 +7,26 @@ name: tosit.tdp.utils.kerberos tasks_from: install -- name: Ranger KMS keytabs creation +- name: Ensure keyadmin user's principal and keytab exist + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: create_principal_keytab + vars: + principal: keyadmin/{{ ansible_fqdn }} + keytab: keyadmin.service.keytab + user: "{{ ranger_kms_user }}" + group: "{{ hadoop_group }}" + mode: "0600" when: krb_create_principals_keytabs - block: - - name: Ensure keyadmin user's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: keyadmin/{{ ansible_fqdn }} - keytab: keyadmin.service.keytab - user: "{{ ranger_kms_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego user's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" -- name: Ranger KMS keytabs check +- name: Ensure keyadmin's keytab is working + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: check_secure_keytab + vars: + principal: keyadmin/{{ ansible_fqdn }} + keytab: keyadmin.service.keytab + user: "{{ ranger_kms_user }}" + group: "{{ hadoop_group }}" + mode: "0600" when: not krb_create_principals_keytabs - block: - - name: Ensure keyadmin's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: keyadmin/{{ ansible_fqdn }} - keytab: keyadmin.service.keytab - user: "{{ ranger_kms_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" diff --git a/roles/spark/historyserver/tasks/kerberos.yml b/roles/spark/historyserver/tasks/kerberos.yml index e2f3f2e0..566a9efc 100644 --- a/roles/spark/historyserver/tasks/kerberos.yml +++ b/roles/spark/historyserver/tasks/kerberos.yml @@ -12,52 +12,26 @@ name: tosit.tdp.hadoop.common tasks_from: kerberos -- name: Spark HS keytabs creation +- name: Ensure spark user's principal and keytab exist + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: create_principal_keytab + vars: + principal: spark/{{ ansible_fqdn }} + keytab: spark.service.keytab + user: "{{ spark_user }}" + group: "{{ hadoop_group }}" + mode: "0600" when: krb_create_principals_keytabs - block: - - name: Ensure spark user's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: spark/{{ ansible_fqdn }} - keytab: spark.service.keytab - user: "{{ spark_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego's keytab exists - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" -- name: Spark HS keytabs check +- name: Ensure spark keytab works + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: check_secure_keytab + vars: + principal: spark/{{ ansible_fqdn }} + keytab: spark.service.keytab + user: "{{ spark_user }}" + group: "{{ hadoop_group }}" + mode: "600" when: not krb_create_principals_keytabs - block: - - name: Ensure spark keytab works - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: spark/{{ ansible_fqdn }} - keytab: spark.service.keytab - user: "{{ spark_user }}" - group: "{{ hadoop_group }}" - mode: "600" - - - name: Ensure HTTP spnego keytab works - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: "HTTP/{{ ansible_fqdn }}" - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "640" diff --git a/roles/yarn/apptimelineserver/tasks/kerberos.yml b/roles/yarn/apptimelineserver/tasks/kerberos.yml index b5a05fea..c41a2b09 100644 --- a/roles/yarn/apptimelineserver/tasks/kerberos.yml +++ b/roles/yarn/apptimelineserver/tasks/kerberos.yml @@ -12,52 +12,26 @@ name: tosit.tdp.hadoop.common tasks_from: kerberos -- name: YARN ATS keytabs creation +- name: Ensure yarn ats user's principal and keytab exist + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: create_principal_keytab + vars: + principal: ats/{{ ansible_fqdn }} + keytab: ats.service.keytab + user: "{{ yarn_user }}" + group: "{{ hadoop_group }}" + mode: "0600" when: krb_create_principals_keytabs - block: - - name: Ensure yarn ats user's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: ats/{{ ansible_fqdn }} - keytab: ats.service.keytab - user: "{{ yarn_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" -- name: YARN ATS keytabs check +- name: Ensure yarn ats user's keytab is working + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: check_secure_keytab + vars: + principal: ats/{{ ansible_fqdn }} + keytab: ats.service.keytab + user: "{{ yarn_user }}" + group: "{{ hadoop_group }}" + mode: "0600" when: not krb_create_principals_keytabs - block: - - name: Ensure yarn ats user's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: ats/{{ ansible_fqdn }} - keytab: ats.service.keytab - user: "{{ yarn_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" diff --git a/roles/yarn/jobhistoryserver/tasks/kerberos.yml b/roles/yarn/jobhistoryserver/tasks/kerberos.yml index f6ee423e..e5d7f212 100644 --- a/roles/yarn/jobhistoryserver/tasks/kerberos.yml +++ b/roles/yarn/jobhistoryserver/tasks/kerberos.yml @@ -12,52 +12,26 @@ name: tosit.tdp.hadoop.common tasks_from: kerberos -- name: YARN JHS keytabs creation +- name: Ensure mapred jhs user's principal and keytab exist + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: create_principal_keytab + vars: + principal: jhs/{{ ansible_fqdn }} + keytab: jhs.service.keytab + user: "{{ mapred_user }}" + group: "{{ hadoop_group }}" + mode: "0600" when: krb_create_principals_keytabs - block: - - name: Ensure mapred jhs user's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: jhs/{{ ansible_fqdn }} - keytab: jhs.service.keytab - user: "{{ mapred_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" -- name: YARN JHS keytabs check +- name: Ensure mapred jhs user's keytab is working + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: check_secure_keytab + vars: + principal: jhs/{{ ansible_fqdn }} + keytab: jhs.service.keytab + user: "{{ mapred_user }}" + group: "{{ hadoop_group }}" + mode: "0600" when: not krb_create_principals_keytabs - block: - - name: Ensure mapred jhs user's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: jhs/{{ ansible_fqdn }} - keytab: jhs.service.keytab - user: "{{ mapred_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" diff --git a/roles/yarn/nodemanager/tasks/kerberos.yml b/roles/yarn/nodemanager/tasks/kerberos.yml index fe604487..9f856644 100644 --- a/roles/yarn/nodemanager/tasks/kerberos.yml +++ b/roles/yarn/nodemanager/tasks/kerberos.yml @@ -12,52 +12,26 @@ name: tosit.tdp.hadoop.common tasks_from: kerberos -- name: YARN RM keytabs creation +- name: Ensure yarn nm user's principal and keytab exist + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: create_principal_keytab + vars: + principal: nm/{{ ansible_fqdn }} + keytab: nm.service.keytab + user: "{{ yarn_user }}" + group: "{{ hadoop_group }}" + mode: "0600" when: krb_create_principals_keytabs - block: - - name: Ensure yarn nm user's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: nm/{{ ansible_fqdn }} - keytab: nm.service.keytab - user: "{{ yarn_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" -- name: YARN RM keytabs check +- name: Ensure yarn nm user's keytab is working + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: check_secure_keytab + vars: + principal: nm/{{ ansible_fqdn }} + keytab: nm.service.keytab + user: "{{ yarn_user }}" + group: "{{ hadoop_group }}" + mode: "0600" when: not krb_create_principals_keytabs - block: - - name: Ensure yarn nm user's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: nm/{{ ansible_fqdn }} - keytab: nm.service.keytab - user: "{{ yarn_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" diff --git a/roles/yarn/resourcemanager/tasks/kerberos.yml b/roles/yarn/resourcemanager/tasks/kerberos.yml index 963a62ef..7c03c21d 100644 --- a/roles/yarn/resourcemanager/tasks/kerberos.yml +++ b/roles/yarn/resourcemanager/tasks/kerberos.yml @@ -20,51 +20,26 @@ group: root mode: "644" -- when: krb_create_principals_keytabs - block: - - name: Ensure yarn rm user's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: rm/{{ ansible_fqdn }} - keytab: rm.service.keytab - user: "{{ yarn_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego's principal and keytab exist - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: create_principal_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" +- name: Ensure yarn rm user's principal and keytab exist + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: create_principal_keytab + vars: + principal: rm/{{ ansible_fqdn }} + keytab: rm.service.keytab + user: "{{ yarn_user }}" + group: "{{ hadoop_group }}" + mode: "0600" + when: krb_create_principals_keytabs -- name: YARN RM keytabs check +- name: Ensure yarn rm user's keytab is working + ansible.builtin.import_role: + name: tosit.tdp.utils.kerberos + tasks_from: check_secure_keytab + vars: + principal: rm/{{ ansible_fqdn }} + keytab: rm.service.keytab + user: "{{ yarn_user }}" + group: "{{ hadoop_group }}" + mode: "0600" when: not krb_create_principals_keytabs - block: - - name: Ensure yarn rm user's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: rm/{{ ansible_fqdn }} - keytab: rm.service.keytab - user: "{{ yarn_user }}" - group: "{{ hadoop_group }}" - mode: "0600" - - - name: Ensure HTTP spnego's keytab is working - ansible.builtin.import_role: - name: tosit.tdp.utils.kerberos - tasks_from: check_secure_keytab - vars: - principal: HTTP/{{ ansible_fqdn }} - keytab: spnego.service.keytab - user: root - group: "{{ hadoop_group }}" - mode: "0640" diff --git a/tdp_lib_dag/hbase.yml b/tdp_lib_dag/hbase.yml index 306c7dac..9f8aaa83 100644 --- a/tdp_lib_dag/hbase.yml +++ b/tdp_lib_dag/hbase.yml @@ -36,6 +36,7 @@ - name: hbase_kerberos_install depends_on: + - kerberos_spnego_install - hbase_client_install - hbase_master_install - hbase_regionserver_install @@ -43,6 +44,7 @@ - name: hbase_phoenix_kerberos_install depends_on: + - kerberos_spnego_install - hbase_phoenix_queryserver_daemon_install - name: hbase_ssl-tls_install diff --git a/tdp_lib_dag/hdfs.yml b/tdp_lib_dag/hdfs.yml index 22c255a1..8a7cc036 100644 --- a/tdp_lib_dag/hdfs.yml +++ b/tdp_lib_dag/hdfs.yml @@ -24,6 +24,7 @@ - name: hdfs_kerberos_install depends_on: + - kerberos_spnego_install - hdfs_namenode_install - hdfs_datanode_install - hdfs_journalnode_install diff --git a/tdp_lib_dag/hive.yml b/tdp_lib_dag/hive.yml index e1800215..1e6e5e00 100644 --- a/tdp_lib_dag/hive.yml +++ b/tdp_lib_dag/hive.yml @@ -15,6 +15,7 @@ - name: hive_kerberos_install depends_on: + - kerberos_spnego_install - hive_client_install - hive_metastore_install - hive_hiveserver2_install diff --git a/tdp_lib_dag/kerberos.yml b/tdp_lib_dag/kerberos.yml new file mode 100644 index 00000000..241125b9 --- /dev/null +++ b/tdp_lib_dag/kerberos.yml @@ -0,0 +1,27 @@ +# Copyright 2022 TOSIT.IO +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: kerberos_spnego_install + depends_on: + - hadoop_client_install + +- name: kerberos_install + noop: yes + depends_on: + - kerberos_spnego_install + +- name: kerberos_config + noop: yes + depends_on: + - kerberos_install + +- name: kerberos_start + noop: yes + depends_on: + - kerberos_config + +- name: kerberos_init + noop: yes + depends_on: + - kerberos_start diff --git a/tdp_lib_dag/ranger.yml b/tdp_lib_dag/ranger.yml index f1fa15e3..d49faa2a 100644 --- a/tdp_lib_dag/ranger.yml +++ b/tdp_lib_dag/ranger.yml @@ -20,6 +20,7 @@ - name: ranger_kerberos_install depends_on: + - kerberos_spnego_install - ranger_admin_install - ranger_kms_install - ranger_usersync_install diff --git a/tdp_lib_dag/spark.yml b/tdp_lib_dag/spark.yml index 6314f48f..0e914319 100644 --- a/tdp_lib_dag/spark.yml +++ b/tdp_lib_dag/spark.yml @@ -10,6 +10,7 @@ - name: spark_kerberos_install depends_on: + - kerberos_spnego_install - spark_client_install - spark_historyserver_install diff --git a/tdp_lib_dag/spark3.yml b/tdp_lib_dag/spark3.yml index 41dc89a2..151fe93a 100644 --- a/tdp_lib_dag/spark3.yml +++ b/tdp_lib_dag/spark3.yml @@ -10,6 +10,7 @@ - name: spark3_kerberos_install depends_on: + - kerberos_spnego_install - spark3_client_install - spark3_historyserver_install diff --git a/tdp_lib_dag/yarn.yml b/tdp_lib_dag/yarn.yml index cc5b555d..1f130832 100644 --- a/tdp_lib_dag/yarn.yml +++ b/tdp_lib_dag/yarn.yml @@ -24,6 +24,7 @@ - name: yarn_kerberos_install depends_on: + - kerberos_spnego_install - yarn_resourcemanager_install - yarn_nodemanager_install - yarn_apptimelineserver_install From 8607c1d65e2bac814335f9c98f239f6fc9ebbe88 Mon Sep 17 00:00:00 2001 From: Mohamed Mehdi BEN AISSA Date: Wed, 17 Jul 2024 15:15:03 +0200 Subject: [PATCH 5/6] feat(spark2|3): add ha support for spark-hs --- playbooks/spark3_kerberos_install.yml | 1 - tdp_vars_defaults/knox/knox.yml | 6 ++++-- tdp_vars_defaults/spark/spark.yml | 2 +- tdp_vars_defaults/spark3/spark3.yml | 2 +- tdp_vars_defaults/tdp-cluster/tdp-cluster.yml | 2 ++ topology.ini | 5 ++--- 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/playbooks/spark3_kerberos_install.yml b/playbooks/spark3_kerberos_install.yml index b229fd9d..21be7fce 100644 --- a/playbooks/spark3_kerberos_install.yml +++ b/playbooks/spark3_kerberos_install.yml @@ -13,7 +13,6 @@ name: tosit.tdp.spark.historyserver tasks_from: kerberos - ansible.builtin.meta: clear_facts # noqa unnamed-task - - name: Spark3 Kerberos Client install hosts: spark3_client strategy: linear diff --git a/tdp_vars_defaults/knox/knox.yml b/tdp_vars_defaults/knox/knox.yml index 1d2d2c9b..7d105e2e 100644 --- a/tdp_vars_defaults/knox/knox.yml +++ b/tdp_vars_defaults/knox/knox.yml @@ -157,11 +157,13 @@ tdpldap_services: location: /ws port: "{{ yarn_rm_https_port }}" SPARKHISTORYUI: - hosts: "{{ groups['spark_hs'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}" + hosts: "{% if spark2_hs_ha_address is defined %}{{ spark2_hs_ha_address | urlsplit('hostname') | split(' ') | list }}{% else %}{{ groups['spark_hs'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}{% endif %}" port: "{{ spark_hs_https_port }}" + scheme: "{% if spark2_hs_ha_address is defined %}{{ spark2_hs_ha_address | urlsplit('scheme') }}://{% endif %}" SPARK3HISTORYUI: - hosts: "{{ groups['spark3_hs'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}" + hosts: "{% if spark3_hs_ha_address is defined %}{{ spark3_hs_ha_address | urlsplit('hostname') | split(' ') | list }}{% else %}{{ groups['spark3_hs'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}{% endif %}" port: "{{ spark3_hs_https_port}}" + scheme: "{% if spark3_hs_ha_address is defined %}{{ spark3_hs_ha_address | urlsplit('scheme') }}://{% endif %}" WEBHBASE: hosts: "{{ groups['hbase_rest'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}" port: "{{ hbase_rest_client_port }}" diff --git a/tdp_vars_defaults/spark/spark.yml b/tdp_vars_defaults/spark/spark.yml index 84b926ac..38472234 100644 --- a/tdp_vars_defaults/spark/spark.yml +++ b/tdp_vars_defaults/spark/spark.yml @@ -70,7 +70,7 @@ spark_truststore_location: /etc/ssl/certs/truststore.jks spark_truststore_password: Truststore123! # Spark History Server kerberos -spark_ui_spnego_principal: "HTTP/{{ ansible_fqdn }}@{{ realm }}" +spark_ui_spnego_principal: "*" spark_ui_spnego_keytab: /etc/security/keytabs/spnego.service.keytab # spark-defaults.conf - common diff --git a/tdp_vars_defaults/spark3/spark3.yml b/tdp_vars_defaults/spark3/spark3.yml index 88a3b704..260efcb9 100644 --- a/tdp_vars_defaults/spark3/spark3.yml +++ b/tdp_vars_defaults/spark3/spark3.yml @@ -71,7 +71,7 @@ hadoop_credentials_properties: value: '{{ spark_keystore_password }}' # Spark History Server kerberos -spark_ui_spnego_principal: "HTTP/{{ ansible_fqdn }}@{{ realm }}" +spark_ui_spnego_principal: "*" spark_ui_spnego_keytab: /etc/security/keytabs/spnego.service.keytab # spark-defaults.conf - common diff --git a/tdp_vars_defaults/tdp-cluster/tdp-cluster.yml b/tdp_vars_defaults/tdp-cluster/tdp-cluster.yml index 8aaf4c7d..cacfb351 100644 --- a/tdp_vars_defaults/tdp-cluster/tdp-cluster.yml +++ b/tdp_vars_defaults/tdp-cluster/tdp-cluster.yml @@ -252,3 +252,5 @@ ldap: ############################# # ranger_ha_address: "http[s]://dns_alias:port" +# spark2_hs_ha_address: "http[s]://dns_alias:port" +# spark3_hs_ha_address: "http[s]://dns_alias:port" diff --git a/topology.ini b/topology.ini index 709ee77d..4a45e9b5 100644 --- a/topology.ini +++ b/topology.ini @@ -97,12 +97,14 @@ master3 edge [spark_hs:children] +master2 master3 [spark_client:children] edge [spark3_hs:children] +master2 master3 [spark3_client:children] @@ -111,9 +113,6 @@ edge [knox:children] edge -[spnego_ha:children] -ranger_admin - # Section Postgresql_client from tdp_prerequisites [postgresql_client:children] ranger_admin From 89aee0592062261d5f4d84c0b859ef9339dbc9b0 Mon Sep 17 00:00:00 2001 From: Mohamed Mehdi BEN AISSA Date: Wed, 17 Jul 2024 19:16:06 +0200 Subject: [PATCH 6/6] feat(yarn): add ha support for jhs --- tdp_vars_defaults/hadoop/hadoop.yml | 12 ++++++------ tdp_vars_defaults/knox/knox.yml | 8 ++++---- tdp_vars_defaults/tdp-cluster/tdp-cluster.yml | 1 + tdp_vars_defaults/yarn/yarn.yml | 2 +- topology.ini | 1 + 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/tdp_vars_defaults/hadoop/hadoop.yml b/tdp_vars_defaults/hadoop/hadoop.yml index 55b6b205..de4fca3f 100644 --- a/tdp_vars_defaults/hadoop/hadoop.yml +++ b/tdp_vars_defaults/hadoop/hadoop.yml @@ -178,7 +178,7 @@ core_site: # kerberos auth for the webuis hadoop.http.authentication.type: kerberos hadoop.http.authentication.kerberos.keytab: /etc/security/keytabs/spnego.service.keytab - hadoop.http.authentication.kerberos.principal: "HTTP/_HOST@{{ realm }}" + hadoop.http.authentication.kerberos.principal: "*" hadoop.http.filter.initializers: org.apache.hadoop.security.AuthenticationFilterInitializer hadoop.http.authentication.simple.anonymous.allowed: "false" hadoop.http.authentication.signature.secret.file: "{{ http_secret_location }}" @@ -210,10 +210,10 @@ mapred_site: yarn.app.mapreduce.am.resource.mb: 1536 mapreduce.application.classpath: /opt/tdp/hadoop/share/hadoop/mapreduce/*,/opt/tdp/hadoop/share/hadoop/mapreduce/lib/*,/etc/hadoop/conf/ # jobhistory conf - mapreduce.jobhistory.address: "{{ groups['mapred_jhs'][0] | tosit.tdp.access_fqdn(hostvars) }}:{{ mapred_jhs_rpc_port }}" - mapreduce.jobhistory.admin.address: "{{ groups['mapred_jhs'][0] | tosit.tdp.access_fqdn(hostvars) }}:{{ mapred_jhs_admin_port }}" - mapreduce.jobhistory.webapp.address: "{{ groups['mapred_jhs'][0] | tosit.tdp.access_fqdn(hostvars) }}:{{ mapred_jhs_http_port }}" - mapreduce.jobhistory.webapp.https.address: "{{ groups['mapred_jhs'][0] | tosit.tdp.access_fqdn(hostvars) }}:{{ mapred_jhs_https_port }}" + mapreduce.jobhistory.address: "0.0.0.0:{{ mapred_jhs_rpc_port }}" + mapreduce.jobhistory.admin.address: "0.0.0.0:{{ mapred_jhs_admin_port }}" + mapreduce.jobhistory.webapp.address: "0.0.0.0:{{ mapred_jhs_http_port }}" + mapreduce.jobhistory.webapp.https.address: "0.0.0.0:{{ mapred_jhs_https_port }}" mapreduce.jobhistory.intermediate-done-dir: /mr-history/tmp mapreduce.jobhistory.done-dir: /mr-history/done mapreduce.jobhistory.principal: jhs/_HOST@{{ realm }} @@ -223,7 +223,7 @@ mapred_site: mapreduce.cluster.administrators: mapred,yarn,knox,hue mapreduce.jobhistory.admin.acl: "*" mapreduce.jobhistory.http.policy: HTTPS_ONLY - mapreduce.jobhistory.webapp.spnego-principal: HTTP/_HOST@{{ realm }} + mapreduce.jobhistory.webapp.spnego-principal: "*" mapreduce.jobhistory.webapp.spnego-keytab-file: /etc/security/keytabs/spnego.service.keytab ### # The yarn.app.mapreduce.am.job.client.port-range property controls the pool of ports available to MR job application masters. diff --git a/tdp_vars_defaults/knox/knox.yml b/tdp_vars_defaults/knox/knox.yml index 7d105e2e..72b51358 100644 --- a/tdp_vars_defaults/knox/knox.yml +++ b/tdp_vars_defaults/knox/knox.yml @@ -148,9 +148,9 @@ tdpldap_services: port: "{{ hdfs_nn_https_port }}" version: 2.7.0 JOBHISTORYUI: - hosts: - - "{{ groups['mapred_jhs'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | first }}" + hosts: "{% if jhs_ha_address is defined %}{{ jhs_ha_address | urlsplit('hostname') | split(' ') | list }}{% else %}{{ groups['mapred_jhs'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}{% endif %}" port: "{{ mapred_jhs_https_port }}" + scheme: "{% if jhs_ha_address is defined %}{{ jhs_ha_address | urlsplit('scheme') }}://{% else %}https://{% endif %}" HIVE: {} RESOURCEMANAGER: hosts: "{{ groups['yarn_rm'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}" @@ -159,11 +159,11 @@ tdpldap_services: SPARKHISTORYUI: hosts: "{% if spark2_hs_ha_address is defined %}{{ spark2_hs_ha_address | urlsplit('hostname') | split(' ') | list }}{% else %}{{ groups['spark_hs'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}{% endif %}" port: "{{ spark_hs_https_port }}" - scheme: "{% if spark2_hs_ha_address is defined %}{{ spark2_hs_ha_address | urlsplit('scheme') }}://{% endif %}" + scheme: "{% if spark2_hs_ha_address is defined %}{{ spark2_hs_ha_address | urlsplit('scheme') }}://{% else %}https://{% endif %}" SPARK3HISTORYUI: hosts: "{% if spark3_hs_ha_address is defined %}{{ spark3_hs_ha_address | urlsplit('hostname') | split(' ') | list }}{% else %}{{ groups['spark3_hs'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}{% endif %}" port: "{{ spark3_hs_https_port}}" - scheme: "{% if spark3_hs_ha_address is defined %}{{ spark3_hs_ha_address | urlsplit('scheme') }}://{% endif %}" + scheme: "{% if spark3_hs_ha_address is defined %}{{ spark3_hs_ha_address | urlsplit('scheme') }}://{% else %}https://{% endif %}" WEBHBASE: hosts: "{{ groups['hbase_rest'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}" port: "{{ hbase_rest_client_port }}" diff --git a/tdp_vars_defaults/tdp-cluster/tdp-cluster.yml b/tdp_vars_defaults/tdp-cluster/tdp-cluster.yml index cacfb351..04c324c5 100644 --- a/tdp_vars_defaults/tdp-cluster/tdp-cluster.yml +++ b/tdp_vars_defaults/tdp-cluster/tdp-cluster.yml @@ -254,3 +254,4 @@ ldap: # ranger_ha_address: "http[s]://dns_alias:port" # spark2_hs_ha_address: "http[s]://dns_alias:port" # spark3_hs_ha_address: "http[s]://dns_alias:port" +# jhs_ha_address: "http[s]://dns_alias:port" diff --git a/tdp_vars_defaults/yarn/yarn.yml b/tdp_vars_defaults/yarn/yarn.yml index 980628c3..50e4cb5d 100644 --- a/tdp_vars_defaults/yarn/yarn.yml +++ b/tdp_vars_defaults/yarn/yarn.yml @@ -125,7 +125,7 @@ yarn_site: yarn.timeline-service.http-authentication.kerberos.keytab: /etc/security/keytabs/spnego.service.keytab yarn.acl.enable: "true" yarn.admin.acl: yarn,knox - yarn.log.server.url: "https://{{ groups['mapred_jhs'][0] | tosit.tdp.access_fqdn(hostvars) }}:{{ mapred_jhs_https_port }}/jobhistory/logs" + yarn.log.server.url: "{% if jhs_ha_address is defined %}{{ jhs_ha_address }}/jobhistory/logs{% else %}https://{{ groups['mapred_jhs'][0] | tosit.tdp.access_fqdn(hostvars) }}:{{ mapred_jhs_https_port }}/jobhistory/logs{% endif %}" # To enable cgroups yarn.nodemanager.linux-container-executor.resources-handler.class: "{% if cgroups_enabled %}org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler{% else %}org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler{% endif %}" yarn.nodemanager.linux-container-executor.cgroups.mount: "false" diff --git a/topology.ini b/topology.ini index 4a45e9b5..7a2ea0b4 100644 --- a/topology.ini +++ b/topology.ini @@ -64,6 +64,7 @@ worker master3 [mapred_jhs:children] +master2 master3 [hive_s2:children]