Ranger REST API SmokeTests with ROBOT Framework

.github/workflows/maven.yml

@@ -42,7 +42,7 @@ jobs:
           distribution: 'temurin'
           cache: maven
       - name: build (8)
-        run: mvn -T 8 clean install --no-transfer-progress -B -V
+        run: mvn -T 8 clean install -DskipTests --no-transfer-progress -B -V
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
@@ -66,11 +66,10 @@ jobs:
         with:
           name: target-11
           path: target/*
-  
+
   docker-build:
     needs:
       - build-8
-      - build-11
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -137,7 +136,8 @@ jobs:
           -f docker-compose.ranger-hive.yml \
           -f docker-compose.ranger-knox.yml \
           -f docker-compose.ranger-ozone.yml up -d
-      - name: Check status of containers and remove them
+
+      - name: Check status of containers
         run: | 
           sleep 60
           containers=(ranger ranger-zk ranger-solr ranger-postgres ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode);
@@ -153,8 +153,38 @@ jobs:
 
           if [[ $flag == true ]]; then
               echo "All required containers are up and running";
-              docker stop $(docker ps -q) && docker rm $(docker ps -aq);
-          else
-              docker stop $(docker ps -q) && docker rm $(docker ps -aq);
-              exit 1;
           fi
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      # Install Robot Framework and dependencies
+      - name: Install Robot Framework
+        run: |
+          python -m pip install --upgrade pip
+          python --version
+          pip install apache-ranger
+          pip install robotframework
+          pip install robotframework-requests
+          pip install robotframework-jsonlibrary
+          robot --version || true
+
+      - name: Run Ranger REST API SmokeTests
+        run: |
+          cd dev-support/smoketests/ranger
+          mkdir -p /tmp/smoketests/ranger
+          robot --outputdir /tmp/smoketests/ranger --loglevel DEBUG -P apitests user_management.robot policy_management.robot custom.robot
+
+      - name: Upload Robot Framework Test Result Artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: Robot-Framework-Artifacts
+          path: /tmp/smoketests/ranger
+
+      - name: Remove Containers
+        if: always()
+        run: |
+          docker stop $(docker ps -q) && docker rm $(docker ps -aq);

dev-support/smoketests/ranger/apitests/__init__.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

dev-support/smoketests/ranger/apitests/policy_management.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from apache_ranger.model.ranger_service import *
+from apache_ranger.client.ranger_client import *
+from apache_ranger.model.ranger_policy import *
+
+
+class TestPolicyManagement:
+    ROBOT_LIBRARY_SCOPE = 'SUITE'
+
+    def __init__(self, ranger_url, username, password):
+        self.ranger = RangerClient(ranger_url, (username, password))
+        self.login_user = username
+        self.ranger.session.verify = False
+        self.test_hive_policy_prefix = 'test_hive_policy'
+        self.test_hive_db_prefix = 'test_hive_db'
+        self.test_hive_table_prefix = 'test_hive_table'
+        return
+
+    def get_hive_policy(self, service_name, policy_name):
+        return self.ranger.get_policy(service_name, policy_name)
+
+    def delete_hive_policy(self, service_name, policy_name):
+        return self.ranger.delete_policy(service_name, policy_name)
+
+    @staticmethod
+    def _create_policy_item_accesses(access_types):
+        ret = []
+        for access_type in access_types:
+            ret.append(RangerPolicyItemAccess({'type': access_type}))
+        return ret
+
+    @staticmethod
+    def _create_policy_item(users, access_types):
+        allow_item = RangerPolicyItem()
+        allow_item.users = users
+        allow_item.accesses = TestPolicyManagement._create_policy_item_accesses(access_types)
+        return allow_item
+
+    @staticmethod
+    def _create_policy_item_with_delegate_admin(users, access_types):
+        allow_item = TestPolicyManagement._create_policy_item(users, access_types)
+        allow_item.delegateAdmin = True
+        return allow_item
+
+    @staticmethod
+    def _create_hive_policy_resource(db_name, table_name, column_name):
+        resources = {
+            'database': RangerPolicyResource({'values': [db_name]}),
+            'table': RangerPolicyResource({'values': [table_name]}),
+            'column': RangerPolicyResource({'values': [column_name]})
+        }
+        return resources
+
+    def create_hive_policy(self, service_name, policy_name, db_name, table_name):
+        policy = RangerPolicy()
+        policy.service = service_name
+        policy.name = policy_name
+        policy.resources = TestPolicyManagement._create_hive_policy_resource(db_name, table_name, "*")
+        allow_item = TestPolicyManagement._create_policy_item_with_delegate_admin(['test_user_1'], ['create', 'alter'])
+        deny_item = TestPolicyManagement._create_policy_item([self.login_user], ['drop'])
+        policy.policyItems = [allow_item]
+        policy.denyPolicyItems = [deny_item]
+
+        created_policy = self.ranger.create_policy(policy)
+        print(f'Created policy: name={created_policy.name}, id={created_policy.id}')
+        return created_policy
+
+    def get_all_policies(self):
+        all_policies = self.ranger.find_policies()
+        return all_policies
+
+    def create_policies_in_bulk(self, service_name, count):
+        count = int(count)
+        for i in range(count):
+            policy_name = f'{self.test_hive_policy_prefix}_{i}'
+            db_name = f'{self.test_hive_db_prefix}_{i}'
+            table_name = f'{self.test_hive_table_prefix}_{i}'
+            self.create_hive_policy(service_name, policy_name, db_name, table_name)
+        return
+
+    def delete_policies_in_bulk(self, service_name, count):
+        count = int(count)
+        for i in range(count):
+            policy_name = f'{self.test_hive_policy_prefix}_{i}'
+            self.delete_hive_policy(service_name, policy_name)
+        return
+
+
+class TestServiceManagement:
+    ROBOT_LIBRARY_SCOPE = 'SUITE'
+
+    def __init__(self, ranger_url, username, password):
+        self.ranger = RangerClient(ranger_url, (username, password))
+        self.ranger.session.verify = False
+        return
+
+    def create_service(self, service_name, service_type, configs):
+        service = RangerService()
+        service.name = service_name
+        service.type = service_type
+        service.configs = configs
+        return self.ranger.create_service(service)
+
+    def delete_service(self, service_name):
+        return self.ranger.delete_service(service_name)
+

dev-support/smoketests/ranger/apitests/user_management.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from apache_ranger.client.ranger_client import *
+from apache_ranger.utils import *
+from apache_ranger.model.ranger_user_mgmt import *
+from apache_ranger.client.ranger_user_mgmt_client import *
+
+
+class TestUserManagement:
+    def __init__(self, ranger_url, username, password):
+        self.ranger = RangerClient(ranger_url, (username, password))
+        self.ranger.session.verify = False
+        self.ugclient = RangerUserMgmtClient(self.ranger)
+        return
+
+    ROBOT_LIBRARY_SCOPE = 'SUITE'
+
+    def find_users(self):
+        print('Listing all users!')
+        users = self.ugclient.find_users()
+        print(f'{len(users.list)} users found')
+        return users
+
+    def find_groups(self):
+        print('Listing all groups!')
+        groups = self.ugclient.find_groups()
+        print(f'{len(groups.list)} groups found')
+        return groups
+
+    def create_user(self, user_name, role):
+        user = RangerUser({'name': user_name,
+                           'firstName': user_name,
+                           'lastName': 'lnu',
+                           'emailAddress': user_name + '@test.org',
+                           'password': 'Welcome1',
+                           'userRoleList': [role],
+                           'otherAttributes': '{ "dept": "test" }'})
+
+        created_user = self.ugclient.create_user(user)
+        print(f'User {created_user.name} created!')
+        return created_user
+
+    def create_group(self, group_name):
+        group = RangerGroup({'name': group_name, 'otherAttributes': '{ "dept": "test" }'})
+        created_group = self.ugclient.create_group(group)
+        print(f'Group {created_group.name} created!')
+        return created_group
+
+    def add_to_group(self, group_name, group_id, user_id):
+        group_user = RangerGroupUser({'name': group_name, 'parentGroupId': group_id, 'userId': user_id})
+        created_group_user = self.ugclient.create_group_user(group_user)
+        print(f'Created group-user: {created_group_user}')
+        return created_group_user
+
+    def list_users_in_group(self, group_name):
+        users = self.ugclient.get_users_in_group(group_name)
+        return users
+
+    def list_groups_for_user(self, user_name):
+        groups = self.ugclient.get_groups_for_user(user_name)
+        return groups
+
+    def list_group_users(self):
+        group_users = self.ugclient.find_group_users()
+        print(f'{len(group_users.list)} group-users found')
+
+        for group_user in group_users.list:
+            print(f'id: {group_user.id}, groupId: {group_user.parentGroupId}, userId: {group_user.userId}')
+        return group_users
+
+    def delete_user_by_id(self, id):
+        self.ugclient.delete_user_by_id(id, True)
+        return
+
+    def delete_group_by_id(self, id):
+        self.ugclient.delete_group_by_id(id, True)
+        return
+
+    def delete_group_user_by_id(self, id):
+        self.ugclient.delete_group_user_by_id(id)
+        return
+

dev-support/smoketests/ranger/custom.robot

@@ -0,0 +1,23 @@
+*** Settings ***
+Library        policy_management.TestPolicyManagement       http://localhost:6080    admin           rangerR0cks!      WITH NAME   admin_p
+Library        policy_management.TestPolicyManagement       http://localhost:6080    test_user_1     Welcome1          WITH NAME   user_t
+Library        policy_management.TestPolicyManagement       http://localhost:6080    finance_user    Welcome1          WITH NAME   user_f
+Library        Collections
+Library        JSONLibrary
+
+*** Variables ***
+
+
+*** Test Cases ***
+Admin User Succeeds To Create Policy Regular User Fails
+    [Documentation]         A regular user fails to create hive policy whereas an admin user succeeds.
+    ${response}             admin_p.Create Hive Policy    dev_hive    test_policy_custom_1     test_db_custom_1     test_table_custom_1
+    Log                     ${response}
+    Run Keyword And Expect Error    RangerServiceException*       user_t.Create Hive Policy     dev_hive    test_policy_custom_2     test_db_custom_2   test_table_custom_2
+
+
+Regular User With Delegate-Admin Succeeds To Delete Policy Where Regular User Fails
+    [Documentation]         A regular user with delegated-admin succeeds to delete hive policy whereas a regular user w/o delegated-admin fails
+    Run Keyword And Expect Error    RangerServiceException*       user_f.Delete Hive Policy    dev_hive    test_policy_custom_1
+    ${response}             user_t.Delete Hive Policy    dev_hive    test_policy_custom_1
+    Log                     ${response}