CephRGWTest: Add integration test for multisite sync policies feature

Signed-off-by: Ionut Balutoiu <[email protected]>

zaza/openstack/charm_tests/ceph/tests.py

@@ -1066,6 +1066,230 @@ def test_003_object_storage_and_secondary_block(self):
         zaza_model.block_until_unit_wl_status(self.secondary_rgw_unit,
                                               'active')
 
+    def test_004_multisite_directional_sync_policy(self):
+        """Verify Multisite Directional Sync Policy."""
+        # Skip multisite tests if not compatible with bundle.
+        if not self.multisite:
+            logging.info('Skipping multisite sync policy verification')
+            return
+
+        container_name = 'zaza-container'
+        primary_obj_name = 'primary-testfile'
+        primary_obj_data = 'Primary test data'
+        secondary_directional_obj_name = 'secondary-directional-testfile'
+        secondary_directional_obj_data = 'Secondary directional test data'
+        secondary_symmetrical_obj_name = 'secondary-symmetrical-testfile'
+        secondary_symmetrical_obj_data = 'Secondary symmetrical test data'
+
+        logging.info('Verifying multisite directional sync policy')
+
+        # Set default sync policy to "allowed", which allows buckets to sync,
+        # but the sync is disabled by default in the zone group. Also, set the
+        # secondary zone sync policy flow type policy to "directional".
+        zaza_model.set_application_config(
+            self.primary_rgw_app,
+            {
+                "sync-policy-state": "allowed",
+            }
+        )
+        zaza_model.set_application_config(
+            self.secondary_rgw_app,
+            {
+                "sync-policy-flow-type": "directional",
+            }
+        )
+        zaza_model.wait_for_unit_idle(self.secondary_rgw_unit)
+        zaza_model.wait_for_unit_idle(self.primary_rgw_unit)
+
+        # Setup multisite relation.
+        multisite_relation = zaza_model.get_relation_id(
+            self.primary_rgw_app, self.secondary_rgw_app,
+            remote_interface_name='secondary'
+        )
+        if multisite_relation is None:
+            logging.info('Configuring Multisite')
+            self.configure_rgw_apps_for_multisite()
+            zaza_model.add_relation(
+                self.primary_rgw_app,
+                self.primary_rgw_app + ":primary",
+                self.secondary_rgw_app + ":secondary"
+            )
+            zaza_model.block_until_unit_wl_status(
+                self.secondary_rgw_unit, "waiting"
+            )
+        zaza_model.block_until_unit_wl_status(
+            self.primary_rgw_unit, "active"
+        )
+        zaza_model.block_until_unit_wl_status(
+            self.secondary_rgw_unit, "active"
+        )
+        zaza_model.wait_for_unit_idle(self.secondary_rgw_unit)
+        zaza_model.wait_for_unit_idle(self.primary_rgw_unit)
+
+        logging.info('Waiting for Data and Metadata to Synchronize')
+        # NOTE: We only check the secondary zone, because the sync policy flow
+        # type is set to "directional" between the primary and secondary zones.
+        self.wait_for_status(self.secondary_rgw_app, is_primary=False)
+
+        # Create bucket on primary RGW zone.
+        logging.info('Creating bucket on primary zone')
+        primary_endpoint = self.get_rgw_endpoint(self.primary_rgw_unit)
+        self.assertNotEqual(primary_endpoint, None)
+
+        access_key, secret_key = self.get_client_keys()
+        primary_client = boto3.resource("s3",
+                                        verify=False,
+                                        endpoint_url=primary_endpoint,
+                                        aws_access_key_id=access_key,
+                                        aws_secret_access_key=secret_key)
+        primary_client.Bucket(container_name).create()
+
+        # Enable sync on the bucket.
+        logging.info('Enabling sync on the bucket from the primary zone')
+        zaza_model.run_action_on_leader(
+            self.primary_rgw_app,
+            'enable-buckets-sync',
+            action_params={
+                'buckets': container_name,
+            },
+            raise_on_failure=True,
+        )
+
+        # Check that sync cannot be enabled using secondary Juju RGW app.
+        with self.assertRaises(zaza_model.ActionFailed):
+            zaza_model.run_action_on_leader(
+                self.secondary_rgw_app,
+                'enable-buckets-sync',
+                action_params={
+                    'buckets': container_name,
+                },
+                raise_on_failure=True,
+            )
+
+        logging.info('Waiting for Data and Metadata to Synchronize')
+        self.wait_for_status(self.secondary_rgw_app, is_primary=False)
+
+        # Perform IO on primary zone bucket.
+        logging.info('Performing IO on primary zone bucket')
+        primary_object = primary_client.Object(
+            container_name,
+            primary_obj_name
+        )
+        primary_object.put(Body=primary_obj_data)
+
+        # Verify that the object is replicated to the secondary zone.
+        logging.info('Verifying that the object is replicated to the '
+                     'secondary zone')
+        secondary_endpoint = self.get_rgw_endpoint(self.secondary_rgw_unit)
+        self.assertNotEqual(secondary_endpoint, None)
+
+        secondary_client = boto3.resource("s3",
+                                          verify=False,
+                                          endpoint_url=secondary_endpoint,
+                                          aws_access_key_id=access_key,
+                                          aws_secret_access_key=secret_key)
+        secondary_data = self.fetch_rgw_object(
+            secondary_client,
+            container_name,
+            primary_obj_name
+        )
+        self.assertEqual(secondary_data, primary_obj_data)
+
+        # Write object to the secondary zone bucket, when the sync policy
+        # flow type is set to "directional" between the zones.
+        logging.info('Writing object to the secondary zone bucket, which '
+                     'should not be replicated to the primary zone')
+        secondary_object = secondary_client.Object(
+            container_name,
+            secondary_directional_obj_name
+        )
+        secondary_object.put(Body=secondary_directional_obj_data)
+
+        # Verify that the object is not replicated to the primary zone.
+        logging.info('Verifying that the object is not replicated to the '
+                     'primary zone')
+        with self.assertRaises(botocore.exceptions.ClientError):
+            self.fetch_rgw_object(
+                primary_client,
+                container_name,
+                secondary_directional_obj_name
+            )
+
+        logging.info('Setting sync policy flow to "symmetrical" on the '
+                     'secondary RGW zone')
+        zaza_model.set_application_config(
+            self.secondary_rgw_app,
+            {
+                "sync-policy-flow-type": "symmetrical",
+            }
+        )
+        zaza_model.wait_for_unit_idle(self.secondary_rgw_unit)
+        zaza_model.wait_for_unit_idle(self.primary_rgw_unit)
+
+        # Write another object to the secondary zone bucket.
+        logging.info('Writing another object to the secondary zone bucket.')
+        secondary_object = secondary_client.Object(
+            container_name,
+            secondary_symmetrical_obj_name
+        )
+        secondary_object.put(Body=secondary_symmetrical_obj_data)
+
+        logging.info('Waiting for Data and Metadata to Synchronize')
+        # NOTE: This time, we check both the primary and secondary zones,
+        # because the sync policy flow type is set to "symmetrical" between
+        # the zones.
+        self.wait_for_status(self.secondary_rgw_app, is_primary=False)
+        self.wait_for_status(self.primary_rgw_app, is_primary=True)
+
+        # Verify that all objects are replicated to the primary zone.
+        logging.info('Verifying that all objects are replicated to the '
+                     'primary zone (including older objects).')
+        test_cases = [
+            {
+                'obj_name': primary_obj_name,
+                'obj_data': primary_obj_data,
+            },
+            {
+                'obj_name': secondary_directional_obj_name,
+                'obj_data': secondary_directional_obj_data,
+            },
+            {
+                'obj_name': secondary_symmetrical_obj_name,
+                'obj_data': secondary_symmetrical_obj_data,
+            },
+        ]
+        for tc in test_cases:
+            logging.info('Verifying that object "{}" is replicated'.format(
+                tc['obj_name']))
+            primary_data = self.fetch_rgw_object(
+                primary_client,
+                container_name,
+                tc['obj_name']
+            )
+            self.assertEqual(primary_data, tc['obj_data'])
+
+        # Cleanup.
+        logging.info('Cleaning up buckets after test case')
+        self.purge_bucket(self.primary_rgw_app, container_name)
+        self.purge_bucket(self.secondary_rgw_app, container_name)
+
+        logging.info('Waiting for Data and Metadata to Synchronize')
+        self.wait_for_status(self.secondary_rgw_app, is_primary=False)
+        self.wait_for_status(self.primary_rgw_app, is_primary=True)
+
+        # Set multisite sync policy state to "enabled" on the primary RGW app.
+        # Paired with "symmetrical" sync policy flow on the secondary RGW app,
+        # this enables bidirectional sync between the zones (which is the
+        # default behaviour without multisite sync policies configured).
+        logging.info('Setting sync policy state to "enabled".')
+        zaza_model.set_application_config(
+            self.primary_rgw_app,
+            {
+                "sync-policy-state": "enabled",
+            }
+        )
+        zaza_model.wait_for_unit_idle(self.primary_rgw_unit)
+
     def test_100_migration_and_multisite_failover(self):
         """Perform multisite migration and verify failover."""
         container_name = 'zaza-container'