diff --git a/.zuul.yaml b/.zuul.yaml index c7cc678..c544d2a 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -5,6 +5,7 @@ check: jobs: - cinder-tempest-plugin-lvm-multiattach + - cinder-tempest-plugin-lvm-concurrency-tests - cinder-tempest-plugin-lvm-lio-barbican - cinder-tempest-plugin-lvm-lio-barbican-centos-9-stream: voting: false @@ -92,6 +93,32 @@ timeout: 10800 - job: + name: cinder-tempest-plugin-lvm-concurrency-tests + description: | + This job runs Cinder concurrency scenario tests from the cinder-tempest-plugin. + These tests involve parallel operations on volumes (e.g., backup creation, attachment), + which can put stress on system resources. + + To avoid hitting resource limits, `tempest_concurrency` is set to 1, ensuring that + the tests themselves run in serial even though each test performs concurrent actions internally. + parent: devstack-tempest + required-projects: + - opendev.org/openstack/tempest + - opendev.org/openstack/cinder-tempest-plugin + - opendev.org/openstack/cinder + vars: + tempest_concurrency: 1 + tox_envlist: all + tempest_test_regex: 'cinder_tempest_plugin.scenario.test_volume_concurrency' + tempest_plugins: + - cinder-tempest-plugin + devstack_local_conf: + test-config: + $TEMPEST_CONFIG: + volume-feature-enabled: + concurrency_tests: True + +- job: name: cinder-tempest-plugin-lvm-barbican-base-abstract description: | This is a base job for lvm with lio & tgt targets diff --git a/cinder_tempest_plugin/common/__init__.py b/cinder_tempest_plugin/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cinder_tempest_plugin/common/concurrency.py b/cinder_tempest_plugin/common/concurrency.py new file mode 100644 index 0000000..0374b12 --- /dev/null +++ b/cinder_tempest_plugin/common/concurrency.py @@ -0,0 +1,55 @@ +# Copyright 2025 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import multiprocessing + +from tempest import config + +CONF = config.CONF + + +def run_concurrent_tasks(target, **kwargs): + """Run a target function concurrently using multiprocessing.""" + manager = multiprocessing.Manager() + resource_ids = manager.list() + # To capture exceptions + errors = manager.list() + resource_count = CONF.volume.concurrent_resource_count + + def wrapped_target(index, resource_ids, **kwargs): + try: + target(index, resource_ids, **kwargs) + except Exception as e: + errors.append(f"Worker {index} failed: {str(e)}") + + processes = [] + for i in range(resource_count): + p = multiprocessing.Process( + target=wrapped_target, + args=(i, resource_ids), + kwargs=kwargs + ) + processes.append(p) + p.start() + + for p in processes: + p.join() + + if errors: + error_msg = "\n".join(errors) + raise RuntimeError( + f"One or more concurrent tasks failed:\n{error_msg}") + + return list(resource_ids) diff --git a/cinder_tempest_plugin/config.py b/cinder_tempest_plugin/config.py index 3d5eb0e..969451e 100644 --- a/cinder_tempest_plugin/config.py +++ b/cinder_tempest_plugin/config.py @@ -30,7 +30,10 @@ '`volume_image_dep_tests` ' 'in cinder-tempest-plugin is deprecated.Alternatively ' '`CONF.volume_feature_enabled.enable_volume_image_dep_tests` ' - 'can be used for dependency tests.') + 'can be used for dependency tests.'), + cfg.BoolOpt('concurrency_tests', + default=False, + help='Enable or disable running concurrency tests.'), ] # The barbican service is discovered by config_tempest [1], and will appear @@ -44,3 +47,9 @@ default=False, help="Whether or not barbican is expected to be available"), ] + +concurrency_option = [ + cfg.IntOpt('concurrent_resource_count', + default=5, + help='Number of resources to create concurrently.'), +] diff --git a/cinder_tempest_plugin/plugin.py b/cinder_tempest_plugin/plugin.py index 79c835c..e9583cd 100644 --- a/cinder_tempest_plugin/plugin.py +++ b/cinder_tempest_plugin/plugin.py @@ -47,6 +47,9 @@ config.register_opt_group(conf, config.volume_feature_group, project_config.cinder_option) + config.register_opt_group(conf, config.volume_group, + project_config.concurrency_option) + # Define the 'barbican' service_available option, but only if the # barbican_tempest_plugin isn't present. It also defines the option, # and we need to avoid a duplicate option registration. @@ -62,6 +65,7 @@ """ opt_lists = [ (config.volume_feature_group.name, project_config.cinder_option), + (config.volume_group.name, project_config.concurrency_option), ] if 'barbican_tempest_plugin' not in sys.modules: diff --git a/cinder_tempest_plugin/scenario/test_volume_concurrency.py b/cinder_tempest_plugin/scenario/test_volume_concurrency.py new file mode 100644 index 0000000..3a27174 --- /dev/null +++ b/cinder_tempest_plugin/scenario/test_volume_concurrency.py @@ -0,0 +1,170 @@ +# Copyright 2025 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tempest.common import utils +from tempest.common import waiters +from tempest import config +from tempest.lib import decorators + +from cinder_tempest_plugin.common import concurrency +from cinder_tempest_plugin.scenario import manager + +CONF = config.CONF + + +class ConcurrentVolumeActionsTest(manager.ScenarioTest): + + @classmethod + def skip_checks(cls): + super(ConcurrentVolumeActionsTest, cls).skip_checks() + if not CONF.volume_feature_enabled.concurrency_tests: + raise cls.skipException( + "Concurrency tests are disabled.") + + def _resource_create(self, index, resource_ids, create_func, + resource_id_key='id', **kwargs): + """Generic resource creation logic. + + Handles both single and indexed resource creation. + If any list-type arguments are passed (e.g., volume_ids), + they are indexed using `index`. + """ + + # Prepare arguments, indexing into lists if necessary + adjusted_kwargs = {} + for key, value in kwargs.items(): + if isinstance(value, list): + # For list arguments, pick the value by index + adjusted_kwargs[key] = value[index] + else: + adjusted_kwargs[key] = value + + resource = create_func(**adjusted_kwargs) + resource_ids.append(resource[resource_id_key]) + + def _attach_volume_action(self, index, resource_ids, server_id, + volume_ids): + """Attach the given volume to the server.""" + volume_id = volume_ids[index] + self.servers_client.attach_volume( + server_id, volumeId=volume_id, device=None) + waiters.wait_for_volume_resource_status( + self.volumes_client, volume_id, 'in-use') + resource_ids.append((server_id, volume_id)) + + def _cleanup_resources(self, resource_ids, delete_func, wait_func): + """Delete and wait for resource cleanup.""" + for res_id in resource_ids: + delete_func(res_id) + wait_func(res_id) + + @utils.services('volume') + @decorators.idempotent_id('ceb4f3c2-b2a4-48f9-82a8-3d32cdb5b375') + def test_create_volumes(self): + """Test parallel volume creation.""" + volume_ids = concurrency.run_concurrent_tasks( + self._resource_create, + create_func=self.create_volume, + ) + + self._cleanup_resources(volume_ids, + self.volumes_client.delete_volume, + self.volumes_client.wait_for_resource_deletion) + + @utils.services('volume') + @decorators.idempotent_id('6aa893a6-dfd0-4a0b-ae15-2fb24342e48d') + def test_create_snapshots(self): + """Test parallel snapshot creation from a single volume.""" + volume = self.create_volume() + + snapshot_ids = concurrency.run_concurrent_tasks( + self._resource_create, + create_func=self.create_volume_snapshot, + volume_id=volume['id'] + ) + + self._cleanup_resources( + snapshot_ids, + self.snapshots_client.delete_snapshot, + self.snapshots_client.wait_for_resource_deletion) + + @utils.services('compute', 'volume') + @decorators.idempotent_id('4c038386-00b0-4a6d-a612-48a4e0a96fa6') + def test_attach_volumes_to_server(self): + """Test parallel volume attachment to a server.""" + server = self.create_server(wait_until='ACTIVE') + server_id = server['id'] + + volume_ids = concurrency.run_concurrent_tasks( + self._resource_create, + create_func=self.create_volume + ) + + attach_ids = concurrency.run_concurrent_tasks( + self._attach_volume_action, + server_id=server_id, + volume_ids=volume_ids + ) + + for server_id, volume_id in attach_ids: + self.servers_client.detach_volume(server_id, volume_id) + waiters.wait_for_volume_resource_status(self.volumes_client, + volume_id, 'available') + + self._cleanup_resources(volume_ids, + self.volumes_client.delete_volume, + self.volumes_client.wait_for_resource_deletion) + + @utils.services('volume') + @decorators.idempotent_id('01f66de8-b217-4588-ab7f-e707d1931156') + def test_create_backups_and_restores(self): + """Test parallel backup creation and restore from multiple volumes.""" + + # Step 1: Create volumes in concurrency + volume_ids = concurrency.run_concurrent_tasks( + self._resource_create, + create_func=self.create_volume + ) + + # Step 2: Create backups in concurrency + backup_ids = concurrency.run_concurrent_tasks( + self._resource_create, + create_func=self.create_backup, + volume_id=volume_ids + ) + + # Step 3: Restore backups in concurrency + restored_vol_ids = concurrency.run_concurrent_tasks( + self._resource_create, + create_func=self.restore_backup, + resource_id_key='volume_id', + backup_id=backup_ids + ) + + # Step 4: Cleanup all resources + self._cleanup_resources( + backup_ids, + self.backups_client.delete_backup, + self.backups_client.wait_for_resource_deletion) + + self._cleanup_resources( + volume_ids, + self.volumes_client.delete_volume, + self.volumes_client.wait_for_resource_deletion) + + self._cleanup_resources( + restored_vol_ids, + self.volumes_client.delete_volume, + self.volumes_client.wait_for_resource_deletion)