Codebase list cinder-tempest-plugin / ec9aee2
Add concurrency tests for Cinder operations This patch introduces parallel resource creation and volume attach scenarios using Python's multiprocessing module. It enables testing concurrent operations on volumes and snapshots including attaching multiple volumes to a single server. Two new configuration options were added: - `concurrency_tests`: Enables or disables concurrency test execution - `concurrent_resource_count`: Number of resources to create concurrently Additionally, a new CI job was added: cinder-tempest-plugin-lvm-concurrency-tests, which is configured to: - Run the concurrency tests in serial (tempest_concurrency: 1) - Avoid hitting environment resource limits by limiting the number of simultaneously running tests - Isolate concurrency logic validation from unrelated Tempest tests Signed-off-by: Liron Kuchlani <lkuchlan@redhat.com> Change-Id: Iebbee32966fe91f8a586cdb84e02981101a91968 lkuchlan authored a year ago Rajat Dhasmana committed 10 months ago
6 changed file(s) with 266 addition(s) and 1 deletion(s). Raw diff Collapse all Expand all
+27
-0
.zuul.yaml less more
44 check:
55 jobs:
66 - cinder-tempest-plugin-lvm-multiattach
7 - cinder-tempest-plugin-lvm-concurrency-tests
78 - cinder-tempest-plugin-lvm-lio-barbican
89 - cinder-tempest-plugin-lvm-lio-barbican-centos-9-stream:
910 voting: false
9192 timeout: 10800
9293
9394 - job:
95 name: cinder-tempest-plugin-lvm-concurrency-tests
96 description: |
97 This job runs Cinder concurrency scenario tests from the cinder-tempest-plugin.
98 These tests involve parallel operations on volumes (e.g., backup creation, attachment),
99 which can put stress on system resources.
100
101 To avoid hitting resource limits, `tempest_concurrency` is set to 1, ensuring that
102 the tests themselves run in serial even though each test performs concurrent actions internally.
103 parent: devstack-tempest
104 required-projects:
105 - opendev.org/openstack/tempest
106 - opendev.org/openstack/cinder-tempest-plugin
107 - opendev.org/openstack/cinder
108 vars:
109 tempest_concurrency: 1
110 tox_envlist: all
111 tempest_test_regex: 'cinder_tempest_plugin.scenario.test_volume_concurrency'
112 tempest_plugins:
113 - cinder-tempest-plugin
114 devstack_local_conf:
115 test-config:
116 $TEMPEST_CONFIG:
117 volume-feature-enabled:
118 concurrency_tests: True
119
120 - job:
94121 name: cinder-tempest-plugin-lvm-barbican-base-abstract
95122 description: |
96123 This is a base job for lvm with lio & tgt targets
+0
-0
cinder_tempest_plugin/common/__init__.py less more
(New empty file)
+55
-0
cinder_tempest_plugin/common/concurrency.py less more
0 # Copyright 2025 Red Hat, Inc.
1 # All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14
15 import multiprocessing
16
17 from tempest import config
18
19 CONF = config.CONF
20
21
22 def run_concurrent_tasks(target, **kwargs):
23 """Run a target function concurrently using multiprocessing."""
24 manager = multiprocessing.Manager()
25 resource_ids = manager.list()
26 # To capture exceptions
27 errors = manager.list()
28 resource_count = CONF.volume.concurrent_resource_count
29
30 def wrapped_target(index, resource_ids, **kwargs):
31 try:
32 target(index, resource_ids, **kwargs)
33 except Exception as e:
34 errors.append(f"Worker {index} failed: {str(e)}")
35
36 processes = []
37 for i in range(resource_count):
38 p = multiprocessing.Process(
39 target=wrapped_target,
40 args=(i, resource_ids),
41 kwargs=kwargs
42 )
43 processes.append(p)
44 p.start()
45
46 for p in processes:
47 p.join()
48
49 if errors:
50 error_msg = "\n".join(errors)
51 raise RuntimeError(
52 f"One or more concurrent tasks failed:\n{error_msg}")
53
54 return list(resource_ids)
+10
-1
cinder_tempest_plugin/config.py less more
2929 '`volume_image_dep_tests` '
3030 'in cinder-tempest-plugin is deprecated.Alternatively '
3131 '`CONF.volume_feature_enabled.enable_volume_image_dep_tests` '
32 'can be used for dependency tests.')
32 'can be used for dependency tests.'),
33 cfg.BoolOpt('concurrency_tests',
34 default=False,
35 help='Enable or disable running concurrency tests.'),
3336 ]
3437
3538 # The barbican service is discovered by config_tempest [1], and will appear
4346 default=False,
4447 help="Whether or not barbican is expected to be available"),
4548 ]
49
50 concurrency_option = [
51 cfg.IntOpt('concurrent_resource_count',
52 default=5,
53 help='Number of resources to create concurrently.'),
54 ]
+4
-0
cinder_tempest_plugin/plugin.py less more
4646 config.register_opt_group(conf, config.volume_feature_group,
4747 project_config.cinder_option)
4848
49 config.register_opt_group(conf, config.volume_group,
50 project_config.concurrency_option)
51
4952 # Define the 'barbican' service_available option, but only if the
5053 # barbican_tempest_plugin isn't present. It also defines the option,
5154 # and we need to avoid a duplicate option registration.
6164 """
6265 opt_lists = [
6366 (config.volume_feature_group.name, project_config.cinder_option),
67 (config.volume_group.name, project_config.concurrency_option),
6468 ]
6569
6670 if 'barbican_tempest_plugin' not in sys.modules:
+170
-0
cinder_tempest_plugin/scenario/test_volume_concurrency.py less more
0 # Copyright 2025 Red Hat, Inc.
1 # All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14
15 from tempest.common import utils
16 from tempest.common import waiters
17 from tempest import config
18 from tempest.lib import decorators
19
20 from cinder_tempest_plugin.common import concurrency
21 from cinder_tempest_plugin.scenario import manager
22
23 CONF = config.CONF
24
25
26 class ConcurrentVolumeActionsTest(manager.ScenarioTest):
27
28 @classmethod
29 def skip_checks(cls):
30 super(ConcurrentVolumeActionsTest, cls).skip_checks()
31 if not CONF.volume_feature_enabled.concurrency_tests:
32 raise cls.skipException(
33 "Concurrency tests are disabled.")
34
35 def _resource_create(self, index, resource_ids, create_func,
36 resource_id_key='id', **kwargs):
37 """Generic resource creation logic.
38
39 Handles both single and indexed resource creation.
40 If any list-type arguments are passed (e.g., volume_ids),
41 they are indexed using `index`.
42 """
43
44 # Prepare arguments, indexing into lists if necessary
45 adjusted_kwargs = {}
46 for key, value in kwargs.items():
47 if isinstance(value, list):
48 # For list arguments, pick the value by index
49 adjusted_kwargs[key] = value[index]
50 else:
51 adjusted_kwargs[key] = value
52
53 resource = create_func(**adjusted_kwargs)
54 resource_ids.append(resource[resource_id_key])
55
56 def _attach_volume_action(self, index, resource_ids, server_id,
57 volume_ids):
58 """Attach the given volume to the server."""
59 volume_id = volume_ids[index]
60 self.servers_client.attach_volume(
61 server_id, volumeId=volume_id, device=None)
62 waiters.wait_for_volume_resource_status(
63 self.volumes_client, volume_id, 'in-use')
64 resource_ids.append((server_id, volume_id))
65
66 def _cleanup_resources(self, resource_ids, delete_func, wait_func):
67 """Delete and wait for resource cleanup."""
68 for res_id in resource_ids:
69 delete_func(res_id)
70 wait_func(res_id)
71
72 @utils.services('volume')
73 @decorators.idempotent_id('ceb4f3c2-b2a4-48f9-82a8-3d32cdb5b375')
74 def test_create_volumes(self):
75 """Test parallel volume creation."""
76 volume_ids = concurrency.run_concurrent_tasks(
77 self._resource_create,
78 create_func=self.create_volume,
79 )
80
81 self._cleanup_resources(volume_ids,
82 self.volumes_client.delete_volume,
83 self.volumes_client.wait_for_resource_deletion)
84
85 @utils.services('volume')
86 @decorators.idempotent_id('6aa893a6-dfd0-4a0b-ae15-2fb24342e48d')
87 def test_create_snapshots(self):
88 """Test parallel snapshot creation from a single volume."""
89 volume = self.create_volume()
90
91 snapshot_ids = concurrency.run_concurrent_tasks(
92 self._resource_create,
93 create_func=self.create_volume_snapshot,
94 volume_id=volume['id']
95 )
96
97 self._cleanup_resources(
98 snapshot_ids,
99 self.snapshots_client.delete_snapshot,
100 self.snapshots_client.wait_for_resource_deletion)
101
102 @utils.services('compute', 'volume')
103 @decorators.idempotent_id('4c038386-00b0-4a6d-a612-48a4e0a96fa6')
104 def test_attach_volumes_to_server(self):
105 """Test parallel volume attachment to a server."""
106 server = self.create_server(wait_until='ACTIVE')
107 server_id = server['id']
108
109 volume_ids = concurrency.run_concurrent_tasks(
110 self._resource_create,
111 create_func=self.create_volume
112 )
113
114 attach_ids = concurrency.run_concurrent_tasks(
115 self._attach_volume_action,
116 server_id=server_id,
117 volume_ids=volume_ids
118 )
119
120 for server_id, volume_id in attach_ids:
121 self.servers_client.detach_volume(server_id, volume_id)
122 waiters.wait_for_volume_resource_status(self.volumes_client,
123 volume_id, 'available')
124
125 self._cleanup_resources(volume_ids,
126 self.volumes_client.delete_volume,
127 self.volumes_client.wait_for_resource_deletion)
128
129 @utils.services('volume')
130 @decorators.idempotent_id('01f66de8-b217-4588-ab7f-e707d1931156')
131 def test_create_backups_and_restores(self):
132 """Test parallel backup creation and restore from multiple volumes."""
133
134 # Step 1: Create volumes in concurrency
135 volume_ids = concurrency.run_concurrent_tasks(
136 self._resource_create,
137 create_func=self.create_volume
138 )
139
140 # Step 2: Create backups in concurrency
141 backup_ids = concurrency.run_concurrent_tasks(
142 self._resource_create,
143 create_func=self.create_backup,
144 volume_id=volume_ids
145 )
146
147 # Step 3: Restore backups in concurrency
148 restored_vol_ids = concurrency.run_concurrent_tasks(
149 self._resource_create,
150 create_func=self.restore_backup,
151 resource_id_key='volume_id',
152 backup_id=backup_ids
153 )
154
155 # Step 4: Cleanup all resources
156 self._cleanup_resources(
157 backup_ids,
158 self.backups_client.delete_backup,
159 self.backups_client.wait_for_resource_deletion)
160
161 self._cleanup_resources(
162 volume_ids,
163 self.volumes_client.delete_volume,
164 self.volumes_client.wait_for_resource_deletion)
165
166 self._cleanup_resources(
167 restored_vol_ids,
168 self.volumes_client.delete_volume,
169 self.volumes_client.wait_for_resource_deletion)