Codebase list cinder-tempest-plugin / 70e909b
Merge "Add concurrency tests for Cinder operations" Zuul authored 8 months ago Gerrit Code Review committed 8 months ago
6 changed file(s) with 266 addition(s) and 1 deletion(s). Raw diff Collapse all Expand all
+27
-0
.zuul.yaml less more
44 check:
55 jobs:
66 - cinder-tempest-plugin-lvm-multiattach
7 - cinder-tempest-plugin-lvm-concurrency-tests
78 - cinder-tempest-plugin-lvm-lio-barbican
89 - cinder-tempest-plugin-lvm-lio-barbican-centos-9-stream:
910 voting: false
9192 timeout: 10800
9293
9394 - job:
95 name: cinder-tempest-plugin-lvm-concurrency-tests
96 description: |
97 This job runs Cinder concurrency scenario tests from the cinder-tempest-plugin.
98 These tests involve parallel operations on volumes (e.g., backup creation, attachment),
99 which can put stress on system resources.
100
101 To avoid hitting resource limits, `tempest_concurrency` is set to 1, ensuring that
102 the tests themselves run in serial even though each test performs concurrent actions internally.
103 parent: devstack-tempest
104 required-projects:
105 - opendev.org/openstack/tempest
106 - opendev.org/openstack/cinder-tempest-plugin
107 - opendev.org/openstack/cinder
108 vars:
109 tempest_concurrency: 1
110 tox_envlist: all
111 tempest_test_regex: 'cinder_tempest_plugin.scenario.test_volume_concurrency'
112 tempest_plugins:
113 - cinder-tempest-plugin
114 devstack_local_conf:
115 test-config:
116 $TEMPEST_CONFIG:
117 volume-feature-enabled:
118 concurrency_tests: True
119
120 - job:
94121 name: cinder-tempest-plugin-lvm-barbican-base-abstract
95122 description: |
96123 This is a base job for lvm with lio & tgt targets
+0
-0
cinder_tempest_plugin/common/__init__.py less more
(New empty file)
+55
-0
cinder_tempest_plugin/common/concurrency.py less more
0 # Copyright 2025 Red Hat, Inc.
1 # All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14
15 import multiprocessing
16
17 from tempest import config
18
19 CONF = config.CONF
20
21
22 def run_concurrent_tasks(target, **kwargs):
23 """Run a target function concurrently using multiprocessing."""
24 manager = multiprocessing.Manager()
25 resource_ids = manager.list()
26 # To capture exceptions
27 errors = manager.list()
28 resource_count = CONF.volume.concurrent_resource_count
29
30 def wrapped_target(index, resource_ids, **kwargs):
31 try:
32 target(index, resource_ids, **kwargs)
33 except Exception as e:
34 errors.append(f"Worker {index} failed: {str(e)}")
35
36 processes = []
37 for i in range(resource_count):
38 p = multiprocessing.Process(
39 target=wrapped_target,
40 args=(i, resource_ids),
41 kwargs=kwargs
42 )
43 processes.append(p)
44 p.start()
45
46 for p in processes:
47 p.join()
48
49 if errors:
50 error_msg = "\n".join(errors)
51 raise RuntimeError(
52 f"One or more concurrent tasks failed:\n{error_msg}")
53
54 return list(resource_ids)
+10
-1
cinder_tempest_plugin/config.py less more
2929 '`volume_image_dep_tests` '
3030 'in cinder-tempest-plugin is deprecated.Alternatively '
3131 '`CONF.volume_feature_enabled.enable_volume_image_dep_tests` '
32 'can be used for dependency tests.')
32 'can be used for dependency tests.'),
33 cfg.BoolOpt('concurrency_tests',
34 default=False,
35 help='Enable or disable running concurrency tests.'),
3336 ]
3437
3538 # The barbican service is discovered by config_tempest [1], and will appear
4346 default=False,
4447 help="Whether or not barbican is expected to be available"),
4548 ]
49
50 concurrency_option = [
51 cfg.IntOpt('concurrent_resource_count',
52 default=5,
53 help='Number of resources to create concurrently.'),
54 ]
+4
-0
cinder_tempest_plugin/plugin.py less more
4646 config.register_opt_group(conf, config.volume_feature_group,
4747 project_config.cinder_option)
4848
49 config.register_opt_group(conf, config.volume_group,
50 project_config.concurrency_option)
51
4952 # Define the 'barbican' service_available option, but only if the
5053 # barbican_tempest_plugin isn't present. It also defines the option,
5154 # and we need to avoid a duplicate option registration.
6164 """
6265 opt_lists = [
6366 (config.volume_feature_group.name, project_config.cinder_option),
67 (config.volume_group.name, project_config.concurrency_option),
6468 ]
6569
6670 if 'barbican_tempest_plugin' not in sys.modules:
+170
-0
cinder_tempest_plugin/scenario/test_volume_concurrency.py less more
0 # Copyright 2025 Red Hat, Inc.
1 # All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14
15 from tempest.common import utils
16 from tempest.common import waiters
17 from tempest import config
18 from tempest.lib import decorators
19
20 from cinder_tempest_plugin.common import concurrency
21 from cinder_tempest_plugin.scenario import manager
22
23 CONF = config.CONF
24
25
26 class ConcurrentVolumeActionsTest(manager.ScenarioTest):
27
28 @classmethod
29 def skip_checks(cls):
30 super(ConcurrentVolumeActionsTest, cls).skip_checks()
31 if not CONF.volume_feature_enabled.concurrency_tests:
32 raise cls.skipException(
33 "Concurrency tests are disabled.")
34
35 def _resource_create(self, index, resource_ids, create_func,
36 resource_id_key='id', **kwargs):
37 """Generic resource creation logic.
38
39 Handles both single and indexed resource creation.
40 If any list-type arguments are passed (e.g., volume_ids),
41 they are indexed using `index`.
42 """
43
44 # Prepare arguments, indexing into lists if necessary
45 adjusted_kwargs = {}
46 for key, value in kwargs.items():
47 if isinstance(value, list):
48 # For list arguments, pick the value by index
49 adjusted_kwargs[key] = value[index]
50 else:
51 adjusted_kwargs[key] = value
52
53 resource = create_func(**adjusted_kwargs)
54 resource_ids.append(resource[resource_id_key])
55
56 def _attach_volume_action(self, index, resource_ids, server_id,
57 volume_ids):
58 """Attach the given volume to the server."""
59 volume_id = volume_ids[index]
60 self.servers_client.attach_volume(
61 server_id, volumeId=volume_id, device=None)
62 waiters.wait_for_volume_resource_status(
63 self.volumes_client, volume_id, 'in-use')
64 resource_ids.append((server_id, volume_id))
65
66 def _cleanup_resources(self, resource_ids, delete_func, wait_func):
67 """Delete and wait for resource cleanup."""
68 for res_id in resource_ids:
69 delete_func(res_id)
70 wait_func(res_id)
71
72 @utils.services('volume')
73 @decorators.idempotent_id('ceb4f3c2-b2a4-48f9-82a8-3d32cdb5b375')
74 def test_create_volumes(self):
75 """Test parallel volume creation."""
76 volume_ids = concurrency.run_concurrent_tasks(
77 self._resource_create,
78 create_func=self.create_volume,
79 )
80
81 self._cleanup_resources(volume_ids,
82 self.volumes_client.delete_volume,
83 self.volumes_client.wait_for_resource_deletion)
84
85 @utils.services('volume')
86 @decorators.idempotent_id('6aa893a6-dfd0-4a0b-ae15-2fb24342e48d')
87 def test_create_snapshots(self):
88 """Test parallel snapshot creation from a single volume."""
89 volume = self.create_volume()
90
91 snapshot_ids = concurrency.run_concurrent_tasks(
92 self._resource_create,
93 create_func=self.create_volume_snapshot,
94 volume_id=volume['id']
95 )
96
97 self._cleanup_resources(
98 snapshot_ids,
99 self.snapshots_client.delete_snapshot,
100 self.snapshots_client.wait_for_resource_deletion)
101
102 @utils.services('compute', 'volume')
103 @decorators.idempotent_id('4c038386-00b0-4a6d-a612-48a4e0a96fa6')
104 def test_attach_volumes_to_server(self):
105 """Test parallel volume attachment to a server."""
106 server = self.create_server(wait_until='ACTIVE')
107 server_id = server['id']
108
109 volume_ids = concurrency.run_concurrent_tasks(
110 self._resource_create,
111 create_func=self.create_volume
112 )
113
114 attach_ids = concurrency.run_concurrent_tasks(
115 self._attach_volume_action,
116 server_id=server_id,
117 volume_ids=volume_ids
118 )
119
120 for server_id, volume_id in attach_ids:
121 self.servers_client.detach_volume(server_id, volume_id)
122 waiters.wait_for_volume_resource_status(self.volumes_client,
123 volume_id, 'available')
124
125 self._cleanup_resources(volume_ids,
126 self.volumes_client.delete_volume,
127 self.volumes_client.wait_for_resource_deletion)
128
129 @utils.services('volume')
130 @decorators.idempotent_id('01f66de8-b217-4588-ab7f-e707d1931156')
131 def test_create_backups_and_restores(self):
132 """Test parallel backup creation and restore from multiple volumes."""
133
134 # Step 1: Create volumes in concurrency
135 volume_ids = concurrency.run_concurrent_tasks(
136 self._resource_create,
137 create_func=self.create_volume
138 )
139
140 # Step 2: Create backups in concurrency
141 backup_ids = concurrency.run_concurrent_tasks(
142 self._resource_create,
143 create_func=self.create_backup,
144 volume_id=volume_ids
145 )
146
147 # Step 3: Restore backups in concurrency
148 restored_vol_ids = concurrency.run_concurrent_tasks(
149 self._resource_create,
150 create_func=self.restore_backup,
151 resource_id_key='volume_id',
152 backup_id=backup_ids
153 )
154
155 # Step 4: Cleanup all resources
156 self._cleanup_resources(
157 backup_ids,
158 self.backups_client.delete_backup,
159 self.backups_client.wait_for_resource_deletion)
160
161 self._cleanup_resources(
162 volume_ids,
163 self.volumes_client.delete_volume,
164 self.volumes_client.wait_for_resource_deletion)
165
166 self._cleanup_resources(
167 restored_vol_ids,
168 self.volumes_client.delete_volume,
169 self.volumes_client.wait_for_resource_deletion)