Merge "Improve snapshot data integrity test"
Zuul authored 4 years ago
Gerrit Code Review committed 4 years ago
| 13 | 13 | # under the License. |
| 14 | 14 | |
| 15 | 15 | from tempest.common import utils |
| 16 | from tempest import config | |
| 17 | from tempest.lib.common.utils import test_utils | |
| 16 | 18 | from tempest.lib import decorators |
| 19 | from tempest.lib import exceptions as lib_exc | |
| 17 | 20 | |
| 18 | 21 | from cinder_tempest_plugin.scenario import manager |
| 22 | ||
| 23 | CONF = config.CONF | |
| 19 | 24 | |
| 20 | 25 | |
| 21 | 26 | class SnapshotDataIntegrityTests(manager.ScenarioTest): |
| 25 | 30 | self.keypair = self.create_keypair() |
| 26 | 31 | self.security_group = self._create_security_group() |
| 27 | 32 | |
| 28 | def _get_file_md5(self, ip_address, filename, mount_path='/mnt', | |
| 29 | private_key=None, server=None): | |
| 30 | ssh_client = self.get_remote_client(ip_address, | |
| 31 | private_key=private_key, | |
| 32 | server=server) | |
| 33 | def _attached_volume_name( | |
| 34 | self, disks_list_before_attach, ip_address, private_key): | |
| 35 | ssh = self.get_remote_client(ip_address, private_key=private_key) | |
| 36 | ||
| 37 | def _wait_for_volume_available_on_system(): | |
| 38 | disks_list_after_attach = ssh.list_disks() | |
| 39 | return len(disks_list_after_attach) > len(disks_list_before_attach) | |
| 40 | ||
| 41 | if not test_utils.call_until_true(_wait_for_volume_available_on_system, | |
| 42 | CONF.compute.build_timeout, | |
| 43 | CONF.compute.build_interval): | |
| 44 | raise lib_exc.TimeoutException | |
| 45 | ||
| 46 | disks_list_after_attach = ssh.list_disks() | |
| 47 | volume_name = [item for item in disks_list_after_attach | |
| 48 | if item not in disks_list_before_attach][0] | |
| 49 | return volume_name | |
| 50 | ||
| 51 | def _get_file_md5(self, ip_address, filename, dev_name=None, | |
| 52 | mount_path='/mnt', private_key=None, server=None): | |
| 53 | ||
| 54 | ssh_client = self.get_remote_client(ip_address, | |
| 55 | private_key=private_key, | |
| 56 | server=server) | |
| 57 | if dev_name is not None: | |
| 58 | ssh_client.exec_command('sudo mount /dev/%s %s' % (dev_name, | |
| 59 | mount_path)) | |
| 33 | 60 | |
| 34 | 61 | md5_sum = ssh_client.exec_command( |
| 35 | 62 | 'sudo md5sum %s/%s|cut -c 1-32' % (mount_path, filename)) |
| 63 | if dev_name is not None: | |
| 64 | ssh_client.exec_command('sudo umount %s' % mount_path) | |
| 36 | 65 | return md5_sum |
| 37 | 66 | |
| 38 | def _count_files(self, ip_address, mount_path='/mnt', private_key=None, | |
| 39 | server=None): | |
| 40 | ssh_client = self.get_remote_client(ip_address, | |
| 41 | private_key=private_key, | |
| 42 | server=server) | |
| 67 | def _count_files(self, ip_address, dev_name=None, mount_path='/mnt', | |
| 68 | private_key=None, server=None): | |
| 69 | ssh_client = self.get_remote_client(ip_address, | |
| 70 | private_key=private_key, | |
| 71 | server=server) | |
| 72 | if dev_name is not None: | |
| 73 | ssh_client.exec_command('sudo mount /dev/%s %s' % (dev_name, | |
| 74 | mount_path)) | |
| 43 | 75 | count = ssh_client.exec_command('sudo ls -l %s | wc -l' % mount_path) |
| 44 | return int(count) - 1 | |
| 45 | ||
| 46 | def _launch_instance_from_snapshot(self, snap): | |
| 47 | volume_snap = self.create_volume(snapshot_id=snap['id'], | |
| 48 | size=snap['size']) | |
| 49 | ||
| 50 | server_snap = self.boot_instance_from_resource( | |
| 51 | source_id=volume_snap['id'], | |
| 52 | source_type='volume', | |
| 53 | keypair=self.keypair, | |
| 54 | security_group=self.security_group) | |
| 55 | ||
| 56 | return server_snap | |
| 57 | ||
| 58 | def create_md5_new_file(self, ip_address, filename, mount_path='/mnt', | |
| 59 | private_key=None, server=None): | |
| 60 | ssh_client = self.get_remote_client(ip_address, | |
| 61 | private_key=private_key, | |
| 62 | server=server) | |
| 63 | ||
| 76 | if dev_name is not None: | |
| 77 | ssh_client.exec_command('sudo umount %s' % mount_path) | |
| 78 | # We subtract 2 from the count since `wc -l` also includes the count | |
| 79 | # of new line character and while creating the filesystem, a | |
| 80 | # lost+found folder is also created | |
| 81 | return int(count) - 2 | |
| 82 | ||
| 83 | def _make_fs(self, ip_address, private_key, server, dev_name, fs='ext4'): | |
| 84 | ssh_client = self.get_remote_client(ip_address, | |
| 85 | private_key=private_key, | |
| 86 | server=server) | |
| 87 | ||
| 88 | ssh_client.make_fs(dev_name, fs=fs) | |
| 89 | ||
| 90 | def create_md5_new_file(self, ip_address, filename, dev_name=None, | |
| 91 | mount_path='/mnt', private_key=None, server=None): | |
| 92 | ssh_client = self.get_remote_client(ip_address, | |
| 93 | private_key=private_key, | |
| 94 | server=server) | |
| 95 | ||
| 96 | if dev_name is not None: | |
| 97 | ssh_client.exec_command('sudo mount /dev/%s %s' % (dev_name, | |
| 98 | mount_path)) | |
| 64 | 99 | ssh_client.exec_command( |
| 65 | 100 | 'sudo dd bs=1024 count=100 if=/dev/urandom of=/%s/%s' % |
| 66 | 101 | (mount_path, filename)) |
| 67 | 102 | md5 = ssh_client.exec_command( |
| 68 | 103 | 'sudo md5sum -b %s/%s|cut -c 1-32' % (mount_path, filename)) |
| 69 | 104 | ssh_client.exec_command('sudo sync') |
| 105 | if dev_name is not None: | |
| 106 | ssh_client.exec_command('sudo umount %s' % mount_path) | |
| 70 | 107 | return md5 |
| 71 | 108 | |
| 72 | def get_md5_from_file(self, instance, filename): | |
| 73 | ||
| 74 | instance_ip = self.get_server_ip(instance) | |
| 109 | def get_md5_from_file(self, instance, instance_ip, filename, | |
| 110 | dev_name=None): | |
| 75 | 111 | |
| 76 | 112 | md5_sum = self._get_file_md5(instance_ip, filename=filename, |
| 113 | dev_name=dev_name, | |
| 77 | 114 | private_key=self.keypair['private_key'], |
| 78 | 115 | server=instance) |
| 79 | count = self._count_files(instance_ip, | |
| 116 | count = self._count_files(instance_ip, dev_name=dev_name, | |
| 80 | 117 | private_key=self.keypair['private_key'], |
| 81 | 118 | server=instance) |
| 82 | 119 | return count, md5_sum |
| 120 | ||
| 121 | def _attach_and_get_volume_device_name(self, server, volume, instance_ip, | |
| 122 | private_key): | |
| 123 | ssh_client = self.get_remote_client( | |
| 124 | instance_ip, private_key=private_key, | |
| 125 | server=server) | |
| 126 | # List disks before volume attachment | |
| 127 | disks_list_before_attach = ssh_client.list_disks() | |
| 128 | # Attach volume | |
| 129 | volume = self.nova_volume_attach(server, volume) | |
| 130 | # Find the difference between disks before and after attachment that | |
| 131 | # gives us the volume device name | |
| 132 | volume_device_name = self._attached_volume_name( | |
| 133 | disks_list_before_attach, instance_ip, private_key) | |
| 134 | return volume_device_name | |
| 83 | 135 | |
| 84 | 136 | @decorators.idempotent_id('ff10644e-5a70-4a9f-9801-8204bb81fb61') |
| 85 | 137 | @utils.services('compute', 'volume', 'image', 'network') |
| 88 | 140 | |
| 89 | 141 | snapshots. The procedure is as follows: |
| 90 | 142 | |
| 91 | 1) create a volume from image | |
| 92 | 2) Boot an instance from the volume | |
| 93 | 3) create file on vm and write data into it | |
| 143 | 1) Create an instance with ephemeral disk | |
| 144 | 2) Create a volume, attach it to the instance and create a filesystem | |
| 145 | on it and mount it | |
| 146 | 3) Mount the volume, create a file and write data into it, Unmount it | |
| 94 | 147 | 4) create snapshot |
| 95 | 148 | 5) repeat 3 and 4 two more times (simply creating 3 snapshots) |
| 96 | 149 | |
| 97 | Now restore the snapshots one by one into volume, create instances | |
| 98 | from it and check the number of files and file content at each | |
| 150 | Now create volume from the snapshots one by one, attach it to the | |
| 151 | instance and check the number of files and file content at each | |
| 99 | 152 | point when snapshot was created. |
| 100 | 153 | """ |
| 101 | 154 | |
| 102 | # Create a volume from image | |
| 103 | volume = self.create_volume_from_image() | |
| 104 | ||
| 105 | # create an instance from bootable volume | |
| 106 | server = self.boot_instance_from_resource( | |
| 107 | source_id=volume['id'], | |
| 108 | source_type='volume', | |
| 109 | keypair=self.keypair, | |
| 110 | security_group=self.security_group) | |
| 155 | # Create an instance | |
| 156 | server = self.create_server( | |
| 157 | key_name=self.keypair['name'], | |
| 158 | security_groups=[{'name': self.security_group['name']}]) | |
| 159 | ||
| 160 | # Create an empty volume | |
| 161 | volume = self.create_volume() | |
| 111 | 162 | |
| 112 | 163 | instance_ip = self.get_server_ip(server) |
| 164 | ||
| 165 | # Attach volume to instance and find it's device name (eg: /dev/vdb) | |
| 166 | volume_device_name = self._attach_and_get_volume_device_name( | |
| 167 | server, volume, instance_ip, self.keypair['private_key']) | |
| 168 | ||
| 169 | # Create filesystem on the volume | |
| 170 | self._make_fs(instance_ip, self.keypair['private_key'], server, | |
| 171 | volume_device_name) | |
| 113 | 172 | |
| 114 | 173 | # Write data to volume |
| 115 | 174 | file1_md5 = self.create_md5_new_file( |
| 116 | instance_ip, filename="file1", | |
| 175 | instance_ip, dev_name=volume_device_name, filename="file1", | |
| 117 | 176 | private_key=self.keypair['private_key'], |
| 118 | 177 | server=instance_ip) |
| 119 | 178 | |
| 122 | 181 | |
| 123 | 182 | # Write data to volume |
| 124 | 183 | file2_md5 = self.create_md5_new_file( |
| 125 | instance_ip, filename="file2", | |
| 184 | instance_ip, dev_name=volume_device_name, filename="file2", | |
| 126 | 185 | private_key=self.keypair['private_key'], |
| 127 | 186 | server=instance_ip) |
| 128 | 187 | |
| 131 | 190 | |
| 132 | 191 | # Write data to volume |
| 133 | 192 | file3_md5 = self.create_md5_new_file( |
| 134 | instance_ip, filename="file3", | |
| 193 | instance_ip, dev_name=volume_device_name, filename="file3", | |
| 135 | 194 | private_key=self.keypair['private_key'], |
| 136 | 195 | server=instance_ip) |
| 137 | 196 | |
| 138 | 197 | # Create third snapshot |
| 139 | 198 | snapshot3 = self.create_volume_snapshot(volume['id'], force=True) |
| 140 | 199 | |
| 141 | # Create volume, instance and check file and contents for snap1 | |
| 142 | instance_1 = self._launch_instance_from_snapshot(snapshot1) | |
| 143 | count_snap_1, md5_file_1 = self.get_md5_from_file(instance_1, | |
| 144 | 'file1') | |
| 200 | # Detach the volume | |
| 201 | self.nova_volume_detach(server, volume) | |
| 202 | ||
| 203 | # Create volume from snapshot, attach it to instance and check file | |
| 204 | # and contents for snap1 | |
| 205 | volume_snap_1 = self.create_volume(snapshot_id=snapshot1['id']) | |
| 206 | volume_device_name = self._attach_and_get_volume_device_name( | |
| 207 | server, volume_snap_1, instance_ip, self.keypair['private_key']) | |
| 208 | count_snap_1, md5_file_1 = self.get_md5_from_file( | |
| 209 | server, instance_ip, 'file1', dev_name=volume_device_name) | |
| 210 | # Detach the volume | |
| 211 | self.nova_volume_detach(server, volume_snap_1) | |
| 145 | 212 | |
| 146 | 213 | self.assertEqual(count_snap_1, 1) |
| 147 | 214 | self.assertEqual(file1_md5, md5_file_1) |
| 148 | 215 | |
| 149 | # Create volume, instance and check file and contents for snap2 | |
| 150 | instance_2 = self._launch_instance_from_snapshot(snapshot2) | |
| 151 | count_snap_2, md5_file_2 = self.get_md5_from_file(instance_2, | |
| 152 | 'file2') | |
| 216 | # Create volume from snapshot, attach it to instance and check file | |
| 217 | # and contents for snap2 | |
| 218 | volume_snap_2 = self.create_volume(snapshot_id=snapshot2['id']) | |
| 219 | volume_device_name = self._attach_and_get_volume_device_name( | |
| 220 | server, volume_snap_2, instance_ip, self.keypair['private_key']) | |
| 221 | count_snap_2, md5_file_2 = self.get_md5_from_file( | |
| 222 | server, instance_ip, 'file2', dev_name=volume_device_name) | |
| 223 | # Detach the volume | |
| 224 | self.nova_volume_detach(server, volume_snap_2) | |
| 153 | 225 | |
| 154 | 226 | self.assertEqual(count_snap_2, 2) |
| 155 | 227 | self.assertEqual(file2_md5, md5_file_2) |
| 156 | 228 | |
| 157 | # Create volume, instance and check file and contents for snap3 | |
| 158 | instance_3 = self._launch_instance_from_snapshot(snapshot3) | |
| 159 | count_snap_3, md5_file_3 = self.get_md5_from_file(instance_3, | |
| 160 | 'file3') | |
| 229 | # Create volume from snapshot, attach it to instance and check file | |
| 230 | # and contents for snap3 | |
| 231 | volume_snap_3 = self.create_volume(snapshot_id=snapshot3['id']) | |
| 232 | volume_device_name = self._attach_and_get_volume_device_name( | |
| 233 | server, volume_snap_3, instance_ip, self.keypair['private_key']) | |
| 234 | count_snap_3, md5_file_3 = self.get_md5_from_file( | |
| 235 | server, instance_ip, 'file3', dev_name=volume_device_name) | |
| 236 | # Detach the volume | |
| 237 | self.nova_volume_detach(server, volume_snap_3) | |
| 161 | 238 | |
| 162 | 239 | self.assertEqual(count_snap_3, 3) |
| 163 | 240 | self.assertEqual(file3_md5, md5_file_3) |