Commit 5b4949240fc44b69579949f4b312e3b61bca8beb - libblockdev

New upstream version 2.25 Michael Biebl 3 years ago

21 changed file(s) with 599 addition(s) and 363 deletion(s). Raw diff Collapse all Expand all

+29

-0

NEWS.rst less more

	0	Libblockdev 2.25
	1	----------------
	2
	3	New minor release of the libblockdev library with multiple fixes. See below
	4	for details.
	5
	6	Full list of changes
	7
	8	Tomas Bzatek (6):
	9	- exec: Fix polling for stdout and stderr
	10	- exec: Use non-blocking read and process the buffer manually
	11	- exec: Clarify the BDUtilsProgExtract callback documentation
	12	- tests: Add bufferbloat exec tests
	13	- tests: Add null-byte exec tests
	14	- lvm: Fix bd_lvm_vdopooldata_* symbols
	15
	16	Vojtech Trefny (10):
	17	- exec: Fix setting locale for util calls
	18	- fs: Do not report error when errors were fixed by e2fsck
	19	- README: Use CI status image for 2.x-branch on 2.x
	20	- fs: Fix compile error in ext_repair caused by cherry pick from master
	21	- Mark all GIR file constants as guint64
	22	- lvm: Set thin metadata limits to match limits LVM uses in lvcreate
	23	- lvm: Do not use thin_metadata_size to recommend thin metadata size
	24	- lvm: Use the UNUSED macro instead of __attribute__((unused))
	25	- Fix max size limit for LVM thinpool metadata
	26	- loop: Retry LOOP_SET_STATUS64 on EAGAIN
	27
	28
0	29	Libblockdev 2.24
1	30	----------------
2	31

-1

README.md less more

0	0	### CI status
1	1
2		<img alt="CI status" src="https://fedorapeople.org/groups/storage_apis/statuses/libblockdev-master.svg" width="100%" height="300ex" />
	2	<img alt="CI status" src="https://fedorapeople.org/groups/storage_apis/statuses/libblockdev-2.x.svg" width="100%" height="300ex" />
3	3
4	4
5	5	### Introduction

-1

configure.ac less more

0	0	# configure.ac for libblockdev
1	1
2		AC_INIT([libblockdev], [2.24], [vpodzime@redhat.com])
	2	AC_INIT([libblockdev], [2.25], [vpodzime@redhat.com])
3	3
4	4	# Disable building static libraries.
5	5	# This needs to be set before initializing automake

+19

-5

dist/libblockdev.spec.in less more

123	123	%define configure_opts %{?python2_copts} %{?python3_copts} %{?bcache_copts} %{?lvm_dbus_copts} %{?btrfs_copts} %{?crypto_copts} %{?dm_copts} %{?loop_copts} %{?lvm_copts} %{?lvm_dbus_copts} %{?mdraid_copts} %{?mpath_copts} %{?swap_copts} %{?kbd_copts} %{?part_copts} %{?fs_copts} %{?nvdimm_copts} %{?vdo_copts} %{?tools_copts} %{?gi_copts}
124	124
125	125	Name: libblockdev
126		Version: 2.24
	126	Version: 2.25
127	127	Release: 1%{?dist}
128	128	Summary: A library for low-level manipulation with block devices
129	129	License: LGPLv2+

386	386	Summary: The LVM plugin for the libblockdev library
387	387	Requires: %{name}-utils%{?_isa} >= 0.11
388	388	Requires: lvm2
389		# for thin_metadata_size
390		Requires: device-mapper-persistent-data
391	389
392	390	%description lvm
393	391	The libblockdev library plugin (and in the same time a standalone library)

410	408	Summary: The LVM plugin for the libblockdev library
411	409	Requires: %{name}-utils%{?_isa} >= 1.4
412	410	Requires: lvm2-dbusd >= 2.02.156
413		# for thin_metadata_size
414		Requires: device-mapper-persistent-data
415	411
416	412	%description lvm-dbus
417	413	The libblockdev library plugin (and in the same time a standalone library)

982	978	%files plugins-all
983	979
984	980	%changelog
	981	* Mon Jan 11 2021 Vojtech Trefny <vtrefny@redhat.com> - 2.25-1
	982	- loop: Retry LOOP_SET_STATUS64 on EAGAIN (vtrefny)
	983	- Fix max size limit for LVM thinpool metadata (vtrefny)
	984	- lvm: Use the UNUSED macro instead of __attribute__((unused)) (vtrefny)
	985	- lvm: Do not use thin_metadata_size to recommend thin metadata size (vtrefny)
	986	- lvm: Set thin metadata limits to match limits LVM uses in lvcreate (vtrefny)
	987	- Mark all GIR file constants as guint64 (vtrefny)
	988	- lvm: Fix bd_lvm_vdopooldata_* symbols (tbzatek)
	989	- fs: Fix compile error in ext_repair caused by cherry pick from master (vtrefny)
	990	- README: Use CI status image for 2.x-branch on 2.x (vtrefny)
	991	- fs: Do not report error when errors were fixed by e2fsck (vtrefny)
	992	- tests: Add null-byte exec tests (tbzatek)
	993	- tests: Add bufferbloat exec tests (tbzatek)
	994	- exec: Clarify the BDUtilsProgExtract callback documentation (tbzatek)
	995	- exec: Use non-blocking read and process the buffer manually (tbzatek)
	996	- exec: Fix polling for stdout and stderr (tbzatek)
	997	- exec: Fix setting locale for util calls (vtrefny)
	998
985	999	* Fri May 22 2020 Vojtech Trefny <vtrefny@redhat.com> - 2.24-1
986	1000	- Mark VDO plugin as deprecated since 2.24 (vtrefny)
987	1001	- Fix multiple uninitialized values discovered by coverity (vtrefny)

-1

src/lib/plugin_apis/btrfs.api less more

2	2	#include <blockdev/utils.h>
3	3
4	4	#define BD_BTRFS_MAIN_VOLUME_ID 5
5		#define BD_BTRFS_MIN_MEMBER_SIZE (128 MiB)
	5	#define BD_BTRFS_MIN_MEMBER_SIZE G_GUINT64_CONSTANT (134217728ULL) // 128 MiB
6	6
7	7	GQuark bd_btrfs_error_quark (void) {
8	8	return g_quark_from_static_string ("g-bd-btrfs-error-quark");

-1

src/lib/plugin_apis/crypto.api less more

0	0	#include <glib.h>
1	1	#include <blockdev/utils.h>
2	2
3		#define BD_CRYPTO_LUKS_METADATA_SIZE (2 MiB)
	3	#define BD_CRYPTO_LUKS_METADATA_SIZE G_GUINT64_CONSTANT (2097152ULL) // 2 MiB
4	4
5	5	GQuark bd_crypto_error_quark (void) {
6	6	return g_quark_from_static_string ("g-bd-crypto-error-quark");

+26

-24

src/lib/plugin_apis/lvm.api less more

14	14	#define BD_LVM_MAX_LV_SIZE G_GUINT64_CONSTANT (9223372036854775808ULL)
15	15
16	16
17		#define BD_LVM_DEFAULT_PE_START (1 MiB)
18		#define BD_LVM_DEFAULT_PE_SIZE (4 MiB)
19		#define BD_LVM_MIN_PE_SIZE (1 KiB)
20		#define BD_LVM_MAX_PE_SIZE (16 GiB)
21		#define BD_LVM_MIN_THPOOL_MD_SIZE (2 MiB)
22		#define BD_LVM_MAX_THPOOL_MD_SIZE (16 GiB)
23		#define BD_LVM_MIN_THPOOL_CHUNK_SIZE (64 KiB)
24		#define BD_LVM_MAX_THPOOL_CHUNK_SIZE (1 GiB)
25		#define BD_LVM_DEFAULT_CHUNK_SIZE (64 KiB)
	17	#define BD_LVM_DEFAULT_PE_START G_GUINT64_CONSTANT (1048576ULL) // 1 MiB
	18	#define BD_LVM_DEFAULT_PE_SIZE G_GUINT64_CONSTANT (4194304ULL) // 4 MiB
	19	#define BD_LVM_MIN_PE_SIZE G_GUINT64_CONSTANT (1024ULL) // 1 KiB
	20	#define BD_LVM_MAX_PE_SIZE G_GUINT64_CONSTANT (17179869184ULL) // 16 GiB
	21	#define BD_LVM_MIN_THPOOL_MD_SIZE G_GUINT64_CONSTANT (4194304ULL) // 4 MiB
	22	#define BD_LVM_MAX_THPOOL_MD_SIZE G_GUINT64_CONSTANT (16978542592ULL) // 15.81 GiB
	23	#define BD_LVM_MIN_THPOOL_CHUNK_SIZE G_GUINT64_CONSTANT (65536ULL) // 64 KiB
	24	#define BD_LVM_MAX_THPOOL_CHUNK_SIZE G_GUINT64_CONSTANT (1073741824ULL) // 1 GiB
	25	#define BD_LVM_DEFAULT_CHUNK_SIZE G_GUINT64_CONSTANT (65536ULL) // 64 KiB
26	26
27	27	/* according to lvmcache (7) */
28		#define BD_LVM_MIN_CACHE_MD_SIZE (8 MiB)
	28	#define BD_LVM_MIN_CACHE_MD_SIZE (8388608ULL) // 8 MiB
29	29
30	30	GQuark bd_lvm_error_quark (void) {
31	31	return g_quark_from_static_string ("g-bd-lvm-error-quark");

359	359	return type;
360	360	}
361	361
362		#define BD_LVM_TYPE_VDODATA (bd_lvm_vdodata_get_type ())
363		GType bd_lvm_vdodata_get_type();
	362	#define BD_LVM_TYPE_VDOPOOLDATA (bd_lvm_vdopooldata_get_type ())
	363	GType bd_lvm_vdopooldata_get_type();
364	364
365	365	/**
366	366	* BDLVMVDOPooldata:

387	387	} BDLVMVDOPooldata;
388	388
389	389	/**
390		* bd_lvm_vdodata_copy: (skip)
	390	* bd_lvm_vdopooldata_copy: (skip)
391	391	*
392	392	* Creates a new copy of @data.
393	393	*/
394		BDLVMVDOPooldata* bd_lvm_vdodata_copy (BDLVMVDOPooldata *data) {
	394	BDLVMVDOPooldata* bd_lvm_vdopooldata_copy (BDLVMVDOPooldata *data) {
395	395	if (data == NULL)
396	396	return NULL;
397	397

410	410	}
411	411
412	412	/**
413		* bd_lvm_vdodata_free: (skip)
	413	* bd_lvm_vdopooldata_free: (skip)
414	414	*
415	415	* Frees @data.
416	416	*/
417		void bd_lvm_vdodata_free (BDLVMVDOPooldata *data) {
	417	void bd_lvm_vdopooldata_free (BDLVMVDOPooldata *data) {
418	418	if (data == NULL)
419	419	return;
420	420
421	421	g_free (data);
422	422	}
423	423
424		GType bd_lvm_vdodata_get_type () {
	424	GType bd_lvm_vdopooldata_get_type () {
425	425	static GType type = 0;
426	426
427	427	if (G_UNLIKELY(type == 0)) {
428	428	type = g_boxed_type_register_static("BDLVMVDOPooldata",
429		(GBoxedCopyFunc) bd_lvm_vdodata_copy,
430		(GBoxedFreeFunc) bd_lvm_vdodata_free);
	429	(GBoxedCopyFunc) bd_lvm_vdopooldata_copy,
	430	(GBoxedFreeFunc) bd_lvm_vdopooldata_free);
431	431	}
432	432
433	433	return type;

703	703	* bd_lvm_get_thpool_meta_size:
704	704	* @size: size of the thin pool
705	705	* @chunk_size: chunk size of the thin pool or 0 to use the default (%BD_LVM_DEFAULT_CHUNK_SIZE)
706		* @n_snapshots: number of snapshots that will be created in the pool
707		* @error: (out): place to store error (if any)
708		*
709		* Returns: recommended size of the metadata space for the specified pool or 0
710		* in case of error
	706	* @n_snapshots: ignored
	707	* @error: (out): place to store error (if any)
	708	*
	709	* Note: This function will be changed in 3.0: the @n_snapshots parameter
	710	* is currently not used and will be removed.
	711	*
	712	* Returns: recommended size of the metadata space for the specified pool
711	713	*
712	714	* Tech category: %BD_LVM_TECH_THIN_CALCS no mode (it is ignored)
713	715	*/

-2

src/lib/plugin_apis/mdraid.api less more

6	6
7	7	/* taken from blivet */
8	8	// these defaults were determined empirically
9		#define BD_MD_SUPERBLOCK_SIZE (2 MiB)
10		#define BD_MD_CHUNK_SIZE (512 KiB)
	9	#define BD_MD_SUPERBLOCK_SIZE G_GUINT64_CONSTANT (2097152ULL) // 2 MiB
	10	#define BD_MD_CHUNK_SIZE G_GUINT64_CONSTANT (524288ULL) // 512 KiB
11	11
12	12	GQuark bd_md_error_quark (void) {
13	13	return g_quark_from_static_string ("g-bd-md-error-quark");

+18

-3

src/plugins/fs/ext.c less more

403	403	const gchar *args_progress[7] = {"e2fsck", "-f", unsafe ? "-y" : "-p", "-C", "1", device, NULL};
404	404	const gchar *args[5] = {"e2fsck", "-f", unsafe ? "-y" : "-p", device, NULL};
405	405	gint status = 0;
	406	gboolean ret = FALSE;
406	407
407	408	if (!check_deps (&avail_deps, DEPS_E2FSCK_MASK, deps, DEPS_LAST, &deps_check_lock, error))
408	409	return FALSE;
409	410
410	411	if (bd_utils_prog_reporting_initialized ()) {
411		return bd_utils_exec_and_report_progress (args_progress, extra, extract_e2fsck_progress, &status, error);
	412	ret = bd_utils_exec_and_report_progress (args_progress, extra, extract_e2fsck_progress, &status, error);
412	413	} else {
413		return bd_utils_exec_and_report_status_error (args, extra, &status, error);
414		}
	414	ret = bd_utils_exec_and_report_status_error (args, extra, &status, error);
	415	}
	416
	417	if (!ret) {
	418	if (status == 1) {
	419	/* no error should be reported for exit code 1 - File system errors corrected */
	420	g_clear_error (error);
	421	ret = TRUE;
	422	} else if (status == 2) {
	423	/* no error should be reported for exit code 2 - File system errors corrected, system should be rebooted */
	424	g_warning ("File system errors on %s were successfully corrected, but system reboot is advised.", device);
	425	g_clear_error (error);
	426	ret = TRUE;
	427	}
	428	}
	429	return ret;
415	430	}
416	431
417	432	/**

+14

-1

src/plugins/loop.c less more

23	23	#include <fcntl.h>
24	24	#include <sys/ioctl.h>
25	25	#include <linux/loop.h>
	26	#include <errno.h>
26	27	#include <blockdev/utils.h>
27	28	#include "loop.h"
28	29

231	232	gint loop_fd = -1;
232	233	struct loop_info64 li64;
233	234	guint64 progress_id = 0;
	235	gint status = 0;
	236	guint n_try = 0;
234	237
235	238	progress_id = bd_utils_report_started ("Started setting up loop device");
236	239

287	290
288	291	bd_utils_report_progress (progress_id, 66, "Associated the loop device");
289	292
290		if (ioctl (loop_fd, LOOP_SET_STATUS64, &li64) < 0) {
	293	/* we may need to try multiple times with some delays in case the device is
	294	busy at the very moment */
	295	for (n_try=10, status=-1; (status != 0) && (n_try > 0); n_try--) {
	296	status = ioctl (loop_fd, LOOP_SET_STATUS64, &li64);
	297	if (status < 0 && errno == EAGAIN)
	298	g_usleep (100 * 1000); /* microseconds */
	299	else
	300	break;
	301	}
	302
	303	if (status != 0) {
291	304	g_set_error (error, BD_LOOP_ERROR, BD_LOOP_ERROR_FAIL,
292	305	"Failed to set status for the %s device: %m", loop_device);
293	306	g_free (loop_device);

+32

-80

src/plugins/lvm-dbus.c less more

19	19	#include <glib.h>
20	20	#include <math.h>
21	21	#include <string.h>
22		#include <libdevmapper.h>
23	22	#include <unistd.h>
24	23	#include <blockdev/utils.h>
25	24	#include <gio/gio.h>

247	246	static volatile guint avail_module_deps = 0;
248	247	static GMutex deps_check_lock;
249	248
250		#define DEPS_THMS 0
251		#define DEPS_THMS_MASK (1 << DEPS_THMS)
252		#define DEPS_LAST 1
253
254		static const UtilDep deps[DEPS_LAST] = {
255		{"thin_metadata_size", NULL, NULL, NULL},
256		};
257
258	249	#define DBUS_DEPS_LVMDBUSD 0
259	250	#define DBUS_DEPS_LVMDBUSD_MASK (1 << DBUS_DEPS_LVMDBUSD)
260	251	#define DBUS_DEPS_LAST 1

300	291	check_ret = check_ret && success;
301	292	}
302	293
303		for (i=0; i < DEPS_LAST; i++) {
304		success = bd_utils_check_util_version (deps[i].name, deps[i].version,
305		deps[i].ver_arg, deps[i].ver_regexp, &error);
306		if (!success)
307		g_warning ("%s", error->message);
308		else
309		g_atomic_int_or (&avail_deps, 1 << i);
310		g_clear_error (&error);
311		check_ret = check_ret && success;
312		}
313
314	294	if (!check_ret)
315	295	g_warning("Cannot load the LVM plugin");
316	296

385	365	g_set_error (error, BD_LVM_ERROR, BD_LVM_ERROR_TECH_UNAVAIL,
386	366	"Only 'query' supported for thin calculations");
387	367	return FALSE;
388		} else if ((mode & BD_LVM_TECH_MODE_QUERY) &&
389		!check_deps (&avail_deps, DEPS_THMS_MASK, deps, DEPS_LAST, &deps_check_lock, error))
390		return FALSE;
391		else
	368	} else
392	369	return TRUE;
393	370	case BD_LVM_TECH_CALCS:
394	371	if (mode & ~BD_LVM_TECH_MODE_QUERY) {

981	958	return data;
982	959	}
983	960
984		static BDLVMVGdata* get_vg_data_from_props (GVariant props, GError *error __attribute__((unused))) {
	961	static BDLVMVGdata* get_vg_data_from_props (GVariant props, GError *error UNUSED) {
985	962	BDLVMVGdata *data = g_new0 (BDLVMVGdata, 1);
986	963	GVariantDict dict;
987	964

1083	1060	return data;
1084	1061	}
1085	1062
1086		static BDLVMVDOPooldata* get_vdo_data_from_props (GVariant props, GError *error __attribute__((unused))) {
	1063	static BDLVMVDOPooldata* get_vdo_data_from_props (GVariant props, GError *error UNUSED) {
1087	1064	BDLVMVDOPooldata *data = g_new0 (BDLVMVDOPooldata, 1);
1088	1065	GVariantDict dict;
1089	1066	gchar *value = NULL;

1187	1164	*
1188	1165	* Tech category: %BD_LVM_TECH_CALCS no mode (it is ignored)
1189	1166	*/
1190		gboolean bd_lvm_is_supported_pe_size (guint64 size, GError **error __attribute__((unused))) {
	1167	gboolean bd_lvm_is_supported_pe_size (guint64 size, GError **error UNUSED) {
1191	1168	return (((size % 2) == 0) && (size >= (BD_LVM_MIN_PE_SIZE)) && (size <= (BD_LVM_MAX_PE_SIZE)));
1192	1169	}
1193	1170

1199	1176	*
1200	1177	* Tech category: %BD_LVM_TECH_CALCS no mode (it is ignored)
1201	1178	*/
1202		guint64 bd_lvm_get_supported_pe_sizes (GError *error __attribute__((unused))) {
	1179	guint64 bd_lvm_get_supported_pe_sizes (GError *error UNUSED) {
1203	1180	guint8 i;
1204	1181	guint64 val = BD_LVM_MIN_PE_SIZE;
1205	1182	guint8 num_items = ((guint8) round (log2 ((double) BD_LVM_MAX_PE_SIZE))) - ((guint8) round (log2 ((double) BD_LVM_MIN_PE_SIZE))) + 2;

1221	1198	*
1222	1199	* Tech category: %BD_LVM_TECH_CALCS no mode (it is ignored)
1223	1200	*/
1224		guint64 bd_lvm_get_max_lv_size (GError **error __attribute__((unused))) {
	1201	guint64 bd_lvm_get_max_lv_size (GError **error UNUSED) {
1225	1202	return BD_LVM_MAX_LV_SIZE;
1226	1203	}
1227	1204

1241	1218	*
1242	1219	* Tech category: %BD_LVM_TECH_CALCS no mode (it is ignored)
1243	1220	*/
1244		guint64 bd_lvm_round_size_to_pe (guint64 size, guint64 pe_size, gboolean roundup, GError **error __attribute__((unused))) {
	1221	guint64 bd_lvm_round_size_to_pe (guint64 size, guint64 pe_size, gboolean roundup, GError **error UNUSED) {
1245	1222	pe_size = RESOLVE_PE_SIZE(pe_size);
1246	1223	guint64 delta = size % pe_size;
1247	1224	if (delta == 0)

1302	1279	* bd_lvm_get_thpool_meta_size:
1303	1280	* @size: size of the thin pool
1304	1281	* @chunk_size: chunk size of the thin pool or 0 to use the default (%BD_LVM_DEFAULT_CHUNK_SIZE)
1305		* @n_snapshots: number of snapshots that will be created in the pool
1306		* @error: (out): place to store error (if any)
1307		*
1308		* Returns: recommended size of the metadata space for the specified pool or 0
1309		* in case of error
	1282	* @n_snapshots: ignored
	1283	* @error: (out): place to store error (if any)
	1284	*
	1285	* Note: This function will be changed in 3.0: the @n_snapshots parameter
	1286	* is currently not used and will be removed.
	1287	*
	1288	* Returns: recommended size of the metadata space for the specified pool
1310	1289	*
1311	1290	* Tech category: %BD_LVM_TECH_THIN_CALCS no mode (it is ignored)
1312	1291	*/
1313		guint64 bd_lvm_get_thpool_meta_size (guint64 size, guint64 chunk_size, guint64 n_snapshots, GError **error) {
1314		/* ub - output in bytes, n - output just the number */
1315		const gchar* args[7] = {"thin_metadata_size", "-ub", "-n", NULL, NULL, NULL, NULL};
1316		gchar *output = NULL;
1317		gboolean success = FALSE;
1318		guint64 ret = 0;
1319
1320		if (!check_deps (&avail_deps, DEPS_THMS_MASK, deps, DEPS_LAST, &deps_check_lock, error))
1321		return 0;
1322
1323		/* s - total size, b - chunk size, m - number of snapshots */
1324		args[3] = g_strdup_printf ("-s%"G_GUINT64_FORMAT, size);
1325		args[4] = g_strdup_printf ("-b%"G_GUINT64_FORMAT,
1326		chunk_size != 0 ? chunk_size : (guint64) BD_LVM_DEFAULT_CHUNK_SIZE);
1327		args[5] = g_strdup_printf ("-m%"G_GUINT64_FORMAT, n_snapshots);
1328
1329		success = bd_utils_exec_and_capture_output (args, NULL, &output, error);
1330		g_free ((gchar*) args[3]);
1331		g_free ((gchar*) args[4]);
1332		g_free ((gchar*) args[5]);
1333
1334		if (!success) {
1335		/* error is already set */
1336		g_free (output);
1337		return 0;
1338		}
1339
1340		ret = g_ascii_strtoull (output, NULL, 0);
1341		if (ret == 0) {
1342		g_set_error (error, BD_LVM_ERROR, BD_LVM_ERROR_PARSE,
1343		"Failed to parse number from thin_metadata_size's output: '%s'",
1344		output);
1345		g_free (output);
1346		return 0;
1347		}
1348
1349		g_free (output);
1350
1351		return MAX (ret, BD_LVM_MIN_THPOOL_MD_SIZE);
	1292	guint64 bd_lvm_get_thpool_meta_size (guint64 size, guint64 chunk_size, guint64 n_snapshots UNUSED, GError **error UNUSED) {
	1293	guint64 md_size = 0;
	1294
	1295	/* based on lvcreate metadata size calculation */
	1296	md_size = UINT64_C(64) * size / (chunk_size ? chunk_size : BD_LVM_DEFAULT_CHUNK_SIZE);
	1297
	1298	if (md_size > BD_LVM_MAX_THPOOL_MD_SIZE)
	1299	md_size = BD_LVM_MAX_THPOOL_MD_SIZE;
	1300	else if (md_size < BD_LVM_MIN_THPOOL_MD_SIZE)
	1301	md_size = BD_LVM_MIN_THPOOL_MD_SIZE;
	1302
	1303	return md_size;
1352	1304	}
1353	1305
1354	1306	/**

1360	1312	*
1361	1313	* Tech category: %BD_LVM_TECH_THIN_CALCS no mode (it is ignored)
1362	1314	*/
1363		gboolean bd_lvm_is_valid_thpool_md_size (guint64 size, GError **error __attribute__((unused))) {
	1315	gboolean bd_lvm_is_valid_thpool_md_size (guint64 size, GError **error UNUSED) {
1364	1316	return ((BD_LVM_MIN_THPOOL_MD_SIZE <= size) && (size <= BD_LVM_MAX_THPOOL_MD_SIZE));
1365	1317	}
1366	1318

1374	1326	*
1375	1327	* Tech category: %BD_LVM_TECH_THIN_CALCS no mode (it is ignored)
1376	1328	*/
1377		gboolean bd_lvm_is_valid_thpool_chunk_size (guint64 size, gboolean discard, GError **error __attribute__((unused))) {
	1329	gboolean bd_lvm_is_valid_thpool_chunk_size (guint64 size, gboolean discard, GError **error UNUSED) {
1378	1330	gdouble size_log2 = 0.0;
1379	1331
1380	1332	if ((size < BD_LVM_MIN_THPOOL_CHUNK_SIZE) \|\| (size > BD_LVM_MAX_THPOOL_CHUNK_SIZE))

2663	2615	*
2664	2616	* Tech category: %BD_LVM_TECH_GLOB_CONF no mode (it is ignored)
2665	2617	*/
2666		gboolean bd_lvm_set_global_config (const gchar new_config, GError *error __attribute__((unused))) {
	2618	gboolean bd_lvm_set_global_config (const gchar new_config, GError *error UNUSED) {
2667	2619	/* XXX: the error attribute will likely be used in the future when
2668	2620	some validation comes into the game */
2669	2621

2688	2640	*
2689	2641	* Tech category: %BD_LVM_TECH_GLOB_CONF no mode (it is ignored)
2690	2642	*/
2691		gchar* bd_lvm_get_global_config (GError **error __attribute__((unused))) {
	2643	gchar* bd_lvm_get_global_config (GError **error UNUSED) {
2692	2644	gchar *ret = NULL;
2693	2645
2694	2646	g_mutex_lock (&global_config_lock);

2707	2659	*
2708	2660	* Tech category: %BD_LVM_TECH_CACHE_CALCS no mode (it is ignored)
2709	2661	*/
2710		guint64 bd_lvm_cache_get_default_md_size (guint64 cache_size, GError **error __attribute__((unused))) {
	2662	guint64 bd_lvm_cache_get_default_md_size (guint64 cache_size, GError **error UNUSED) {
2711	2663	return MAX ((guint64) cache_size / 1000, BD_LVM_MIN_CACHE_MD_SIZE);
2712	2664	}
2713	2665

2717	2669	*
2718	2670	* Get LV type string from flags.
2719	2671	*/
2720		static const gchar* get_lv_type_from_flags (BDLVMCachePoolFlags flags, gboolean meta, GError **error __attribute__((unused))) {
	2672	static const gchar* get_lv_type_from_flags (BDLVMCachePoolFlags flags, gboolean meta, GError **error UNUSED) {
2721	2673	if (!meta) {
2722	2674	if (flags & BD_LVM_CACHE_POOL_STRIPED)
2723	2675	return "striped";

+37

-65

src/plugins/lvm.c less more

19	19	#include <glib.h>
20	20	#include <math.h>
21	21	#include <string.h>
22		#include <libdevmapper.h>
23	22	#include <unistd.h>
24	23	#include <blockdev/utils.h>
25	24

154	153	g_free (data);
155	154	}
156	155
157		BDLVMVDOPooldata* bd_lvm_vdodata_copy (BDLVMVDOPooldata *data) {
	156	BDLVMVDOPooldata* bd_lvm_vdopooldata_copy (BDLVMVDOPooldata *data) {
158	157	if (data == NULL)
159	158	return NULL;
160	159

163	162	new_data->operating_mode = data->operating_mode;
164	163	new_data->compression_state = data->compression_state;
165	164	new_data->index_state = data->index_state;
	165	new_data->write_policy = data->write_policy;
166	166	new_data->used_size = data->used_size;
167	167	new_data->saving_percent = data->saving_percent;
	168	new_data->index_memory_size = data->index_memory_size;
168	169	new_data->deduplication = data->deduplication;
169	170	new_data->compression = data->compression;
170	171	return new_data;
171	172	}
172	173
173		void bd_lvm_vdodata_free (BDLVMVDOPooldata *data) {
	174	void bd_lvm_vdopooldata_free (BDLVMVDOPooldata *data) {
174	175	if (data == NULL)
175	176	return;
176	177

210	211
211	212	#define DEPS_LVM 0
212	213	#define DEPS_LVM_MASK (1 << DEPS_LVM)
213		#define DEPS_THMS 1
214		#define DEPS_THMS_MASK (1 << DEPS_THMS)
215		#define DEPS_LAST 2
	214	#define DEPS_LAST 1
216	215
217	216	static const UtilDep deps[DEPS_LAST] = {
218	217	{"lvm", LVM_MIN_VERSION, "version", "LVM version:\\s+([\\d\\.]+)"},
219		{"thin_metadata_size", NULL, NULL, NULL},
220	218	};
221	219
222	220	#define FEATURES_VDO 0

232	230	#define MODULE_DEPS_LAST 1
233	231
234	232	static const gchar*const module_deps[MODULE_DEPS_LAST] = { "kvdo" };
	233
	234	#define UNUSED __attribute__((unused))
235	235
236	236	/**
237	237	* bd_lvm_check_deps:

314	314	g_set_error (error, BD_LVM_ERROR, BD_LVM_ERROR_TECH_UNAVAIL,
315	315	"Only 'query' supported for thin calculations");
316	316	return FALSE;
317		} else if ((mode & BD_LVM_TECH_MODE_QUERY) &&
318		!check_deps (&avail_deps, DEPS_THMS_MASK, deps, DEPS_LAST, &deps_check_lock, error))
319		return FALSE;
320		else
	317	} else
321	318	return TRUE;
322	319	case BD_LVM_TECH_CALCS:
323	320	if (mode & ~BD_LVM_TECH_MODE_QUERY) {

702	699	*
703	700	* Tech category: %BD_LVM_TECH_CALCS no mode (it is ignored)
704	701	*/
705		gboolean bd_lvm_is_supported_pe_size (guint64 size, GError **error __attribute__((unused))) {
	702	gboolean bd_lvm_is_supported_pe_size (guint64 size, GError **error UNUSED) {
706	703	return (((size % 2) == 0) && (size >= (BD_LVM_MIN_PE_SIZE)) && (size <= (BD_LVM_MAX_PE_SIZE)));
707	704	}
708	705

714	711	*
715	712	* Tech category: %BD_LVM_TECH_CALCS no mode (it is ignored)
716	713	*/
717		guint64 bd_lvm_get_supported_pe_sizes (GError *error __attribute__((unused))) {
	714	guint64 bd_lvm_get_supported_pe_sizes (GError *error UNUSED) {
718	715	guint8 i;
719	716	guint64 val = BD_LVM_MIN_PE_SIZE;
720	717	guint8 num_items = ((guint8) round (log2 ((double) BD_LVM_MAX_PE_SIZE))) - ((guint8) round (log2 ((double) BD_LVM_MIN_PE_SIZE))) + 2;

736	733	*
737	734	* Tech category: %BD_LVM_TECH_CALCS no mode (it is ignored)
738	735	*/
739		guint64 bd_lvm_get_max_lv_size (GError **error __attribute__((unused))) {
	736	guint64 bd_lvm_get_max_lv_size (GError **error UNUSED) {
740	737	return BD_LVM_MAX_LV_SIZE;
741	738	}
742	739

756	753	*
757	754	* Tech category: %BD_LVM_TECH_CALCS no mode (it is ignored)
758	755	*/
759		guint64 bd_lvm_round_size_to_pe (guint64 size, guint64 pe_size, gboolean roundup, GError **error __attribute__((unused))) {
	756	guint64 bd_lvm_round_size_to_pe (guint64 size, guint64 pe_size, gboolean roundup, GError **error UNUSED) {
760	757	pe_size = RESOLVE_PE_SIZE(pe_size);
761	758	guint64 delta = size % pe_size;
762	759	if (delta == 0)

817	814	* bd_lvm_get_thpool_meta_size:
818	815	* @size: size of the thin pool
819	816	* @chunk_size: chunk size of the thin pool or 0 to use the default (%BD_LVM_DEFAULT_CHUNK_SIZE)
820		* @n_snapshots: number of snapshots that will be created in the pool
821		* @error: (out): place to store error (if any)
822		*
823		* Returns: recommended size of the metadata space for the specified pool or 0
824		* in case of error
	817	* @n_snapshots: ignored
	818	* @error: (out): place to store error (if any)
	819	*
	820	* Note: This function will be changed in 3.0: the @n_snapshots parameter
	821	* is currently not used and will be removed.
	822	*
	823	* Returns: recommended size of the metadata space for the specified pool
825	824	*
826	825	* Tech category: %BD_LVM_TECH_THIN_CALCS no mode (it is ignored)
827	826	*/
828		guint64 bd_lvm_get_thpool_meta_size (guint64 size, guint64 chunk_size, guint64 n_snapshots, GError **error) {
829		/* ub - output in bytes, n - output just the number */
830		const gchar* args[7] = {"thin_metadata_size", "-ub", "-n", NULL, NULL, NULL, NULL};
831		gchar *output = NULL;
832		gboolean success = FALSE;
833		guint64 ret = 0;
834
835		if (!check_deps (&avail_deps, DEPS_THMS_MASK, deps, DEPS_LAST, &deps_check_lock, error))
836		return 0;
837
838		/* s - total size, b - chunk size, m - number of snapshots */
839		args[3] = g_strdup_printf ("-s%"G_GUINT64_FORMAT, size);
840		args[4] = g_strdup_printf ("-b%"G_GUINT64_FORMAT,
841		chunk_size != 0 ? chunk_size : (guint64) BD_LVM_DEFAULT_CHUNK_SIZE);
842		args[5] = g_strdup_printf ("-m%"G_GUINT64_FORMAT, n_snapshots);
843
844		success = bd_utils_exec_and_capture_output (args, NULL, &output, error);
845		g_free ((gchar*) args[3]);
846		g_free ((gchar*) args[4]);
847		g_free ((gchar*) args[5]);
848
849		if (!success) {
850		/* error is already set */
851		g_free (output);
852		return 0;
853		}
854
855		ret = g_ascii_strtoull (output, NULL, 0);
856		if (ret == 0) {
857		g_set_error (error, BD_LVM_ERROR, BD_LVM_ERROR_PARSE,
858		"Failed to parse number from thin_metadata_size's output: '%s'",
859		output);
860		g_free (output);
861		return 0;
862		}
863
864		g_free (output);
865
866		return MAX (ret, BD_LVM_MIN_THPOOL_MD_SIZE);
	827	guint64 bd_lvm_get_thpool_meta_size (guint64 size, guint64 chunk_size, guint64 n_snapshots UNUSED, GError **error UNUSED) {
	828	guint64 md_size = 0;
	829
	830	/* based on lvcreate metadata size calculation */
	831	md_size = UINT64_C(64) * size / (chunk_size ? chunk_size : BD_LVM_DEFAULT_CHUNK_SIZE);
	832
	833	if (md_size > BD_LVM_MAX_THPOOL_MD_SIZE)
	834	md_size = BD_LVM_MAX_THPOOL_MD_SIZE;
	835	else if (md_size < BD_LVM_MIN_THPOOL_MD_SIZE)
	836	md_size = BD_LVM_MIN_THPOOL_MD_SIZE;
	837
	838	return md_size;
867	839	}
868	840
869	841	/**

875	847	*
876	848	* Tech category: %BD_LVM_TECH_THIN_CALCS no mode (it is ignored)
877	849	*/
878		gboolean bd_lvm_is_valid_thpool_md_size (guint64 size, GError **error __attribute__((unused))) {
	850	gboolean bd_lvm_is_valid_thpool_md_size (guint64 size, GError **error UNUSED) {
879	851	return ((BD_LVM_MIN_THPOOL_MD_SIZE <= size) && (size <= BD_LVM_MAX_THPOOL_MD_SIZE));
880	852	}
881	853

889	861	*
890	862	* Tech category: %BD_LVM_TECH_THIN_CALCS no mode (it is ignored)
891	863	*/
892		gboolean bd_lvm_is_valid_thpool_chunk_size (guint64 size, gboolean discard, GError **error __attribute__((unused))) {
	864	gboolean bd_lvm_is_valid_thpool_chunk_size (guint64 size, gboolean discard, GError **error UNUSED) {
893	865	gdouble size_log2 = 0.0;
894	866
895	867	if ((size < BD_LVM_MIN_THPOOL_CHUNK_SIZE) \|\| (size > BD_LVM_MAX_THPOOL_CHUNK_SIZE))

2010	1982	*
2011	1983	* Tech category: %BD_LVM_TECH_GLOB_CONF no mode (it is ignored)
2012	1984	*/
2013		gboolean bd_lvm_set_global_config (const gchar new_config, GError *error __attribute__((unused))) {
	1985	gboolean bd_lvm_set_global_config (const gchar new_config, GError *error UNUSED) {
2014	1986	/* XXX: the error attribute will likely be used in the future when
2015	1987	some validation comes into the game */
2016	1988

2035	2007	*
2036	2008	* Tech category: %BD_LVM_TECH_GLOB_CONF no mode (it is ignored)
2037	2009	*/
2038		gchar* bd_lvm_get_global_config (GError **error __attribute__((unused))) {
	2010	gchar* bd_lvm_get_global_config (GError **error UNUSED) {
2039	2011	gchar *ret = NULL;
2040	2012
2041	2013	g_mutex_lock (&global_config_lock);

2054	2026	*
2055	2027	* Tech category: %BD_LVM_TECH_CACHE_CALCS no mode (it is ignored)
2056	2028	*/
2057		guint64 bd_lvm_cache_get_default_md_size (guint64 cache_size, GError **error __attribute__((unused))) {
	2029	guint64 bd_lvm_cache_get_default_md_size (guint64 cache_size, GError **error UNUSED) {
2058	2030	return MAX ((guint64) cache_size / 1000, BD_LVM_MIN_CACHE_MD_SIZE);
2059	2031	}
2060	2032

2064	2036	*
2065	2037	* Get LV type string from flags.
2066	2038	*/
2067		static const gchar* get_lv_type_from_flags (BDLVMCachePoolFlags flags, gboolean meta, GError **error __attribute__((unused))) {
	2039	static const gchar* get_lv_type_from_flags (BDLVMCachePoolFlags flags, gboolean meta, GError **error UNUSED) {
2068	2040	if (!meta) {
2069	2041	if (flags & BD_LVM_CACHE_POOL_STRIPED)
2070	2042	return "striped";

-2

src/plugins/lvm.h less more

0	0	#include <glib.h>
1	1	#include <blockdev/utils.h>
	2	#include <libdevmapper.h>
2	3
3	4	#ifndef BD_LVM
4	5	#define BD_LVM

20	21	#define USE_DEFAULT_PE_SIZE 0
21	22	#define RESOLVE_PE_SIZE(size) ((size) == USE_DEFAULT_PE_SIZE ? BD_LVM_DEFAULT_PE_SIZE : (size))
22	23
23		#define BD_LVM_MIN_THPOOL_MD_SIZE (2 MiB)
24		#define BD_LVM_MAX_THPOOL_MD_SIZE (16 GiB)
	24	/* lvm constant for thin pool metadata size is actually half of this
	25	but when they calculate the actual metadata size they double the limit
	26	so lets just double the limit here too */
	27	#define BD_LVM_MIN_THPOOL_MD_SIZE (4 MiB)
	28
	29	/* DM_THIN_MAX_METADATA_SIZE is in 512 sectors */
	30	#define BD_LVM_MAX_THPOOL_MD_SIZE (DM_THIN_MAX_METADATA_SIZE * 512)
	31
25	32	#define BD_LVM_MIN_THPOOL_CHUNK_SIZE (64 KiB)
26	33	#define BD_LVM_MAX_THPOOL_CHUNK_SIZE (1 GiB)
27	34	#define BD_LVM_DEFAULT_CHUNK_SIZE (64 KiB)

-0

src/python/gi/overrides/BlockDev.py less more

461	461	return _lvm_get_thpool_padding(size, pe_size, included)
462	462	__all__.append("lvm_get_thpool_padding")
463	463
	464	_lvm_get_thpool_meta_size = BlockDev.lvm_get_thpool_meta_size
	465	@override(BlockDev.lvm_get_thpool_meta_size)
	466	def lvm_get_thpool_meta_size(size, chunk_size=0, n_snapshots=0):
	467	return _lvm_get_thpool_meta_size(size, chunk_size, n_snapshots)
	468	__all__.append("lvm_get_thpool_meta_size")
	469
464	470	_lvm_pvcreate = BlockDev.lvm_pvcreate
465	471	@override(BlockDev.lvm_pvcreate)
466	472	def lvm_pvcreate(device, data_alignment=0, metadata_size=0, extra=None, **kwargs):

+188

-135

src/utils/exec.c less more

21	21	#include "extra_arg.h"
22	22	#include <syslog.h>
23	23	#include <stdlib.h>
	24	#include <poll.h>
	25	#include <fcntl.h>
24	26	#include <errno.h>
25	27	#include <sys/types.h>
26	28	#include <sys/wait.h>

137	139	}
138	140
139	141	return;
140		}
141
142		static void set_c_locale(gpointer user_data __attribute__((unused))) {
143		if (setenv ("LC_ALL", "C", 1) != 0)
144		g_warning ("Failed to set LC_ALL=C for a child process!");
145	142	}
146	143
147	144	/**

193	190	const BDExtraArg **extra_p = NULL;
194	191	gint exit_status = 0;
195	192	guint i = 0;
	193	gchar **old_env = NULL;
	194	gchar **new_env = NULL;
196	195
197	196	if (extra) {
198	197	args_len = g_strv_length ((gchar **) argv);

218	217	args[i] = NULL;
219	218	}
220	219
	220	old_env = g_get_environ ();
	221	new_env = g_environ_setenv (old_env, "LC_ALL", "C", TRUE);
	222
221	223	task_id = log_running (args ? args : argv);
222		success = g_spawn_sync (NULL, args ? (gchar ) args : (gchar ) argv, NULL, G_SPAWN_SEARCH_PATH,
223		(GSpawnChildSetupFunc) set_c_locale, NULL,
224		&stdout_data, &stderr_data, &exit_status, error);
	224	success = g_spawn_sync (NULL, args ? (gchar ) args : (gchar ) argv, new_env, G_SPAWN_SEARCH_PATH,
	225	NULL, NULL, &stdout_data, &stderr_data, &exit_status, error);
225	226	if (!success) {
226	227	/* error is already populated from the call */
	228	g_strfreev (new_env);
227	229	g_free (stdout_data);
228	230	g_free (stderr_data);
229	231	return FALSE;
230	232	}
	233	g_strfreev (new_env);
231	234
232	235	/* g_spawn_sync set the status in the same way waitpid() does, we need
233	236	to get the process exit code manually (this is similar to calling

266	269
267	270	g_free (stdout_data);
268	271	g_free (stderr_data);
	272	return TRUE;
	273	}
	274
	275	/* buffer size in bytes used to read from stdout and stderr */
	276	#define _EXEC_BUF_SIZE 64*1024
	277
	278	/* similar to g_strstr_len() yet treats 'null' byte as @needle. */
	279	static gchar bd_strchr_len_null (const gchar haystack, gssize haystack_len, const gchar needle) {
	280	gchar *ret;
	281	gchar *ret_null;
	282
	283	ret = memchr (haystack, needle, haystack_len);
	284	ret_null = memchr (haystack, 0, haystack_len);
	285	if (ret && ret_null)
	286	return MIN (ret, ret_null);
	287	else
	288	return MAX (ret, ret_null);
	289	}
	290
	291	static gboolean
	292	_process_fd_event (gint fd, struct pollfd poll_fd, GString read_buffer, GString filtered_buffer, gsize read_buffer_pos, gboolean *done,
	293	guint64 progress_id, guint8 progress, BDUtilsProgExtract prog_extract, GError *error) {
	294	gchar buf[_EXEC_BUF_SIZE] = { 0 };
	295	ssize_t num_read;
	296	gchar *line;
	297	gchar *newline_pos;
	298	int errno_saved;
	299	gboolean eof = FALSE;
	300
	301	if (! *done && (poll_fd->revents & POLLIN)) {
	302	/* read until we get EOF (0) or error (-1), expecting EAGAIN */
	303	while ((num_read = read (fd, buf, _EXEC_BUF_SIZE)) > 0)
	304	g_string_append_len (read_buffer, buf, num_read);
	305	errno_saved = errno;
	306
	307	/* process the fresh data by lines */
	308	if (read_buffer->len > *read_buffer_pos) {
	309	gchar *buf_ptr;
	310	gsize buf_len;
	311
	312	while ((buf_ptr = read_buffer->str + *read_buffer_pos,
	313	buf_len = read_buffer->len - *read_buffer_pos,
	314	newline_pos = bd_strchr_len_null (buf_ptr, buf_len, '\n'))) {
	315	line = g_strndup (buf_ptr, newline_pos - buf_ptr + 1);
	316	if (prog_extract && prog_extract (line, progress))
	317	bd_utils_report_progress (progress_id, *progress, NULL);
	318	else
	319	g_string_append (filtered_buffer, line);
	320	g_free (line);
	321	*read_buffer_pos = newline_pos - read_buffer->str + 1;
	322	}
	323	}
	324
	325	/* read error */
	326	if (num_read < 0 && errno_saved != EAGAIN && errno_saved != EINTR) {
	327	g_set_error (error, BD_UTILS_EXEC_ERROR, BD_UTILS_EXEC_ERROR_FAILED,
	328	"Error reading from pipe: %s", g_strerror (errno_saved));
	329	return FALSE;
	330	}
	331
	332	/* EOF */
	333	if (num_read == 0)
	334	eof = TRUE;
	335	}
	336
	337	if (poll_fd->revents & POLLHUP \|\| poll_fd->revents & POLLERR \|\| poll_fd->revents & POLLNVAL)
	338	eof = TRUE;
	339
	340	if (eof) {
	341	*done = TRUE;
	342	/* process the remaining buffer */
	343	line = read_buffer->str + *read_buffer_pos;
	344	/* GString guarantees the buffer is always NULL-terminated. */
	345	if (strlen (line) > 0) {
	346	if (prog_extract && prog_extract (line, progress))
	347	bd_utils_report_progress (progress_id, *progress, NULL);
	348	else
	349	g_string_append (filtered_buffer, line);
	350	}
	351	}
	352
269	353	return TRUE;
270	354	}
271	355

280	364	gchar *msg = NULL;
281	365	GPid pid = 0;
282	366	gint out_fd = 0;
283		GIOChannel *out_pipe = NULL;
284	367	gint err_fd = 0;
285		GIOChannel *err_pipe = NULL;
286		gchar *line = NULL;
287	368	gint child_ret = -1;
288	369	gint status = 0;
289	370	gboolean ret = FALSE;
290		GIOStatus io_status = G_IO_STATUS_NORMAL;
291	371	gint poll_status = 0;
292	372	guint i = 0;
293	373	guint8 completion = 0;
294		GPollFD fds[2] = {ZERO_INIT, ZERO_INIT};
	374	struct pollfd fds[2] = { ZERO_INIT, ZERO_INIT };
	375	int flags;
295	376	gboolean out_done = FALSE;
296	377	gboolean err_done = FALSE;
297		GString *stdout_data = g_string_new (NULL);
298		GString *stderr_data = g_string_new (NULL);
	378	GString *stdout_data;
	379	GString *stdout_buffer;
	380	GString *stderr_data;
	381	GString *stderr_buffer;
	382	gsize stdout_buffer_pos = 0;
	383	gsize stderr_buffer_pos = 0;
	384	gchar **old_env = NULL;
	385	gchar **new_env = NULL;
	386	gboolean success = TRUE;
299	387
300	388	/* TODO: share this code between functions */
301	389	if (extra) {

324	412
325	413	task_id = log_running (args ? args : argv);
326	414
327		ret = g_spawn_async_with_pipes (NULL, args ? (gchar) args : (gchar) argv, NULL,
	415	old_env = g_get_environ ();
	416	new_env = g_environ_setenv (old_env, "LC_ALL", "C", TRUE);
	417
	418	ret = g_spawn_async_with_pipes (NULL, args ? (gchar) args : (gchar) argv, new_env,
328	419	G_SPAWN_DEFAULT\|G_SPAWN_SEARCH_PATH\|G_SPAWN_DO_NOT_REAP_CHILD,
329	420	NULL, NULL, &pid, NULL, &out_fd, &err_fd, error);
330	421
	422	g_strfreev (new_env);
	423
331	424	if (!ret) {
332	425	/* error is already populated */
333		g_string_free (stdout_data, TRUE);
334		g_string_free (stderr_data, TRUE);
335	426	g_free (args);
336	427	return FALSE;
337	428	}

343	434	g_free (args);
344	435	g_free (msg);
345	436
346		out_pipe = g_io_channel_unix_new (out_fd);
347		err_pipe = g_io_channel_unix_new (err_fd);
348
349		g_io_channel_set_encoding (out_pipe, NULL, NULL);
350		g_io_channel_set_encoding (err_pipe, NULL, NULL);
	437	/* set both fds for non-blocking read */
	438	flags = fcntl (out_fd, F_GETFL, 0);
	439	if (fcntl (out_fd, F_SETFL, flags \| O_NONBLOCK))
	440	g_warning ("_utils_exec_and_report_progress: Failed to set out_fd non-blocking: %m");
	441	flags = fcntl (err_fd, F_GETFL, 0);
	442	if (fcntl (err_fd, F_SETFL, flags \| O_NONBLOCK))
	443	g_warning ("_utils_exec_and_report_progress: Failed to set err_fd non-blocking: %m");
	444
	445	stdout_data = g_string_new (NULL);
	446	stdout_buffer = g_string_new (NULL);
	447	stderr_data = g_string_new (NULL);
	448	stderr_buffer = g_string_new (NULL);
351	449
352	450	fds[0].fd = out_fd;
353	451	fds[1].fd = err_fd;
354		fds[0].events = G_IO_IN \| G_IO_HUP \| G_IO_ERR;
355		fds[1].events = G_IO_IN \| G_IO_HUP \| G_IO_ERR;
356		while (!out_done \|\| !err_done) {
357		poll_status = g_poll (fds, 2, -1);
	452	fds[0].events = POLLIN \| POLLHUP \| POLLERR;
	453	fds[1].events = POLLIN \| POLLHUP \| POLLERR;
	454	while (! (out_done && err_done)) {
	455	poll_status = poll (fds, 2, -1 /* timeout */);
	456	g_warn_if_fail (poll_status != 0); /* no timeout specified, zero should never be returned */
358	457	if (poll_status < 0) {
	458	if (errno == EAGAIN \|\| errno == EINTR)
	459	continue;
359	460	g_set_error (error, BD_UTILS_EXEC_ERROR, BD_UTILS_EXEC_ERROR_FAILED,
360		"Failed to poll output FDs.");
	461	"Failed to poll output FDs: %m");
361	462	bd_utils_report_finished (progress_id, (*error)->message);
362		g_io_channel_shutdown (out_pipe, FALSE, NULL);
363		g_io_channel_unref (out_pipe);
364		g_io_channel_shutdown (err_pipe, FALSE, NULL);
365		g_io_channel_unref (err_pipe);
366		g_string_free (stdout_data, TRUE);
367		g_string_free (stderr_data, TRUE);
368		return FALSE;
369		} else if (poll_status != 2)
370		/* both revents fields were not filled yet */
371		continue;
372		if (!(fds[0].revents & G_IO_IN))
373		out_done = TRUE;
374		while (!out_done) {
375		io_status = g_io_channel_read_line (out_pipe, &line, NULL, NULL, error);
376		if (io_status == G_IO_STATUS_NORMAL) {
377		if (prog_extract && prog_extract (line, &completion))
378		bd_utils_report_progress (progress_id, completion, NULL);
379		else
380		g_string_append (stdout_data, line);
381		g_free (line);
382		} else if (io_status == G_IO_STATUS_EOF) {
383		out_done = TRUE;
384		} else if (error && (*error)) {
	463	success = FALSE;
	464	break;
	465	}
	466
	467	if (!out_done) {
	468	if (! _process_fd_event (out_fd, &fds[0], stdout_buffer, stdout_data, &stdout_buffer_pos, &out_done, progress_id, &completion, prog_extract, error)) {
385	469	bd_utils_report_finished (progress_id, (*error)->message);
386		g_io_channel_shutdown (out_pipe, FALSE, NULL);
387		g_io_channel_unref (out_pipe);
388		g_io_channel_shutdown (err_pipe, FALSE, NULL);
389		g_io_channel_unref (err_pipe);
390		g_string_free (stdout_data, TRUE);
391		g_string_free (stderr_data, TRUE);
392		return FALSE;
	470	success = FALSE;
	471	break;
393	472	}
394	473	}
395		if (!(fds[1].revents & G_IO_IN))
396		err_done = TRUE;
397		while (!err_done) {
398		io_status = g_io_channel_read_line (err_pipe, &line, NULL, NULL, error);
399		if (io_status == G_IO_STATUS_NORMAL) {
400		g_string_append (stderr_data, line);
401		g_free (line);
402		} else if (io_status == G_IO_STATUS_EOF) {
403		err_done = TRUE;
404		} else if (error && (*error)) {
	474
	475	if (!err_done) {
	476	if (! _process_fd_event (err_fd, &fds[1], stderr_buffer, stderr_data, &stderr_buffer_pos, &err_done, progress_id, &completion, prog_extract, error)) {
405	477	bd_utils_report_finished (progress_id, (*error)->message);
406		g_io_channel_shutdown (out_pipe, FALSE, NULL);
407		g_io_channel_unref (out_pipe);
408		g_io_channel_shutdown (err_pipe, FALSE, NULL);
409		g_io_channel_unref (err_pipe);
410		g_string_free (stdout_data, TRUE);
411		g_string_free (stderr_data, TRUE);
412		return FALSE;
	478	success = FALSE;
	479	break;
413	480	}
414	481	}
415	482	}
416	483
	484	g_string_free (stdout_buffer, TRUE);
	485	g_string_free (stderr_buffer, TRUE);
	486	close (out_fd);
	487	close (err_fd);
	488
417	489	child_ret = waitpid (pid, &status, 0);
418		*proc_status = WEXITSTATUS(status);
419		if (child_ret > 0) {
420		if (*proc_status != 0) {
421		if (stderr_data->str && (g_strcmp0 ("", stderr_data->str) != 0))
422		msg = stderr_data->str;
423		else
424		msg = stdout_data->str;
425		g_set_error (error, BD_UTILS_EXEC_ERROR, BD_UTILS_EXEC_ERROR_FAILED,
426		"Process reported exit code %d: %s", *proc_status, msg);
427		bd_utils_report_finished (progress_id, (*error)->message);
428		g_io_channel_shutdown (out_pipe, FALSE, NULL);
429		g_io_channel_unref (out_pipe);
430		g_io_channel_shutdown (err_pipe, FALSE, NULL);
431		g_io_channel_unref (err_pipe);
432		g_string_free (stdout_data, TRUE);
433		g_string_free (stderr_data, TRUE);
434		return FALSE;
435		}
436		if (WIFSIGNALED(status)) {
437		g_set_error (error, BD_UTILS_EXEC_ERROR, BD_UTILS_EXEC_ERROR_FAILED,
438		"Process killed with a signal");
439		bd_utils_report_finished (progress_id, (*error)->message);
440		g_io_channel_shutdown (out_pipe, FALSE, NULL);
441		g_io_channel_unref (out_pipe);
442		g_io_channel_shutdown (err_pipe, FALSE, NULL);
443		g_io_channel_unref (err_pipe);
444		g_string_free (stdout_data, TRUE);
445		g_string_free (stderr_data, TRUE);
446		return FALSE;
447		}
448		} else if (child_ret == -1) {
449		if (errno != ECHILD) {
450		errno = 0;
451		g_set_error (error, BD_UTILS_EXEC_ERROR, BD_UTILS_EXEC_ERROR_FAILED,
452		"Failed to wait for the process");
453		bd_utils_report_finished (progress_id, (*error)->message);
454		g_io_channel_shutdown (out_pipe, FALSE, NULL);
455		g_io_channel_unref (out_pipe);
456		g_io_channel_shutdown (err_pipe, FALSE, NULL);
457		g_io_channel_unref (err_pipe);
458		g_string_free (stdout_data, TRUE);
459		g_string_free (stderr_data, TRUE);
460		return FALSE;
461		}
462		/* no such process (the child exited before we tried to wait for it) */
463		errno = 0;
464		}
465
466		bd_utils_report_finished (progress_id, "Completed");
	490	*proc_status = WEXITSTATUS (status);
	491	if (success) {
	492	if (child_ret > 0) {
	493	if (*proc_status != 0) {
	494	msg = stderr_data->len > 0 ? stderr_data->str : stdout_data->str;
	495	g_set_error (error, BD_UTILS_EXEC_ERROR, BD_UTILS_EXEC_ERROR_FAILED,
	496	"Process reported exit code %d: %s", *proc_status, msg);
	497	bd_utils_report_finished (progress_id, (*error)->message);
	498	success = FALSE;
	499	} else if (WIFSIGNALED (status)) {
	500	g_set_error (error, BD_UTILS_EXEC_ERROR, BD_UTILS_EXEC_ERROR_FAILED,
	501	"Process killed with a signal");
	502	bd_utils_report_finished (progress_id, (*error)->message);
	503	success = FALSE;
	504	}
	505	} else if (child_ret == -1) {
	506	if (errno != ECHILD) {
	507	errno = 0;
	508	g_set_error (error, BD_UTILS_EXEC_ERROR, BD_UTILS_EXEC_ERROR_FAILED,
	509	"Failed to wait for the process");
	510	bd_utils_report_finished (progress_id, (*error)->message);
	511	success = FALSE;
	512	} else {
	513	/* no such process (the child exited before we tried to wait for it) */
	514	errno = 0;
	515	}
	516	}
	517	if (success)
	518	bd_utils_report_finished (progress_id, "Completed");
	519	}
467	520	log_out (task_id, stdout_data->str, stderr_data->str);
468	521	log_done (task_id, *proc_status);
469	522
470		/* we don't care about the status here */
471		g_io_channel_shutdown (out_pipe, FALSE, error);
472		g_io_channel_unref (out_pipe);
473		g_io_channel_shutdown (err_pipe, FALSE, error);
474		g_io_channel_unref (err_pipe);
475
476		if (stdout)
	523	if (success && stdout)
477	524	*stdout = g_string_free (stdout_data, FALSE);
478	525	else
479	526	g_string_free (stdout_data, TRUE);
480		if (stderr)
	527	if (success && stderr)
481	528	*stderr = g_string_free (stderr_data, FALSE);
482	529	else
483	530	g_string_free (stderr_data, TRUE);
484	531
485		return TRUE;
	532	return success;
486	533	}
487	534
488	535	/**
489	536	* bd_utils_exec_and_report_progress:
490	537	* @argv: (array zero-terminated=1): the argv array for the call
491	538	* @extra: (allow-none) (array zero-terminated=1): extra arguments
492		* @prog_extract: (scope notified): function for extracting progress information
	539	* @prog_extract: (scope notified) (nullable): function for extracting progress information
493	540	* @proc_status: (out): place to store the process exit status
494	541	* @error: (out): place to store error (if any)
	542	*
	543	* Note that any NULL bytes read from standard output and standard error
	544	* output are treated as separators similar to newlines and @prog_extract
	545	* will be called with the respective chunk.
495	546	*
496	547	* Returns: whether the @argv was successfully executed (no error and exit code 0) or not
497	548	*/

505	556	* @extra: (allow-none) (array zero-terminated=1): extra arguments
506	557	* @output: (out): variable to store output to
507	558	* @error: (out): place to store error (if any)
	559	*
	560	* Note that any NULL bytes read from standard output and standard error
	561	* output will be discarded.
508	562	*
509	563	* Returns: whether the @argv was successfully executed capturing the output or not
510	564	*/

536	590	g_free (stderr);
537	591	return TRUE;
538	592	}
539
540	593	}
541	594
542	595	/**

+22

-2

src/utils/exec.h less more

30	30
31	31	/**
32	32	* BDUtilsProgExtract:
33		* @line: line from extract progress from
	33	* @line: line to extract progress from
34	34	* @completion: (out): percentage of completion
35	35	*
36		* Returns: whether the line was a progress reporting line or not
	36	* Callback function used to process a line captured from spawned command's standard
	37	* output and standard error output. Typically used to extract completion percentage
	38	* of a long-running job.
	39	*
	40	* Note that both outputs are read simultaneously with no guarantees of message order
	41	* this function is called with.
	42	*
	43	* The value the @completion points to may contain value previously returned from
	44	* this callback or zero when called for the first time. This is useful for extractors
	45	* where only some kind of a tick mark is printed out as a progress and previous value
	46	* is needed to compute an incremented value. It's important to keep in mind that this
	47	* function is only called over lines, i.e. progress reporting printing out tick marks
	48	* (e.g. dots) without a newline character might not work properly.
	49	*
	50	* The @line string usually contains trailing newline character, which may be absent
	51	* however in case the spawned command exits without printing one. It's guaranteed
	52	* this function is called over remaining buffer no matter what the trailing
	53	* character is.
	54	*
	55	* Returns: whether the line was a progress reporting line and should be excluded
	56	* from the collected standard output string or not.
37	57	*/
38	58	typedef gboolean (BDUtilsProgExtract) (const gchar line, guint8 *completion);
39	59

-4

tests/library_test.py less more

348	348
349	349	# try reinitializing with only some utilities being available and thus
350	350	# only some plugins able to load
351		with fake_path("tests/lib_missing_utils", keep_utils=["swapon", "swapoff", "mkswap", "lvm",
352		"thin_metadata_size", "swaplabel"]):
	351	with fake_path("tests/lib_missing_utils", keep_utils=["swapon", "swapoff", "mkswap", "lvm", "swaplabel"]):
353	352	succ, loaded = BlockDev.try_reinit(self.requested_plugins, True, None)
354	353	self.assertFalse(succ)
355	354	for plug_name in ("swap", "lvm", "crypto"):

360	359
361	360	# now the same with a subset of plugins requested
362	361	plugins = BlockDev.plugin_specs_from_names(["lvm", "swap", "crypto"])
363		with fake_path("tests/lib_missing_utils", keep_utils=["swapon", "swapoff", "mkswap", "lvm",
364		"thin_metadata_size", "swaplabel"]):
	362	with fake_path("tests/lib_missing_utils", keep_utils=["swapon", "swapoff", "mkswap", "lvm","swaplabel"]):
365	363	succ, loaded = BlockDev.try_reinit(plugins, True, None)
366	364	self.assertTrue(succ)
367	365	self.assertEqual(set(loaded), set(["swap", "lvm", "crypto"]))

+17

-17

tests/lvm_dbus_tests.py less more

140	140	def test_get_thpool_meta_size(self):
141	141	"""Verify that getting recommended thin pool metadata size works as expected"""
142	142
143		# no idea how thin_metadata_size works, but let's at least check that
144		# the function works and returns what thin_metadata_size says
145		out1 = subprocess.check_output(["thin_metadata_size", "-ub", "-n", "-b64k", "-s1t", "-m100"])
146		self.assertEqual(int(out1), BlockDev.lvm_get_thpool_meta_size (1 * 1024*4, 64 1024, 100))
147
148		out2 = subprocess.check_output(["thin_metadata_size", "-ub", "-n", "-b128k", "-s1t", "-m100"])
149		self.assertEqual(int(out2), BlockDev.lvm_get_thpool_meta_size (1 * 1024*4, 128 1024, 100))
150
151		# twice the chunk_size -> roughly half the metadata needed
152		self.assertAlmostEqual(float(out1) / float(out2), 2, places=2)
153
154		# unless thin_metadata_size gives a value that is not valid (too small)
155		self.assertEqual(BlockDev.lvm_get_thpool_meta_size (100 * 1024*2, 128 1024, 100),
	143	# metadata size is calculated as 64 * pool_size / chunk_size
	144	self.assertEqual(BlockDev.lvm_get_thpool_meta_size(1 * 1024*4, 64 1024), 1 * 1024**3)
	145
	146	self.assertEqual(BlockDev.lvm_get_thpool_meta_size(1 * 1024*4, 128 1024), 512 * 1024**2)
	147
	148	# lower limit is 4 MiB
	149	self.assertEqual(BlockDev.lvm_get_thpool_meta_size(100 * 1024*2, 128 1024),
156	150	BlockDev.LVM_MIN_THPOOL_MD_SIZE)
	151
	152	# upper limit is 31.62 GiB
	153	self.assertEqual(BlockDev.lvm_get_thpool_meta_size(100 * 1024*4, 64 1024),
	154	BlockDev.LVM_MAX_THPOOL_MD_SIZE)
157	155
158	156	@tag_test(TestTags.NOSTORAGE)
159	157	def test_is_valid_thpool_md_size(self):
160	158	"""Verify that is_valid_thpool_md_size works as expected"""
161	159
162		self.assertTrue(BlockDev.lvm_is_valid_thpool_md_size(2 * 1024**2))
163		self.assertTrue(BlockDev.lvm_is_valid_thpool_md_size(3 * 1024**2))
164		self.assertTrue(BlockDev.lvm_is_valid_thpool_md_size(16 * 1024**3))
	160	self.assertTrue(BlockDev.lvm_is_valid_thpool_md_size(4 * 1024**2))
	161	self.assertTrue(BlockDev.lvm_is_valid_thpool_md_size(5 * 1024**2))
	162	self.assertTrue(BlockDev.lvm_is_valid_thpool_md_size(15 * 1024**3))
165	163
166	164	self.assertFalse(BlockDev.lvm_is_valid_thpool_md_size(1 * 1024**2))
167		self.assertFalse(BlockDev.lvm_is_valid_thpool_md_size(17 * 1024**3))
	165	self.assertFalse(BlockDev.lvm_is_valid_thpool_md_size(3 * 1024**2))
	166	self.assertFalse(BlockDev.lvm_is_valid_thpool_md_size(16 * 1024**3))
	167	self.assertFalse(BlockDev.lvm_is_valid_thpool_md_size(32 * 1024**3))
168	168
169	169	@tag_test(TestTags.NOSTORAGE)
170	170	def test_is_valid_thpool_chunk_size(self):

+14

-17

tests/lvm_test.py less more

133	133	def test_get_thpool_meta_size(self):
134	134	"""Verify that getting recommended thin pool metadata size works as expected"""
135	135
136		# no idea how thin_metadata_size works, but let's at least check that
137		# the function works and returns what thin_metadata_size says
138		out1 = subprocess.check_output(["thin_metadata_size", "-ub", "-n", "-b64k", "-s1t", "-m100"])
139		self.assertEqual(int(out1), BlockDev.lvm_get_thpool_meta_size (1 * 1024*4, 64 1024, 100))
140
141		out2 = subprocess.check_output(["thin_metadata_size", "-ub", "-n", "-b128k", "-s1t", "-m100"])
142		self.assertEqual(int(out2), BlockDev.lvm_get_thpool_meta_size (1 * 1024*4, 128 1024, 100))
143
144		# twice the chunk_size -> roughly half the metadata needed
145		self.assertAlmostEqual(float(out1) / float(out2), 2, places=2)
146
147		# unless thin_metadata_size gives a value that is not valid (too small)
148		self.assertEqual(BlockDev.lvm_get_thpool_meta_size (100 * 1024*2, 128 1024, 100),
	136
	137	self.assertEqual(BlockDev.lvm_get_thpool_meta_size(1 * 1024*4, 64 1024),
	138	1 * 1024**3)
	139
	140	self.assertEqual(BlockDev.lvm_get_thpool_meta_size(1 * 1024*4, 128 1024),
	141	512 * 1024**2)
	142
	143	self.assertEqual(BlockDev.lvm_get_thpool_meta_size(100 * 1024*2, 128 1024),
149	144	BlockDev.LVM_MIN_THPOOL_MD_SIZE)
150	145
151	146	@tag_test(TestTags.NOSTORAGE)
152	147	def test_is_valid_thpool_md_size(self):
153	148	"""Verify that is_valid_thpool_md_size works as expected"""
154	149
155		self.assertTrue(BlockDev.lvm_is_valid_thpool_md_size(2 * 1024**2))
156		self.assertTrue(BlockDev.lvm_is_valid_thpool_md_size(3 * 1024**2))
157		self.assertTrue(BlockDev.lvm_is_valid_thpool_md_size(16 * 1024**3))
	150	self.assertTrue(BlockDev.lvm_is_valid_thpool_md_size(4 * 1024**2))
	151	self.assertTrue(BlockDev.lvm_is_valid_thpool_md_size(5 * 1024**2))
	152	self.assertTrue(BlockDev.lvm_is_valid_thpool_md_size(15 * 1024**3))
158	153
159	154	self.assertFalse(BlockDev.lvm_is_valid_thpool_md_size(1 * 1024**2))
160		self.assertFalse(BlockDev.lvm_is_valid_thpool_md_size(17 * 1024**3))
	155	self.assertFalse(BlockDev.lvm_is_valid_thpool_md_size(3 * 1024**2))
	156	self.assertFalse(BlockDev.lvm_is_valid_thpool_md_size(16 * 1024**3))
	157	self.assertFalse(BlockDev.lvm_is_valid_thpool_md_size(32 * 1024**3))
161	158
162	159	@tag_test(TestTags.NOSTORAGE)
163	160	def test_is_valid_thpool_chunk_size(self):

-1

tests/utils.py less more

69	69	finally:
70	70	os.environ["PATH"] = old_path
71	71
72		ALL_UTILS = {"lvm", "thin_metadata_size", "btrfs", "mkswap", "swaplabel", "multipath", "mpathconf", "dmsetup", "mdadm", "make-bcache", "sgdisk", "sfdisk"}
	72	ALL_UTILS = {"lvm", "btrfs", "mkswap", "swaplabel", "multipath", "mpathconf", "dmsetup", "mdadm", "make-bcache", "sgdisk", "sfdisk"}
73	73
74	74	@contextmanager
75	75	def fake_path(path=None, keep_utils=None, all_but=None):

+159

-1

tests/utils_test.py less more

0	0	import unittest
1	1	import re
2	2	import os
	3	import six
3	4	import overrides_hack
4		from utils import fake_utils, create_sparse_tempfile, create_lio_device, delete_lio_device, run_command, TestTags, tag_test
	5	from utils import fake_utils, create_sparse_tempfile, create_lio_device, delete_lio_device, run_command, TestTags, tag_test, read_file
5	6
6	7	from gi.repository import BlockDev, GLib
7	8

64	65	succ = BlockDev.utils_exec_and_report_error(["true"])
65	66	self.assertTrue(succ)
66	67
	68	with six.assertRaisesRegex(self, GLib.GError, r"Process reported exit code 1"):
	69	succ = BlockDev.utils_exec_and_report_error(["/bin/false"])
	70
67	71	succ, out = BlockDev.utils_exec_and_capture_output(["echo", "hi"])
68	72	self.assertTrue(succ)
69	73	self.assertEqual(out, "hi\n")

168	172
169	173	# exit code != 0
170	174	self.assertTrue(BlockDev.utils_check_util_version("libblockdev-fake-util-fail", "1.1", "version", "Version:\\s(.*)"))
	175
	176	@tag_test(TestTags.NOSTORAGE, TestTags.CORE)
	177	def test_exec_locale(self):
	178	"""Verify that setting locale for exec functions works as expected"""
	179
	180	succ, out = BlockDev.utils_exec_and_capture_output(["locale"])
	181	self.assertTrue(succ)
	182	self.assertIn("LC_ALL=C", out)
	183
	184	@tag_test(TestTags.NOSTORAGE, TestTags.CORE)
	185	def test_exec_buffer_bloat(self):
	186	"""Verify that very large output from a command is handled properly"""
	187
	188	# easy 64kB of data
	189	cnt = 65536
	190	succ, out = BlockDev.utils_exec_and_capture_output(["bash", "-c", "for i in {1..%d}; do echo -n .; done" % cnt])
	191	self.assertTrue(succ)
	192	self.assertEquals(len(out), cnt)
	193
	194	succ, out = BlockDev.utils_exec_and_capture_output(["bash", "-c", "for i in {1..%d}; do echo -n .; echo -n \# >&2; done" % cnt])
	195	self.assertTrue(succ)
	196	self.assertEquals(len(out), cnt)
	197
	198	# now exceed the system pipe buffer size
	199	# pipe(7): The maximum size (in bytes) of individual pipes that can be set by users without the CAP_SYS_RESOURCE capability.
	200	cnt = int(read_file("/proc/sys/fs/pipe-max-size")) + 100
	201	self.assertGreater(cnt, 0)
	202
	203	succ, out = BlockDev.utils_exec_and_capture_output(["bash", "-c", "for i in {1..%d}; do echo -n .; done" % cnt])
	204	self.assertTrue(succ)
	205	self.assertEquals(len(out), cnt)
	206
	207	succ, out = BlockDev.utils_exec_and_capture_output(["bash", "-c", "for i in {1..%d}; do echo -n .; echo -n \# >&2; done" % cnt])
	208	self.assertTrue(succ)
	209	self.assertEquals(len(out), cnt)
	210
	211	# make use of some newlines
	212	succ, out = BlockDev.utils_exec_and_capture_output(["bash", "-c", "for i in {1..%d}; do echo -n .; if [ $(($i%%500)) -eq 0 ]; then echo ''; fi; done" % cnt])
	213	self.assertTrue(succ)
	214	self.assertGreater(len(out), cnt)
	215
	216	succ, out = BlockDev.utils_exec_and_capture_output(["bash", "-c", "for i in {1..%d}; do echo -n .; echo -n \# >&2; if [ $(($i%%500)) -eq 0 ]; then echo ''; echo '' >&2; fi; done" % cnt])
	217	self.assertTrue(succ)
	218	self.assertGreater(len(out), cnt)
	219
	220
	221	EXEC_PROGRESS_MSG = "Aloha, I'm the progress line you should match."
	222
	223	def my_exec_progress_func_concat(self, line):
	224	"""Expect an concatenated string"""
	225	s = ""
	226	for i in range(10):
	227	s += self.EXEC_PROGRESS_MSG
	228	self.assertEqual(line, s)
	229	self.num_matches += 1
	230	return 0
	231
	232	def my_exec_progress_func(self, line):
	233	self.assertTrue(re.match(r".%s." % self.EXEC_PROGRESS_MSG, line))
	234	self.num_matches += 1
	235	return 0
	236
	237	def test_exec_buffer_bloat_progress(self):
	238	"""Verify that progress reporting can handle large output"""
	239
	240	# no newlines, should match just a single occurrence on EOF
	241	cnt = 10
	242	self.num_matches = 0
	243	status = BlockDev.utils_exec_and_report_progress(["bash", "-c", "for i in {1..10}; do echo -n \"%s\"; done" % self.EXEC_PROGRESS_MSG], None, self.my_exec_progress_func_concat)
	244	self.assertTrue(status)
	245	self.assertEqual(self.num_matches, 1)
	246
	247	# ten matches
	248	self.num_matches = 0
	249	status = BlockDev.utils_exec_and_report_progress(["bash", "-c", "for i in {1..%d}; do echo \"%s\"; done" % (cnt, self.EXEC_PROGRESS_MSG)], None, self.my_exec_progress_func)
	250	self.assertTrue(status)
	251	self.assertEqual(self.num_matches, cnt)
	252
	253	# 100k matches
	254	cnt = 100000
	255	self.num_matches = 0
	256	status = BlockDev.utils_exec_and_report_progress(["bash", "-c", "for i in {1..%d}; do echo \"%s\"; done" % (cnt, self.EXEC_PROGRESS_MSG)], None, self.my_exec_progress_func)
	257	self.assertTrue(status)
	258	self.assertEqual(self.num_matches, cnt)
	259
	260	# 100k matches on stderr
	261	self.num_matches = 0
	262	status = BlockDev.utils_exec_and_report_progress(["bash", "-c", "for i in {1..%d}; do echo \"%s\" >&2; done" % (cnt, self.EXEC_PROGRESS_MSG)], None, self.my_exec_progress_func)
	263	self.assertTrue(status)
	264	self.assertEqual(self.num_matches, cnt)
	265
	266	# 100k matches on stderr and stdout
	267	self.num_matches = 0
	268	status = BlockDev.utils_exec_and_report_progress(["bash", "-c", "for i in {1..%d}; do echo \"%s\"; echo \"%s\" >&2; done" % (cnt, self.EXEC_PROGRESS_MSG, self.EXEC_PROGRESS_MSG)], None, self.my_exec_progress_func)
	269	self.assertTrue(status)
	270	self.assertEqual(self.num_matches, cnt * 2)
	271
	272	# stdout and stderr output, non-zero return from the command and very long exception message
	273	self.num_matches = 0
	274	with six.assertRaisesRegex(self, GLib.GError, r"Process reported exit code 66"):
	275	status = BlockDev.utils_exec_and_report_progress(["bash", "-c", "for i in {1..%d}; do echo \"%s\"; echo \"%s\" >&2; done; exit 66" % (cnt, self.EXEC_PROGRESS_MSG, self.EXEC_PROGRESS_MSG)], None, self.my_exec_progress_func)
	276	self.assertEqual(self.num_matches, cnt * 2)
	277
	278	# no progress reporting callback given, tests slightly different code path
	279	status = BlockDev.utils_exec_and_report_progress(["bash", "-c", "for i in {1..%d}; do echo \"%s\"; echo \"%s\" >&2; done" % (cnt, self.EXEC_PROGRESS_MSG, self.EXEC_PROGRESS_MSG)], None, None)
	280	self.assertTrue(status)
	281
	282	def test_exec_null_bytes(self):
	283	"""Verify that null bytes in the output are processed properly"""
	284
	285	TEST_DATA = ["First line", "Second line", "Third line"]
	286
	287	status, out = BlockDev.utils_exec_and_capture_output(["bash", "-c", "echo -e \"%s\\0%s\\n%s\"" % (TEST_DATA[0], TEST_DATA[1], TEST_DATA[2])])
	288	self.assertTrue(status)
	289	self.assertTrue(TEST_DATA[0] in out)
	290	self.assertTrue(TEST_DATA[1] in out)
	291	self.assertTrue(TEST_DATA[2] in out)
	292	self.assertFalse("kuku!" in out)
	293
	294	# ten matches
	295	cnt = 10
	296	self.num_matches = 0
	297	status = BlockDev.utils_exec_and_report_progress(["bash", "-c", "for i in {1..%d}; do echo -e \"%s\\0%s\"; done" % (cnt, self.EXEC_PROGRESS_MSG, self.EXEC_PROGRESS_MSG)], None, self.my_exec_progress_func)
	298	self.assertTrue(status)
	299	self.assertEqual(self.num_matches, cnt * 2)
	300
	301	# 100k matches
	302	cnt = 100000
	303	self.num_matches = 0
	304	status = BlockDev.utils_exec_and_report_progress(["bash", "-c", "for i in {1..%d}; do echo -e \"%s\\0%s\"; done" % (cnt, self.EXEC_PROGRESS_MSG, self.EXEC_PROGRESS_MSG)], None, self.my_exec_progress_func)
	305	self.assertTrue(status)
	306	self.assertEqual(self.num_matches, cnt * 2)
	307
	308	# 100k matches on stderr
	309	self.num_matches = 0
	310	status = BlockDev.utils_exec_and_report_progress(["bash", "-c", "for i in {1..%d}; do echo -e \"%s\\0%s\" >&2; done" % (cnt, self.EXEC_PROGRESS_MSG, self.EXEC_PROGRESS_MSG)], None, self.my_exec_progress_func)
	311	self.assertTrue(status)
	312	self.assertEqual(self.num_matches, cnt * 2)
	313
	314	# 100k matches on stderr and stdout
	315	self.num_matches = 0
	316	status = BlockDev.utils_exec_and_report_progress(["bash", "-c", "for i in {1..%d}; do echo -e \"%s\\0%s\"; echo -e \"%s\\0%s\" >&2; done" % (cnt, self.EXEC_PROGRESS_MSG, self.EXEC_PROGRESS_MSG, self.EXEC_PROGRESS_MSG, self.EXEC_PROGRESS_MSG)], None, self.my_exec_progress_func)
	317	self.assertTrue(status)
	318	self.assertEqual(self.num_matches, cnt * 4)
	319
	320	# stdout and stderr output, non-zero return from the command and very long exception message
	321	self.num_matches = 0
	322	with six.assertRaisesRegex(self, GLib.GError, r"Process reported exit code 66"):
	323	status = BlockDev.utils_exec_and_report_progress(["bash", "-c", "for i in {1..%d}; do echo -e \"%s\\0%s\"; echo -e \"%s\\0%s\" >&2; done; exit 66" % (cnt, self.EXEC_PROGRESS_MSG, self.EXEC_PROGRESS_MSG, self.EXEC_PROGRESS_MSG, self.EXEC_PROGRESS_MSG)], None, self.my_exec_progress_func)
	324	self.assertEqual(self.num_matches, cnt * 4)
	325
	326	# no progress reporting callback given, tests slightly different code path
	327	status = BlockDev.utils_exec_and_report_progress(["bash", "-c", "for i in {1..%d}; do echo -e \"%s\\0%s\"; echo -e \"%s\\0%s\" >&2; done" % (cnt, self.EXEC_PROGRESS_MSG, self.EXEC_PROGRESS_MSG, self.EXEC_PROGRESS_MSG, self.EXEC_PROGRESS_MSG)], None, None)
	328	self.assertTrue(status)
171	329
172	330
173	331	class UtilsDevUtilsTestCase(UtilsTestCase):