Commit d5a6f70013efa675bbe37cbb40ce8c30894b733f - tcmu

New upstream version 1.5.4 Sébastien Delafond 3 years ago

27 changed file(s) with 1483 addition(s) and 590 deletion(s). Raw diff Collapse all Expand all

+115

-105

CMakeLists.txt less more

0	0	cmake_minimum_required (VERSION 2.8 FATAL_ERROR)
1	1	project (tcmu-runner C)
2		set(VERSION 1.5.2)
	2	set(VERSION 1.5.4)
3	3	set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Wall -Wdeclaration-after-statement -std=c99")
4	4
5	5	include(GNUInstallDirs)

67	67	${LIBNL_LIB}
68	68	${LIBNL_GENL_LIB}
69	69	${GLIB_LIBRARIES}
	70	${PTHREAD}
70	71	${TCMALLOC_LIB}
71	72	)
72	73	install(TARGETS tcmu LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})

95	96
96	97	# Stuff for building the main binary
97	98	add_executable(tcmu-runner
	99	tcmur_work.c
98	100	tcmur_cmd_handler.c
99	101	tcmur_aio.c
100	102	tcmur_device.c

169	171	)
170	172
171	173	if (with-fbo)
172		# Stuff for building the file optical handler
173		add_library(handler_file_optical
174		SHARED
175		scsi.c
176		file_optical.c
177		)
178		set_target_properties(handler_file_optical
179		PROPERTIES
180		PREFIX ""
181		)
182		target_include_directories(handler_file_optical
183		PUBLIC ${PROJECT_SOURCE_DIR}/ccan
184		)
185		target_link_libraries(handler_file_optical ${PTHREAD} ${TCMALLOC_LIB})
186		install(TARGETS handler_file_optical DESTINATION ${CMAKE_INSTALL_LIBDIR}/tcmu-runner)
	174	# Stuff for building the file optical handler
	175	add_library(handler_file_optical
	176	SHARED
	177	scsi.c
	178	file_optical.c
	179	)
	180
	181	set_target_properties(handler_file_optical
	182	PROPERTIES
	183	PREFIX ""
	184	)
	185
	186	target_include_directories(handler_file_optical
	187	PUBLIC ${PROJECT_SOURCE_DIR}/ccan
	188	)
	189	target_link_libraries(handler_file_optical ${PTHREAD} ${TCMALLOC_LIB})
	190	install(TARGETS handler_file_optical DESTINATION ${CMAKE_INSTALL_LIBDIR}/tcmu-runner)
187	191	endif (with-fbo)
188	192
189	193	# The minimal library consumer

194	198	target_link_libraries(consumer tcmu)
195	199
196	200	if (with-zbc)
197		# Stuff for building the file zbc handler
198		add_library(handler_file_zbc
199		SHARED
200		scsi.c
201		file_zbc.c
202		)
203		set_target_properties(handler_file_zbc
204		PROPERTIES
205		PREFIX ""
206		)
207		target_include_directories(handler_file_zbc
208		PUBLIC ${PROJECT_SOURCE_DIR}/ccan
209		)
210		target_link_libraries(handler_file_zbc ${TCMALLOC_LIB})
211		install(TARGETS handler_file_zbc DESTINATION ${CMAKE_INSTALL_LIBDIR}/tcmu-runner)
	201	# Stuff for building the file zbc handler
	202	add_library(handler_file_zbc
	203	SHARED
	204	scsi.c
	205	file_zbc.c
	206	)
	207	set_target_properties(handler_file_zbc
	208	PROPERTIES
	209	PREFIX ""
	210	)
	211	target_include_directories(handler_file_zbc
	212	PUBLIC ${PROJECT_SOURCE_DIR}/ccan
	213	)
	214	target_link_libraries(handler_file_zbc ${TCMALLOC_LIB})
	215	install(TARGETS handler_file_zbc DESTINATION ${CMAKE_INSTALL_LIBDIR}/tcmu-runner)
212	216	endif (with-zbc)
213	217
214	218	if (with-rbd)
215		find_library(LIBRBD rbd)
216
217		# Stuff for building the rbd handler
218		add_library(handler_rbd
219		SHARED
220		rbd.c
221		)
222		set_target_properties(handler_rbd
223		PROPERTIES
224		PREFIX ""
225		)
226		target_include_directories(handler_rbd
227		PUBLIC ${PROJECT_SOURCE_DIR}/ccan
228		)
229		target_link_libraries(handler_rbd
230		${LIBRBD}
231		${TCMALLOC_LIB}
232		)
233		install(TARGETS handler_rbd DESTINATION ${CMAKE_INSTALL_LIBDIR}/tcmu-runner)
	219	find_library(LIBRBD rbd)
	220
	221	# Stuff for building the rbd handler
	222	add_library(handler_rbd
	223	SHARED
	224	rbd.c
	225	)
	226	set_target_properties(handler_rbd
	227	PROPERTIES
	228	PREFIX ""
	229	)
	230	target_include_directories(handler_rbd
	231	PUBLIC ${PROJECT_SOURCE_DIR}/ccan
	232	)
	233	target_link_libraries(handler_rbd
	234	${LIBRBD}
	235	${TCMALLOC_LIB}
	236	)
	237	install(TARGETS handler_rbd DESTINATION ${CMAKE_INSTALL_LIBDIR}/tcmu-runner)
234	238	endif (with-rbd)
235	239
236	240	if (with-glfs)
237		find_library(GFAPI gfapi)
238
239		set(GFAPI_VERSION760 0)
240
241		pkg_check_modules(GFAPI760 glusterfs-api>=7.6 QUIET)
242		if (GFAPI760_FOUND)
243		set(GFAPI_VERSION760 1)
244		endif (GFAPI760_FOUND)
245
246		# Stuff for building the glfs handler
247		add_library(handler_glfs
248		SHARED
249		glfs.c
250		)
251		set_target_properties(handler_glfs
252		PROPERTIES
253		PREFIX ""
254		)
255		target_include_directories(handler_glfs
256		PUBLIC ${PROJECT_SOURCE_DIR}/ccan
257		)
258		target_link_libraries(handler_glfs
259		${GFAPI}
260		${TCMALLOC_LIB}
261		)
262		install(TARGETS handler_glfs DESTINATION ${CMAKE_INSTALL_LIBDIR}/tcmu-runner)
	241	find_library(GFAPI gfapi)
	242
	243	set(GFAPI_VERSION760 0)
	244
	245	pkg_check_modules(GFAPI760 glusterfs-api>=7.6 QUIET)
	246	if (GFAPI760_FOUND)
	247	set(GFAPI_VERSION760 1)
	248	endif (GFAPI760_FOUND)
	249
	250	set(GFAPI_VERSION766 0)
	251
	252	pkg_check_modules(GFAPI766 glusterfs-api>=7.6.6 QUIET)
	253	if (GFAPI766_FOUND)
	254	set(GFAPI_VERSION766 1)
	255	endif (GFAPI766_FOUND)
	256
	257	# Stuff for building the glfs handler
	258	add_library(handler_glfs
	259	SHARED
	260	glfs.c
	261	)
	262	set_target_properties(handler_glfs
	263	PROPERTIES
	264	PREFIX ""
	265	)
	266	target_include_directories(handler_glfs
	267	PUBLIC ${PROJECT_SOURCE_DIR}/ccan
	268	)
	269	target_link_libraries(handler_glfs
	270	${GFAPI}
	271	${TCMALLOC_LIB}
	272	)
	273	install(TARGETS handler_glfs DESTINATION ${CMAKE_INSTALL_LIBDIR}/tcmu-runner)
263	274	endif (with-glfs)
264	275
265	276	if (with-qcow)
266		find_package(ZLIB REQUIRED)
267
268		# Stuff for building the qcow handler
269		add_library(handler_qcow
270		SHARED
271		qcow.c
272		)
273		set_target_properties(handler_qcow
274		PROPERTIES
275		PREFIX ""
276		)
277		target_include_directories(handler_qcow
278		PUBLIC ${PROJECT_SOURCE_DIR}/ccan
279		)
280
281		CHECK_INCLUDE_FILE("linux/falloc.h" HAVE_LINUX_FALLOC)
282		if (HAVE_LINUX_FALLOC)
283		set_target_properties(handler_qcow
284		PROPERTIES
285		COMPILE_FLAGS "-DHAVE_LINUX_FALLOC"
286		)
287		endif (HAVE_LINUX_FALLOC)
288		target_link_libraries(handler_qcow
289		${ZLIB_LIBRARIES}
290		${TCMALLOC_LIB}
291		)
292		install(TARGETS handler_qcow DESTINATION ${CMAKE_INSTALL_LIBDIR}/tcmu-runner)
	277	find_package(ZLIB REQUIRED)
	278
	279	# Stuff for building the qcow handler
	280	add_library(handler_qcow
	281	SHARED
	282	qcow.c
	283	)
	284	set_target_properties(handler_qcow
	285	PROPERTIES
	286	PREFIX ""
	287	)
	288	target_include_directories(handler_qcow
	289	PUBLIC ${PROJECT_SOURCE_DIR}/ccan
	290	)
	291
	292	CHECK_INCLUDE_FILE("linux/falloc.h" HAVE_LINUX_FALLOC)
	293	if (HAVE_LINUX_FALLOC)
	294	set_target_properties(handler_qcow
	295	PROPERTIES
	296	COMPILE_FLAGS "-DHAVE_LINUX_FALLOC"
	297	)
	298	endif (HAVE_LINUX_FALLOC)
	299	target_link_libraries(handler_qcow
	300	${ZLIB_LIBRARIES}
	301	${TCMALLOC_LIB}
	302	)
	303	install(TARGETS handler_qcow DESTINATION ${CMAKE_INSTALL_LIBDIR}/tcmu-runner)
293	304	endif (with-qcow)
294	305
295	306	# stamp out a header file to pass some of the CMake settings

317	328	if (SUPPORT_SYSTEMD)
318	329	install(FILES tcmu-runner.service DESTINATION /usr/lib/systemd/system/)
319	330	endif (SUPPORT_SYSTEMD)
320		install(FILES tcmu-runner.8
321		DESTINATION ${CMAKE_INSTALL_PREFIX}/share/man/man8)
	331	install(FILES tcmu-runner.8 DESTINATION ${CMAKE_INSTALL_PREFIX}/share/man/man8)

+53

-30

alua.c less more

24	24	#include "libtcmu_priv.h"
25	25	#include "tcmu-runner.h"
26	26	#include "tcmur_device.h"
	27	#include "tcmur_work.h"
27	28	#include "target.h"
28	29	#include "alua.h"
29	30

431	432	* lock state to avoid later blacklist errors.
432	433	*/
433	434	pthread_mutex_lock(&rdev->state_lock);
434		if (rdev->lock_state == TCMUR_DEV_LOCK_LOCKED) {
	435	if (rdev->lock_state == TCMUR_DEV_LOCK_WRITE_LOCKED) {
435	436	tcmu_dev_dbg(dev, "Dropping lock\n");
436	437	rdev->lock_state = TCMUR_DEV_LOCK_UNLOCKED;
437	438	}

541	542	return !!rhandler->lock;
542	543	}
543	544
544		static void alua_lock_thread_fn(void arg)
545		{
	545	static void alua_event_work_fn(void *arg)
	546	{
	547	struct tcmu_device *dev = arg;
	548
546	549	/* TODO: set UA based on bgly's patches */
547		tcmu_acquire_dev_lock(arg, -1);
548		return NULL;
549		}
550
551		int alua_implicit_transition(struct tcmu_device dev, struct tcmulib_cmd cmd)
	550	tcmu_acquire_dev_lock(dev, -1);
	551	}
	552
	553	int alua_implicit_transition(struct tcmu_device dev, struct tcmulib_cmd cmd,
	554	bool is_read)
552	555	{
553	556	struct tcmur_device *rdev = tcmu_dev_get_private(dev);
554		pthread_attr_t attr;
555	557	int ret = TCMU_STS_OK;
556	558
557	559	if (!lock_is_required(dev))
558	560	return ret;
559	561
560	562	pthread_mutex_lock(&rdev->state_lock);
561		if (rdev->lock_state == TCMUR_DEV_LOCK_LOCKED) {
	563	if (rdev->lock_state == TCMUR_DEV_LOCK_WRITE_LOCKED) {
	564	/* For both read/write cases in this state is good */
562	565	goto done;
563		} else if (rdev->lock_state == TCMUR_DEV_LOCK_LOCKING) {
564		tcmu_dev_dbg(dev, "Lock acquisition operation is already in process.\n");
	566	} else if (rdev->lock_state == TCMUR_DEV_LOCK_WRITE_LOCKING) {
	567	/* For both read/write cases in this state should return busy */
	568	tcmu_dev_dbg(dev, "Write lock acquisition operation is already in process.\n");
565	569	ret = TCMU_STS_BUSY;
566	570	goto done;
567		}
568
569		tcmu_dev_info(dev, "Starting lock acquisition operation.\n");
570
571		rdev->lock_state = TCMUR_DEV_LOCK_LOCKING;
572
573		/*
574		* Make the lock_thread as detached to fix the memory leakage bug.
575		*/
576		pthread_attr_init(&attr);
577		pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
	571	} else if (rdev->lock_state == TCMUR_DEV_LOCK_READ_LOCKING) {
	572	/* For both read/write they need to retry */
	573	tcmu_dev_dbg(dev, "Read lock acquisition operation is already in process.\n");
	574	ret = TCMU_STS_BUSY;
	575	goto done;
	576	} else if (is_read) {
	577	if (rdev->lock_state == TCMUR_DEV_LOCK_READ_LOCKED)
	578	goto done;
	579
	580	tcmu_dev_info(dev, "Starting read lock acquisition operation.\n");
	581
	582	/*
	583	* Since we are here, current lock state should be one of:
	584	* TCMUR_DEV_LOCK_UNLOCKED
	585	* TCMUR_DEV_LOCK_UNKNOWN
	586	*
	587	* Will not acquire the lock, just reopen the device.
	588	*/
	589	rdev->lock_state = TCMUR_DEV_LOCK_READ_LOCKING;
	590	} else {
	591	tcmu_dev_info(dev, "Starting write lock acquisition operation.\n");
	592
	593	/*
	594	* Since we are here, current lock state should be one of:
	595	* TCMUR_DEV_LOCK_UNLOCKED
	596	* TCMUR_DEV_LOCK_UNKNOWN
	597	* TCMUR_DEV_LOCK_READ_LOCKING
	598	* TCMUR_DEV_LOCK_READ_LOCKED
	599	*
	600	* May will reopen the deivce and Will acquire the lock later.
	601	*/
	602	rdev->lock_state = TCMUR_DEV_LOCK_WRITE_LOCKING;
	603	}
578	604
579	605	/*
580	606	* The initiator is going to be queueing commands, so do this
581	607	* in the background to avoid command timeouts.
582	608	*/
583		if (pthread_create(&rdev->lock_thread, &attr, alua_lock_thread_fn,
584		dev)) {
	609	if (tcmur_run_work(rdev->event_work, dev, alua_event_work_fn)) {
585	610	tcmu_dev_err(dev, "Could not start implicit transition thread:%s\n",
586	611	strerror(errno));
587	612	rdev->lock_state = TCMUR_DEV_LOCK_UNLOCKED;

589	614	} else {
590	615	ret = TCMU_STS_BUSY;
591	616	}
592
593		pthread_attr_destroy(&attr);
594	617
595	618	done:
596	619	pthread_mutex_unlock(&rdev->state_lock);

767	790	return ret;
768	791	}
769	792
770		int alua_check_state(struct tcmu_device dev, struct tcmulib_cmd cmd)
	793	int alua_check_state(struct tcmu_device dev, struct tcmulib_cmd cmd, bool is_read)
771	794	{
772	795	struct tcmur_device *rdev = tcmu_dev_get_private(dev);
773	796
774	797	if (rdev->failover_type == TCMUR_DEV_FAILOVER_EXPLICIT) {
775		if (rdev->lock_state != TCMUR_DEV_LOCK_LOCKED) {
	798	if (rdev->lock_state != TCMUR_DEV_LOCK_WRITE_LOCKED) {
776	799	tcmu_dev_dbg(dev, "device lock not held.\n");
777	800	return TCMU_STS_FENCED;
778	801	}
779	802	} else if (rdev->failover_type == TCMUR_DEV_FAILOVER_IMPLICIT) {
780		return alua_implicit_transition(dev, cmd);
	803	return alua_implicit_transition(dev, cmd, is_read);
781	804	}
782	805
783	806	return TCMU_STS_OK;

-2

alua.h less more

44	44	struct tgt_port tcmu_get_enabled_port(struct list_head );
45	45	int tcmu_get_alua_grps(struct tcmu_device , struct list_head );
46	46	void tcmu_release_alua_grps(struct list_head *);
47		int alua_implicit_transition(struct tcmu_device dev, struct tcmulib_cmd cmd);
	47	int alua_implicit_transition(struct tcmu_device dev, struct tcmulib_cmd cmd,
	48	bool is_read);
48	49	bool lock_is_required(struct tcmu_device *dev);
49		int alua_check_state(struct tcmu_device dev, struct tcmulib_cmd cmd);
	50	int alua_check_state(struct tcmu_device dev, struct tcmulib_cmd cmd, bool is_read);
50	51
51	52	#endif

-5

api.c less more

313	313	return copied;
314	314	}
315	315
316		#define CDB_TO_BUF_SIZE(bytes) ((bytes) * 3 + 1)
	316	#define CDB_TO_BUF_SIZE(bytes) ((bytes) * 3 + 2)
317	317	#define CDB_FIX_BYTES 64 /* 64 bytes for default */
318	318	#define CDB_FIX_SIZE CDB_TO_BUF_SIZE(CDB_FIX_BYTES)
319	319	void tcmu_cdb_print_info(struct tcmu_device *dev,
320	320	const struct tcmulib_cmd *cmd,
321	321	const char *info)
322	322	{
323		int i, n, bytes;
	323	int i, n, bytes, info_len = 0;
324	324	char fix[CDB_FIX_SIZE], *buf;
325	325
326	326	buf = fix;

329	329	if (bytes < 0)
330	330	return;
331	331
332		if (bytes > CDB_FIX_SIZE) {
333		buf = malloc(CDB_TO_BUF_SIZE(bytes));
	332	if (info)
	333	info_len = strlen(info);
	334
	335	if (CDB_TO_BUF_SIZE(bytes) + info_len > CDB_FIX_SIZE) {
	336	buf = malloc(CDB_TO_BUF_SIZE(bytes) + info_len);
334	337	if (!buf) {
335	338	tcmu_dev_err(dev, "out of memory\n");
336	339	return;

352	355	tcmu_dev_dbg_scsi_cmd(dev, "%s", buf);
353	356	}
354	357
355		if (bytes > CDB_FIX_SIZE)
	358	if (buf != fix)
356	359	free(buf);
357	360	}
358	361

+26

-2

extra/install_dep.sh less more

0		#!/bin/sh
	0	#!/bin/bash
1	1	#
2	2	# Userspace side of the LIO TCM-User backstore
3	3	#

22	22	yum search librbd-devel \| grep -q "N/S matched" && LIBRBD=librbd \|\| LIBRBD=librbd1
23	23	$SUDO yum install -y $LIBRBD-devel
24	24	;;
	25	debian)
	26	# Update APT cache
	27	$SUDO apt update
	28
	29	# for generic
	30	$SUDO apt install -y cmake make gcc zlib1g kmod
	31	$SUDO apt install -y libnl-3-dev libnl-genl-3-dev libglib2.0-0 libkmod-dev libgoogle-perftools-dev
	32
	33	# for glusterfs
	34	$SUDO apt install -y libglusterfs-dev
	35
	36	# for ceph
	37	$SUDO apt install -y librados2 librbd-dev
	38	;;
	39	sles\|opensuse-tumbleweed)
	40	# for generic
	41	$SUDO zypper install -y cmake make gcc libnl3-200 glib2 zlib kmod
	42	$SUDO zypper install -y libnl3-devel glib2-devel zlib-devel libkmod-devel gperftools-devel
	43
	44	#for glusterfs
	45	$SUDO zypper install -y glusterfs-devel glusterfs
	46	#for ceph
	47	$SUDO zypper install -y librbd-devel librados-devel librados2
	48	;;
25	49	*)
26		echo "TODO: only fedora/rhel/centos are supported for now!"
	50	echo "TODO: distro not supported for now!"
27	51	;;
28	52	esac
29	53	else

-0

file_optical.c less more

1003	1003	struct fbo_state *state = tcmur_dev_get_private(dev);
1004	1004	uint8_t sense[SENSE_BUFFERSIZE];
1005	1005
	1006	tcmu_set_thread_name("fbo-cache", dev);
	1007
1006	1008	pthread_mutex_lock(&state->state_mtx);
1007	1009	state->async_cache_count++;
1008	1010	state->flags \|= FBO_BUSY_EVENT;

1318	1320	struct fbo_state *state = tcmur_dev_get_private(dev);
1319	1321	uint8_t sense[SENSE_BUFFERSIZE];
1320	1322
	1323	tcmu_set_thread_name("fbo-format", dev);
	1324
1321	1325	pthread_mutex_lock(&state->state_mtx);
1322	1326	state->flags \|= FBO_BUSY_EVENT \| FBO_FORMAT_IMMED;
1323	1327	pthread_mutex_unlock(&state->state_mtx);

-14

file_zbc.c less more

951	951	case 0x00:
952	952	/* Supported VPD pages */
953	953	data[3] = 5;
954		data[4] = 0x83;
955		data[5] = 0xb0;
956		data[6] = 0xb1;
957		data[7] = 0xb6;
958
959		tcmu_memcpy_into_iovec(iovec, iov_cnt, data, 8);
	954	data[4] = 0x0;
	955	data[5] = 0x83;
	956	data[6] = 0xb0;
	957	data[7] = 0xb1;
	958	data[8] = 0xb6;
	959
	960	tcmu_memcpy_into_iovec(iovec, iov_cnt, data, 9);
960	961	break;
961	962
962	963	case 0x83:

2127	2128
2128	2129	/* Get the zone of the current LBA */
2129	2130	zone = zbc_get_zone(zdev, lba, false);
2130		if (lba + nr_lbas > zone->start + zone->len) {
2131		tcmu_dev_err(dev,
2132		"Write boundary violation lba %"PRIu64", xfer len %zu\n",
2133		lba, nr_lbas);
2134		return tcmu_sense_set_data(cmd->sense_buf,
2135		ILLEGAL_REQUEST,
2136		ASC_WRITE_BOUNDARY_VIOLATION);
2137		}
2138	2131
2139	2132	/* If the zone is not open, implicitly open it */
2140	2133	if (zbc_zone_seq(zone) && !zbc_zone_is_open(zone)) {

+161

-41

glfs.c less more

36	36	/* cache protection */
37	37	pthread_mutex_t glfs_lock;
38	38
	39	#if GFAPI_VERSION766
	40	#define GF_ENFORCE_MANDATORY_LOCK "trusted.glusterfs.enforce-mandatory-lock"
	41	#endif
	42
39	43	typedef enum gluster_transport {
40	44	GLUSTER_TRANSPORT_TCP,
41	45	GLUSTER_TRANSPORT_UNIX,

70	74	glfs_t *fs;
71	75	glfs_fd_t *gfd;
72	76	gluster_server *hosts;
	77	bool no_fencing;
73	78
74	79	/*
75	80	* Current tcmu helper API reports WCE=1, but doesn't

98	103	darray(char *) cfgstring;
99	104	} gluster_cacheconn;
100	105
101		static darray(struct gluster_cacheconn *) glfs_cache = darray_new();
	106	static darray(struct gluster_cacheconn *) glfs_cache;
102	107
103	108	const char *const gluster_transport_lookup[] = {
104	109	[GLUSTER_TRANSPORT_TCP] = "tcp",

529	534	goto unref;
530	535	}
531	536
	537	#if GFAPI_VERSION766
	538	ret = glfs_fsetxattr(gfsp->gfd, GF_ENFORCE_MANDATORY_LOCK, "set", 4, 0);
	539	if (ret) {
	540	if (errno == EINVAL) {
	541	gfsp->no_fencing = true;
	542	} else {
	543	tcmu_dev_err(dev,"glfs_fsetxattr failed: %m\n");
	544	goto close;
	545	}
	546	}
	547	#endif
	548
532	549	ret = glfs_lstat(gfsp->fs, gfsp->hosts->path, &st);
533	550	if (ret) {
534	551	tcmu_dev_warn(dev, "glfs_lstat failed: %m\n");

545	562	if (round_down(st.st_size, block_size) == dev_size)
546	563	goto out;
547	564
548		if (!reopen) {
549		ret = -EINVAL;
550		goto close;
551		}
552
553		/*
554		* If we are here this should be in reopen path,
555		* then we should also update the device size in
556		* kernel.
557		*/
558		tcmu_dev_info(dev,
559		"device size and backing size disagree:device %lld backing %lld\n",
560		dev_size, (long long) st.st_size);
561
	565	if (reopen)
	566	goto out;
	567
	568	tcmu_dev_warn(dev,
	569	"device size (%lld) and backing file size (%lld) not matching, updating it to kernel\n",
	570	(long long)dev_size, (long long) st.st_size);
	571
	572	/* Update the device size in kernel. */
562	573	ret = tcmur_dev_update_size(dev, st.st_size);
563	574	if (ret)
564	575	goto close;
	576
	577	tcmu_dev_info(dev, "loaded with size (%lld)\n", (long long) st.st_size);
565	578	}
566	579
567	580	out:

599	612	glfs_cbk_cookie *cookie = data;
600	613	struct tcmu_device *dev = cookie->dev;
601	614	struct tcmur_cmd *tcmur_cmd = cookie->tcmur_cmd;
	615	#if GFAPI_VERSION766
	616	struct glfs_state *gfsp = tcmur_dev_get_private(dev);
	617	#endif
602	618	size_t length = cookie->length;
603
604		if (ret < 0 \|\| ret != length) {
	619	int err = -errno;
	620
	621	if (ret < 0) {
	622	switch (err) {
	623	case -ETIMEDOUT:
	624	/*
	625	* TIMEDOUT is a scenario where the fop can
	626	* not reach the server for 30 minutes.
	627	*/
	628	tcmu_dev_err(dev, "Timing out cmd after 30 minutes.\n");
	629
	630	tcmu_notify_cmd_timed_out(dev);
	631	ret = TCMU_STS_TIMEOUT;
	632	break;
	633	#if GFAPI_VERSION766
	634	case -EAGAIN:
	635	case -EBUSY:
	636	case -ENOTCONN:
	637	/*
	638	* The lock maybe preemted and then any IO to
	639	* land on the file without a lock will be
	640	* rejected with EBUSY.
	641	*
	642	* And if local is disconnected, we should get
	643	* ENOTCONN for further requests. But if the
	644	* connection is reestablished and at the same
	645	* time another client has taken the lock we
	646	* will get EBUSY too.
	647	*/
	648	if (!gfsp->no_fencing) {
	649	tcmu_dev_dbg(dev, "failed with errno %d.\n", err);
	650	tcmu_notify_lock_lost(dev);
	651	ret = TCMU_STS_BUSY;
	652	break;
	653	}
	654	#endif
	655	default:
	656	tcmu_dev_dbg(dev, "failed with errno %d.\n", err);
	657	ret = TCMU_STS_HW_ERR;
	658	}
	659	} else if (ret != length) {
	660	tcmu_dev_dbg(dev, "ret(%zu) != length(%zu).\n", ret, length);
	661
605	662	/* Read/write/flush failed */
606	663	switch (cookie->op) {
607	664	case TCMU_GLFS_READ:
	665	/* ENOENT for READ operation means EOF,
	666	* see glusterfs commit 9fe5c6d3
	667	*/
	668	if (err == -ENOENT) {
	669	ret = TCMU_STS_OK;
	670	break;
	671	}
608	672	ret = TCMU_STS_RD_ERR;
609	673	break;
610	674	case TCMU_GLFS_WRITE:

703	767	/* Let the targetcli command return success */
704	768	ret = 0;
705	769	} else if (st.st_size != cfg->data.dev_size) {
706		tcmu_dev_err(dev,
707		"device size and backing size disagree: device %"PRId64" backing %lld\n",
708		cfg->data.dev_size, (long long) st.st_size);
709		ret = -EINVAL;
	770	/*
	771	* Currently we cannot update the size to kernel here,
	772	* because it will be overrided by kernel with the old
	773	* value after it gets the genl reply.
	774	*/
	775	tcmu_dev_warn(dev,
	776	"device size (%lld) and backing file size (%lld) not matching, and ignoring it\n",
	777	(long long)cfg->data.dev_size, (long long) st.st_size);
	778	return -EINVAL;
710	779	}
711	780	return ret;
712	781	case TCMULIB_CFG_DEV_CFGSTR:

816	885	return TCMU_STS_NO_RESOURCE;
817	886	}
818	887
819		/*
820		* Return scsi status or TCMU_STS_NOT_HANDLED
821		*/
822		static int tcmu_glfs_handle_cmd(struct tcmu_device *dev,
823		struct tcmur_cmd *tcmur_cmd)
824		{
825		struct tcmulib_cmd *cmd = tcmur_cmd->lib_cmd;
826		uint8_t *cdb = cmd->cdb;
	888	#if GFAPI_VERSION766
	889	static int tcmu_glfs_to_sts(int rc)
	890	{
	891	switch (rc) {
	892	case 0:
	893	return TCMU_STS_OK;
	894	case -ENOTCONN:
	895	return TCMU_STS_FENCED;
	896	default:
	897	return TCMU_STS_HW_ERR;
	898	}
	899	}
	900
	901	static int tcmu_glfs_lock(struct tcmu_device *dev, uint16_t tag)
	902	{
	903	struct glfs_state *state = tcmur_dev_get_private(dev);
	904	struct flock lock;
827	905	int ret;
828	906
829		switch(cdb[0]) {
830		case WRITE_SAME:
831		case WRITE_SAME_16:
832		ret = tcmur_handle_writesame(dev, tcmur_cmd,
833		tcmu_glfs_writesame);
834		break;
835		default:
836		ret = TCMU_STS_NOT_HANDLED;
837		}
838
839		return ret;
	907	if (state->no_fencing)
	908	return 0;
	909
	910	lock.l_type = F_WRLCK \| F_RDLCK;
	911	lock.l_whence = SEEK_SET;
	912	lock.l_start = 0;
	913	lock.l_len = 0;
	914
	915	ret = glfs_file_lock(state->gfd, F_SETLK, &lock, GLFS_LK_MANDATORY);
	916	if (ret)
	917	tcmu_dev_err(dev, "glfs_file_lock failed: %m\n");
	918
	919	return tcmu_glfs_to_sts(ret);
	920	}
	921
	922	static int tcmu_glfs_unlock(struct tcmu_device *dev)
	923	{
	924	struct glfs_state *state = tcmur_dev_get_private(dev);
	925	struct flock lock;
	926	int ret;
	927
	928	if (state->no_fencing)
	929	return 0;
	930
	931	lock.l_type = F_UNLCK;
	932	lock.l_whence = SEEK_SET;
	933	lock.l_start = 0;
	934	lock.l_len = 0;
	935
	936	ret = glfs_file_lock(state->gfd, F_SETLK, &lock, GLFS_LK_MANDATORY);
	937	if (ret)
	938	tcmu_dev_err(dev, "glfs_file_lock failed: %m\n");
	939
	940	return tcmu_glfs_to_sts(ret);
	941	}
	942	#endif
	943
	944	static int tcmu_glfs_init(void)
	945	{
	946	darray_init(glfs_cache);
	947	return 0;
	948	}
	949
	950	static void tcmu_glfs_destroy(void)
	951	{
	952	darray_free(glfs_cache);
840	953	}
841	954
842	955	/*

871	984	.reconfig = tcmu_glfs_reconfig,
872	985	.flush = tcmu_glfs_flush,
873	986	.unmap = tcmu_glfs_discard,
874		.handle_cmd = tcmu_glfs_handle_cmd,
	987	.writesame = tcmu_glfs_writesame,
875	988
876	989	.update_logdir = tcmu_glfs_update_logdir,
	990
	991	#if GFAPI_VERSION766
	992	.lock = tcmu_glfs_lock,
	993	.unlock = tcmu_glfs_unlock,
	994	#endif
	995	.init = tcmu_glfs_init,
	996	.destroy = tcmu_glfs_destroy,
877	997	};
878	998
879	999	/* Entry point must be named "handler_init". */

+278

-114

libtcmu.c less more

42	42	[TCMU_ATTR_SUPP_KERN_CMD_REPLY] = { .type = NLA_U8 },
43	43	};
44	44
45		static int add_device(struct tcmulib_context ctx, char dev_name,
	45	static int device_add(struct tcmulib_context ctx, char dev_name,
46	46	char *cfgstring, bool reopen);
47		static void remove_device(struct tcmulib_context ctx, char dev_name,
	47	static void device_remove(struct tcmulib_context ctx, char dev_name,
48	48	bool should_block);
49	49	static int handle_netlink(struct nl_cache_ops unused, struct genl_cmd cmd,
50	50	struct genl_info info, void arg);

216	216	switch (cmd->c_id) {
217	217	case TCMU_CMD_ADDED_DEVICE:
218	218	reply_cmd = TCMU_CMD_ADDED_DEVICE_DONE;
219		ret = add_device(ctx, buf,
	219	ret = device_add(ctx, buf,
220	220	nla_get_string(info->attrs[TCMU_ATTR_DEVICE]),
221	221	false);
222	222	break;
223	223	case TCMU_CMD_REMOVED_DEVICE:
224	224	reply_cmd = TCMU_CMD_REMOVED_DEVICE_DONE;
225		remove_device(ctx, buf, false);
	225	device_remove(ctx, buf, false);
226	226	ret = 0;
227	227	break;
228	228	case TCMU_CMD_RECONFIG_DEVICE:

392	392	return dev->map->flags & TCMU_MAILBOX_FLAG_CAP_OOOC;
393	393	}
394	394
395		static int add_device(struct tcmulib_context ctx, char dev_name,
396		char *cfgstring, bool reopen)
397		{
398		struct tcmu_device *dev;
399		struct tcmu_mailbox *mb;
400		char str_buf[256];
401		bool reset_supp = true;
	395	/* Read a size_t from a file. Returns -1 on error. */
	396	static ssize_t read_size(const char *filename)
	397	{
402	398	int fd;
403		int ret;
404		char ptr, oldptr;
405		char *reason = NULL;
	399	int len, rc;
	400	char buf[256], *endbuf;
	401	ssize_t ret;
	402
	403	fd = open(filename, O_RDONLY);
	404	if (fd == -1)
	405	goto err;
	406
	407	len = read(fd, buf, sizeof(buf)-1);
	408	rc = close(fd);
	409	if (len <= 0 \|\| rc == -1)
	410	goto err;
	411
	412	buf[len] = '\0'; /* null-terminate */
	413	ret = strtoull(buf, &endbuf, 0);
	414	if (buf == endbuf \|\| ret == ULLONG_MAX)
	415	goto err;
	416
	417	return ret;
	418
	419	err:
	420	tcmu_warn("cannot read size from %s\n", filename);
	421	return -1;
	422	}
	423
	424	/* Extract configuration parameters into dev. */
	425	static bool device_parse_cfg(struct tcmu_device *dev,
	426	const char dev_name, const char cfgstring)
	427	{
406	428	int len;
407
408		dev = calloc(1, sizeof(*dev));
409		if (!dev) {
410		tcmu_err("calloc failed in add_device\n");
411		return -ENOMEM;
412		}
413
414		snprintf(dev->dev_name, sizeof(dev->dev_name), "%s", dev_name);
415
	429	const char ptr, oldptr;
	430
	431	len = snprintf(dev->dev_name, sizeof(dev->dev_name), "%s", dev_name);
	432	if (len >= sizeof(dev->dev_name)) {
	433	tcmu_err("device name too long for tcmu_device\n");
	434	goto err_recompile;
	435	}
	436
	437	/* Check valid cfgstring */
416	438	oldptr = cfgstring;
417	439	ptr = strchr(oldptr, '/');
418		if (!ptr) {
419		tcmu_err("invalid cfgstring\n");
420		goto err_free;
421		}
422
423		if (strncmp(cfgstring, "tcm-user", ptr-oldptr)) {
424		tcmu_err("invalid cfgstring\n");
425		goto err_free;
426		}
	440	if (!ptr)
	441	goto err_badcfg;
	442	if (strncmp(cfgstring, "tcm-user", ptr-oldptr))
	443	goto err_badcfg;
427	444
428	445	/* Get HBA name */
429	446	oldptr = ptr+1;
430	447	ptr = strchr(oldptr, '/');
431		if (!ptr) {
432		tcmu_err("invalid cfgstring\n");
433		goto err_free;
434		}
	448	if (!ptr)
	449	goto err_badcfg;
435	450	len = ptr-oldptr;
436		snprintf(dev->tcm_hba_name, sizeof(dev->tcm_hba_name), "user_%.*s", len, oldptr);
	451	len = snprintf(dev->tcm_hba_name, sizeof(dev->tcm_hba_name), "user_%.*s", len, oldptr);
	452	if (len >= sizeof(dev->tcm_hba_name)) {
	453	tcmu_err("hba name too long for tcmu_device\n");
	454	goto err_recompile;
	455	}
437	456
438	457	/* Get device name */
439	458	oldptr = ptr+1;
440	459	ptr = strchr(oldptr, '/');
441		if (!ptr) {
442		tcmu_err("invalid cfgstring\n");
443		goto err_free;
444		}
	460	if (!ptr)
	461	goto err_badcfg;
445	462	len = ptr-oldptr;
446		snprintf(dev->tcm_dev_name, sizeof(dev->tcm_dev_name), "%.*s", len, oldptr);
	463	len = snprintf(dev->tcm_dev_name, sizeof(dev->tcm_dev_name), "%.*s", len, oldptr);
	464	if (len >= sizeof(dev->tcm_dev_name)) {
	465	tcmu_err("tcm device name too long for tcmu_device\n");
	466	goto err_recompile;
	467	}
447	468
448	469	/* The rest is the handler-specific cfgstring */
449	470	oldptr = ptr+1;
450		ptr = strchr(oldptr, '/');
451		snprintf(dev->cfgstring, sizeof(dev->cfgstring), "%s", oldptr);
	471	len = snprintf(dev->cfgstring, sizeof(dev->cfgstring), "%s", oldptr);
	472	if (len >= sizeof(dev->cfgstring)) {
	473	tcmu_warn("additional handler cfgstring was truncated\n");
	474	/* not a terminal error. snprintf() will null-terminate */
	475	}
	476	return true;
	477
	478	err_badcfg:
	479	tcmu_err("invalid cfgstring: expecting \"tcm-user/<hba_name>/<tcm_device_name>/<handler_name_config>\"\n");
	480	err_recompile: /* consider expanding string lengths in dev */
	481	return false;
	482	}
	483
	484	static void device_close_shm(struct tcmu_device *dev)
	485	{
	486	int ret;
	487
	488	ret = close(dev->fd);
	489	if (ret != 0) {
	490	tcmu_err("could not close device fd for %s: %d\n", dev->dev_name, errno);
	491	}
	492	ret = munmap(dev->map, dev->map_len);
	493	if (ret != 0) {
	494	tcmu_err("could not unmap device %s: %d\n", dev->dev_name, errno);
	495	}
	496	}
	497
	498	static bool device_open_shm(struct tcmu_device *dev)
	499	{
	500	size_t mmap_size;
	501	char *mmap_name;
	502	off_t mmap_offset;
	503
	504	/* get filename, size and offset */
	505	mmap_name = tcmu_dev_get_memory_info(dev, NULL, &mmap_size, &mmap_offset);
	506	if (!mmap_name)
	507	goto err_fail;
	508
	509	/* cache the map size */
	510	dev->map_len = mmap_size;
	511
	512	/* open the map */
	513	dev->fd = open(mmap_name, O_RDWR \| O_NONBLOCK \| O_CLOEXEC);
	514	if (dev->fd == -1) {
	515	tcmu_err("could not open %s\n", mmap_name);
	516	goto err_mmap_name;
	517	}
	518
	519	/* bring the map into memory */
	520	dev->map = mmap(NULL, dev->map_len, PROT_READ\|PROT_WRITE, MAP_SHARED, dev->fd, mmap_offset);
	521	if (dev->map == MAP_FAILED) {
	522	tcmu_err("could not mmap %s\n", mmap_name);
	523	goto err_fd_close;
	524	}
	525
	526	if (dev->map->version != KERN_IFACE_VER) {
	527	tcmu_err("Kernel interface version mismatch: wanted %d got %d\n",
	528	KERN_IFACE_VER, dev->map->version);
	529	goto err_munmap;
	530	}
	531
	532	free(mmap_name);
	533	return true;
	534
	535	err_munmap:
	536	munmap(dev->map, dev->map_len);
	537	err_fd_close:
	538	close(dev->fd);
	539	err_mmap_name:
	540	free(mmap_name);
	541	err_fail:
	542	return false;
	543	}
	544
	545	static int device_add(struct tcmulib_context ctx, char dev_name,
	546	char *cfgstring, bool reopen)
	547	{
	548	struct tcmu_device *dev;
	549	char *reason = NULL;
	550	int rc;
	551	bool reset_supp = true;
	552
	553	dev = calloc(1, sizeof(*dev));
	554	if (!dev) {
	555	tcmu_err("calloc failed for device_add()\n");
	556	return -ENOMEM;
	557	}
	558
	559	if (!device_parse_cfg(dev, dev_name, cfgstring))
	560	goto err_free;
452	561
453	562	dev->handler = find_handler(ctx, dev->cfgstring);
454	563	if (!dev->handler) {

472	581	* from a fresh slate. We will unblock below when we are
473	582	* completely setup.
474	583	*/
475		ret = tcmu_cfgfs_dev_exec_action(dev, "block_dev", 1);
	584	rc = tcmu_cfgfs_dev_exec_action(dev, "block_dev", 1);
476	585	/*
477	586	* As long as the block_dev file existed, try to reset
478	587	* just in case the kernel was in a invald state.
479	588	*/
480		if (ret == -ENOENT) {
	589	if (rc == -ENOENT) {
481	590	reset_supp = false;
482	591	} else {
483	592	/*
484	593	* Force a retry of the outstanding commands.
485	594	*/
486		ret = tcmu_cfgfs_dev_exec_action(dev, "reset_ring", 1);
487		if (ret)
488		tcmu_dev_err(dev, "Could not reset ring %d.\n", ret);
	595	rc = tcmu_cfgfs_dev_exec_action(dev, "reset_ring", 1);
	596	if (rc)
	597	tcmu_dev_err(dev, "Could not reset ring %d.\n", rc);
489	598	}
490	599	}
491	600
492		snprintf(str_buf, sizeof(str_buf), "/dev/%s", dev_name);
493
494		dev->fd = open(str_buf, O_RDWR \| O_NONBLOCK \| O_CLOEXEC);
495		if (dev->fd == -1) {
496		tcmu_err("could not open %s\n", str_buf);
	601	if (!device_open_shm(dev))
497	602	goto err_unblock;
498		}
499
500		snprintf(str_buf, sizeof(str_buf), "/sys/class/uio/%s/maps/map0/size", dev->dev_name);
501		fd = open(str_buf, O_RDONLY);
502		if (fd == -1) {
503		tcmu_err("could not open %s\n", str_buf);
504		goto err_fd_close;
505		}
506
507		ret = read(fd, str_buf, sizeof(str_buf));
508		close(fd);
509		if (ret <= 0) {
510		tcmu_err("could not read size of map0\n");
511		goto err_fd_close;
512		}
513		str_buf[ret-1] = '\0'; /* null-terminate and chop off the \n */
514
515		dev->map_len = strtoull(str_buf, NULL, 0);
516		if (dev->map_len == ULLONG_MAX) {
517		tcmu_err("could not get map length\n");
518		goto err_fd_close;
519		}
520
521		dev->map = mmap(NULL, dev->map_len, PROT_READ\|PROT_WRITE, MAP_SHARED, dev->fd, 0);
522		if (dev->map == MAP_FAILED) {
523		tcmu_err("could not mmap: %m\n");
524		goto err_fd_close;
525		}
526
527		mb = dev->map;
528		if (mb->version != KERN_IFACE_VER) {
529		tcmu_err("Kernel interface version mismatch: wanted %d got %d\n",
530		KERN_IFACE_VER, mb->version);
531		goto err_munmap;
532		}
533
534		dev->cmd_tail = mb->cmd_tail;
	603
	604	dev->cmd_tail = dev->map->cmd_tail;
535	605	dev->ctx = ctx;
536	606
537		ret = dev->handler->added(dev);
538		if (ret != 0) {
	607	rc = dev->handler->added(dev);
	608	if (rc != 0) {
539	609	tcmu_err("handler open failed for %s\n", dev->dev_name);
540		goto err_munmap;
	610	goto err_closeshm;
541	611	}
542	612
543	613	darray_append(ctx->devices, dev);

547	617
548	618	return 0;
549	619
550		err_munmap:
551		munmap(dev->map, dev->map_len);
552		err_fd_close:
553		close(dev->fd);
	620	err_closeshm:
	621	device_close_shm(dev);
554	622	err_unblock:
555	623	if (reopen && reset_supp)
556	624	tcmu_cfgfs_dev_exec_action(dev, "block_dev", 0);
557	625	err_free:
558	626	free(dev);
559
560	627	return -ENOENT;
561	628	}
562	629

567	634
568	635	darray_foreach_reverse(dev_ptr, ctx->devices) {
569	636	dev = *dev_ptr;
570		remove_device(ctx, dev->dev_name, true);
571		}
572		}
573
574		static void remove_device(struct tcmulib_context ctx, char dev_name,
	637	device_remove(ctx, dev->dev_name, true);
	638	}
	639	}
	640
	641	static void device_remove(struct tcmulib_context ctx, char dev_name,
575	642	bool should_block)
576	643	{
577	644	struct tcmu_device *dev;
578		int i, ret;
	645	int i;
579	646
580	647	dev = lookup_dev_by_name(ctx, dev_name, &i);
581	648	if (!dev) {

597	664
598	665	dev->handler->removed(dev);
599	666
600		ret = close(dev->fd);
601		if (ret != 0) {
602		tcmu_err("could not close device fd %s: %d\n", dev_name, errno);
603		}
604		ret = munmap(dev->map, dev->map_len);
605		if (ret != 0) {
606		tcmu_err("could not unmap device %s: %d\n", dev_name, errno);
607		}
	667	device_close_shm(dev);
608	668
609	669	if (should_block)
610	670	tcmu_cfgfs_dev_exec_action(dev, "block_dev", 0);

688	748	if (read_uio_name(dirent_list[i]->d_name, &dev_name))
689	749	continue;
690	750
691		if (add_device(ctx, dirent_list[i]->d_name, dev_name, true) < 0) {
	751	if (device_add(ctx, dirent_list[i]->d_name, dev_name, true) < 0) {
692	752	free (dev_name);
693	753	continue;
694	754	}

772	832	void tcmu_dev_set_private(struct tcmu_device dev, void private)
773	833	{
774	834	dev->hm_private = private;
	835	}
	836
	837	const char tcmu_dev_get_uio_name(struct tcmu_device dev)
	838	{
	839	return dev->dev_name;
	840	}
	841
	842	void tcmu_set_thread_name(const char prefix, struct tcmu_device dev)
	843	{
	844	const char *uio = dev ? tcmu_dev_get_uio_name(dev) : NULL;
	845	char cname[TCMU_THREAD_NAME_LEN];
	846	char *pname;
	847
	848	if (pthread_getname_np(pthread_self(), cname, TCMU_THREAD_NAME_LEN))
	849	return;
	850
	851	/*
	852	* If we are trying to set the pthread name in the
	853	* event work thread, we must ignore it.
	854	*/
	855	if (!strcmp(cname, "ework-thread")) {
	856	tcmu_dev_warn(dev, "Do not set name for event work thread in the callback fn\n");
	857	return;
	858	}
	859
	860	if (!prefix) {
	861	tcmu_dev_err(dev, "Failed to set name for thread %lu\n",
	862	pthread_self());
	863	return;
	864	}
	865
	866	if (asprintf(&pname, "%s%s%s", prefix, uio ? "-" : "", uio ? uio : "") == -1) {
	867	tcmu_dev_err(dev, "Could not allocate thread name.\n");
	868	return;
	869	}
	870
	871	if (strlen(pname) >= TCMU_THREAD_NAME_LEN) {
	872	tcmu_dev_warn(dev, "Cannot set thread name to %s. Name must be less than %d chars. ",
	873	pname, TCMU_THREAD_NAME_LEN);
	874	pname[TCMU_THREAD_NAME_LEN - 1] = '\0';
	875	tcmu_dev_warn(dev, "Truncating to %s.\n", pname);
	876	}
	877
	878	if (pthread_setname_np(pthread_self(), pname))
	879	tcmu_dev_err(dev, "Could not set thread name to %s\n", pname);
	880	free(pname);
775	881	}
776	882
777	883	void tcmu_dev_set_num_lbas(struct tcmu_device *dev, uint64_t num_lbas)

921	1027	int tcmu_dev_get_fd(struct tcmu_device *dev)
922	1028	{
923	1029	return dev->fd;
	1030	}
	1031
	1032	/**
	1033	* tcmu_dev_get_memory_info - retrieve information about tcmu's shared
	1034	* memory block. If the memory is mapped to another address (e.g., in
	1035	* another process) then the information in struct iovec can be converted
	1036	* using these values. Returns the filename of the mmap() that fd uses, or
	1037	* NULL if information is not available. The caller must free() the
	1038	* returned filename.
	1039	* @dev: tcmu device
	1040	* @base: receives the base address of the shared memory region
	1041	* @len: receives the size of the shared memory region
	1042	* @offset: receives the offset within fd for mmap() (conventionally 0).
	1043	*/
	1044	char *
	1045	tcmu_dev_get_memory_info(struct tcmu_device dev, void *base,
	1046	size_t len, off_t offset)
	1047	{
	1048	char *mmap_name;
	1049	const char *namefmt = "/dev/%s";
	1050	const char *sizefmt = "/sys/class/uio/%s/maps/map0/size";
	1051
	1052	if (asprintf(&mmap_name, namefmt, dev->dev_name) == -1) {
	1053	tcmu_err("cannot construct device map filename\n");
	1054	goto err_fail;
	1055	}
	1056	if (base)
	1057	*base = dev->map;
	1058	if (len) {
	1059	if (dev->map_len != 0) {
	1060	/* cached */
	1061	*len = dev->map_len;
	1062	} else {
	1063	/* get length of map from file */
	1064	ssize_t size;
	1065	char *size_name;
	1066
	1067	if (asprintf(&size_name, sizefmt, dev->dev_name) == -1) {
	1068	tcmu_err("cannot construct device map size filename\n");
	1069	goto err_free;
	1070	}
	1071	size = read_size(size_name);
	1072	free(size_name);
	1073	if (size == -1) {
	1074	tcmu_err("unable to read device map0 size\n");
	1075	goto err_free;
	1076	}
	1077	*len = size;
	1078	}
	1079	}
	1080	if (offset)
	1081	*offset = 0;
	1082	return mmap_name;
	1083
	1084	err_free:
	1085	free(mmap_name);
	1086	err_fail:
	1087	return NULL;
924	1088	}
925	1089
926	1090	char tcmu_dev_get_cfgstring(struct tcmu_device dev)

-2

libtcmu_common.h less more

61	61	TCMU_STS_TOO_MANY_TGT_DESC,
62	62	};
63	63
	64	#define TCMU_THREAD_NAME_LEN 16
	65
64	66	#define SENSE_BUFFERSIZE 96
65	67
66	68	#define CFGFS_ROOT "/sys/kernel/config/target"

105	107	/* Set/Get methods for the opaque tcmu_device */
106	108	void tcmu_dev_get_private(struct tcmu_device dev);
107	109	void tcmu_dev_set_private(struct tcmu_device dev, void priv);
108		void tcmu_get_daemon_dev_private(struct tcmu_device dev);
109		void tcmu_set_daemon_dev_private(struct tcmu_device dev, void priv);
	110	const char tcmu_dev_get_uio_name(struct tcmu_device dev);
	111	void tcmu_set_thread_name(const char prefix, struct tcmu_device dev);
110	112	int tcmu_dev_get_fd(struct tcmu_device *dev);
	113	char tcmu_dev_get_memory_info(struct tcmu_device dev, void **base,
	114	size_t len, off_t offset);
111	115	char tcmu_dev_get_cfgstring(struct tcmu_device dev);
112	116	void tcmu_dev_set_num_lbas(struct tcmu_device *dev, uint64_t num_lbas);
113	117	uint64_t tcmu_dev_get_num_lbas(struct tcmu_device *dev);

-0

libtcmu_config.c less more

400	400	char buf[BUF_LEN];
401	401	char *p;
402	402
	403	tcmu_set_thread_name("dyn-config", NULL);
	404
403	405	monitor = inotify_init();
404	406	if (monitor == -1) {
405	407	tcmu_err("Failed to init inotify %m\n");

+13

-3

libtcmu_log.c less more

182	182	{
183	183	char timestamp[TCMU_TIME_STRING_BUFLEN] = {0, };
184	184
	185	if (!output)
	186	return;
	187
185	188	if (time_string_now(timestamp) < 0)
186	189	return;
187	190

399	402	int fd = (intptr_t) data;
400	403	char buf, msg;
401	404	int count, ret, written = 0, r, pid = 0;
	405	char pname[TCMU_THREAD_NAME_LEN];
402	406
403	407	if (fd == -1)
404	408	return -1;

407	411	if (pid <= 0)
408	412	return -1;
409	413
	414	if (pthread_getname_np(pthread_self(), pname, TCMU_THREAD_NAME_LEN))
	415	return -1;
	416
410	417	/*
411	418	* format: timestamp pid [loglevel] msg
412	419	*/
413		ret = asprintf(&msg, "%s %d [%s] %s", timestamp, pid, loglevel_string(pri), str);
	420	ret = asprintf(&msg, "%s %d:%s [%s] %s", timestamp, pid, pname,
	421	loglevel_string(pri), str);
414	422	if (ret < 0)
415	423	return -1;
416	424

515	523
516	524	static void log_thread_start(void arg)
517	525	{
518		tcmu_logbuf = arg;
	526	tcmu_set_thread_name("logger", NULL);
519	527
520	528	pthread_cleanup_push(log_cleanup, arg);
521	529

538	546	return false;
539	547
540	548	if (strlen(path) >= PATH_MAX - TCMU_LOG_FILENAME_MAX) {
541		tcmu_err("--tcmu-log-dir='%s' cannot exceed %d characters\n",
	549	tcmu_err("The length of log dir path '%s' exceeds %d characters\n",
542	550	path, PATH_MAX - TCMU_LOG_FILENAME_MAX - 1);
543	551	return false;
544	552	}

686	694	if (ret < 0)
687	695	tcmu_err("create file output error \n");
688	696
	697	tcmu_logbuf = logbuf;
689	698	ret = pthread_create(&logbuf->thread_id, NULL, log_thread_start,
690	699	logbuf);
691	700	if (ret) {
	701	tcmu_logbuf = NULL;
692	702	log_cleanup(logbuf);
693	703	return ret;
694	704	}

-0

main-syms.txt less more

3	3	tcmur_handle_writesame;
4	4	tcmu_notify_lock_lost;
5	5	tcmu_notify_conn_lost;
	6	tcmu_notify_cmd_timed_out;
	7	tcmu_event_name;
6	8	tcmur_dev_update_size;
7	9	tcmur_dev_set_private;
8	10	tcmur_dev_get_private;

+53

-11

main.c less more

47	47	#include "version.h"
48	48	#include "libtcmu_config.h"
49	49	#include "libtcmu_log.h"
	50	#include "tcmur_work.h"
50	51
51	52	#define TCMU_LOCK_FILE "/run/tcmu.lock"
52	53

66	67	int tcmur_register_handler(struct tcmur_handler *handler)
67	68	{
68	69	struct tcmur_handler *h;
	70	int ret;
69	71	int i;
70	72
71	73	for (i = 0; i < darray_size(g_runner_handlers); i++) {

77	79	}
78	80	}
79	81
	82	if (handler->init) {
	83	ret = handler->init();
	84	if (ret) {
	85	tcmu_err("Failed to init handler %s, ret = %d\n",
	86	handler->subtype, ret);
	87	return ret;
	88	}
	89	}
	90
	91	tcmu_info("Handler %s is registered\n", handler->subtype);
80	92	darray_append(g_runner_handlers, handler);
81	93	return 0;
82	94	}

104	116	g_free((char*)handler->opaque);
105	117	g_free((char*)handler->subtype);
106	118	g_free((char*)handler->cfg_desc);
	119	if (handler->destroy)
	120	handler->destroy();
107	121	g_free(handler);
108	122	}
109	123

119	133	}
120	134
121	135	return ret;
	136	}
	137
	138	static void tcmur_unregister_all_dbus_handlers(void)
	139	{
	140	struct tcmur_handler *handler;
	141	int i;
	142	for (i = 0; i < darray_size(g_runner_handlers); i++) {
	143	handler = darray_item(g_runner_handlers, i);
	144	if (handler->_is_dbus_handler == true) {
	145	if (tcmur_unregister_handler(handler))
	146	free_dbus_handler(handler);
	147	}
	148	}
	149	darray_free(g_runner_handlers);
122	150	}
123	151
124	152	static int is_handler(const struct dirent *dirent)

599	627	* The lock thread can fire off the recovery thread, so make sure
600	628	* it is done first.
601	629	*/
602		tcmu_cancel_lock_thread(dev);
	630	tcmur_flush_work(rdev->event_work);
603	631	tcmu_cancel_recovery(dev);
604	632
605	633	tcmu_release_dev_lock(dev);

722	750	}
723	751
724	752	tcmur_cmd->timed_out = true;
	753	/*
	754	* These time outs are only currently used for diagnostic
	755	* purposes right now, so we do not want to escalate the
	756	* error handler and just return true here.
	757	*/
	758	tcmu_notify_cmd_timed_out(dev);
725	759	}
726	760	pthread_spin_unlock(&rdev->lock);
727	761	}

754	788	struct pollfd pfd;
755	789	int ret;
756	790	bool dev_stopping = false;
	791
	792	tcmu_set_thread_name("cmdproc", dev);
757	793
758	794	pthread_cleanup_push(tcmur_stop_device, dev);
759	795

1028	1064
1029	1065	rdev->flags \|= TCMUR_DEV_FLAG_IS_OPEN;
1030	1066
1031		ret = pthread_cond_init(&rdev->lock_cond, NULL);
1032		if (ret) {
1033		ret = -ret;
	1067	rdev->event_work = tcmur_create_work();
	1068	if (!rdev->event_work) {
	1069	ret = -ENOMEM;
1034	1070	goto close_dev;
1035	1071	}
1036	1072

1038	1074	dev);
1039	1075	if (ret) {
1040	1076	ret = -ret;
1041		goto cleanup_lock_cond;
	1077	goto cleanup_event_work;
1042	1078	}
1043	1079
1044	1080	return 0;
1045	1081
1046		cleanup_lock_cond:
1047		pthread_cond_destroy(&rdev->lock_cond);
	1082	cleanup_event_work:
	1083	tcmur_destroy_work(rdev->event_work);
1048	1084	close_dev:
1049	1085	rhandler->close(dev);
1050	1086	cleanup_aio_tracking:

1091	1127	cleanup_io_work_queue(dev, false);
1092	1128	cleanup_aio_tracking(rdev);
1093	1129
1094		ret = pthread_cond_destroy(&rdev->lock_cond);
1095		if (ret != 0)
1096		tcmu_err("could not cleanup lock cond %d\n", ret);
	1130	tcmur_destroy_work(rdev->event_work);
1097	1131
1098	1132	ret = pthread_mutex_destroy(&rdev->state_lock);
1099	1133	if (ret != 0)

1189	1223	{
1190	1224	darray(struct tcmulib_handler) handlers = darray_new();
1191	1225	struct tcmulib_context *tcmulib_context;
1192		struct tcmur_handler **tmp_r_handler;
	1226	struct tcmur_handler *tmp_r_handler, r_handler;
	1227	struct tcmulib_handler *handler;
1193	1228	GMainLoop *loop;
1194	1229	GIOChannel *libtcmu_gio;
1195	1230	guint reg_id;

1401	1436	tcmu_unwatch_config(tcmu_cfg);
1402	1437	tcmulib_close(tcmulib_context);
1403	1438	err_free_handlers:
	1439	tcmur_unregister_all_dbus_handlers();
	1440
	1441	darray_foreach(handler, handlers) {
	1442	r_handler = handler->hm_private;
	1443	if (r_handler && r_handler->destroy)
	1444	r_handler->destroy();
	1445	}
1404	1446	darray_free(handlers);
1405	1447	close_fd:
1406	1448	if (reset_nl_supp)

+283

-52

rbd.c less more

24	24
25	25	#include <scsi/scsi.h>
26	26
	27	#include "darray.h"
27	28	#include "tcmu-runner.h"
28	29	#include "tcmur_cmd_handler.h"
29	30	#include "libtcmu.h"

79	80	char *osd_op_timeout;
80	81	char *conf_path;
81	82	char *id;
	83	char *addrs;
82	84	};
83	85
84	86	enum rbd_aio_type {

106	108	size_t iov_cnt;
107	109	};
108	110
	111	static pthread_mutex_t blacklist_caches_lock = PTHREAD_MUTEX_INITIALIZER;
	112	static darray(char *) blacklist_caches;
	113
109	114	#ifdef LIBRADOS_SUPPORTS_SERVICES
110	115
111	116	#ifdef RBD_LOCK_ACQUIRE_SUPPORT
112		static void tcmu_rbd_service_status_update(struct tcmu_device *dev,
113		bool has_lock)
114		{
115		struct tcmu_rbd_state *state = tcmur_dev_get_private(dev);
	117	static int tcmu_rbd_service_status_update(struct tcmu_device *dev,
	118	bool has_lock)
	119	{
	120	struct tcmu_rbd_state *state = tcmur_dev_get_private(dev);
	121	struct tcmur_device *rdev = tcmu_dev_get_private(dev);
116	122	char *status_buf = NULL;
117	123	int ret;
118	124
119		ret = asprintf(&status_buf, "%s%c%s%c", "lock_owner", '\0',
120		has_lock ? "true" : "false", '\0');
	125	ret = asprintf(&status_buf,
	126	"%s%c%s%c%s%c%"PRIu64"%c%s%c%"PRIu64"%c%s%c%"PRIu64"%c",
	127	"lock_owner", '\0', has_lock ? "true" : "false", '\0',
	128	"lock_lost_cnt", '\0', rdev->lock_lost_cnt, '\0',
	129	"conn_lost_cnt", '\0', rdev->conn_lost_cnt, '\0',
	130	"cmd_timed_out_cnt", '\0', rdev->cmd_timed_out_cnt, '\0');
121	131	if (ret < 0) {
122	132	tcmu_dev_err(dev, "Could not allocate status buf. Service will not be updated.\n");
123		return;
	133	return ret;
124	134	}
125	135
126	136	ret = rados_service_update_status(state->cluster, status_buf);

130	140	}
131	141
132	142	free(status_buf);
133		}
	143	return ret;
	144	}
	145
134	146	#endif /* RBD_LOCK_ACQUIRE_SUPPORT */
	147
	148	static int tcmu_rbd_report_event(struct tcmu_device *dev)
	149	{
	150	struct tcmur_device *rdev = tcmu_dev_get_private(dev);
	151
	152	/*
	153	* We ignore the specific event and report all the current counter
	154	* values, because tools like gwcli/dashboard may not see every
	155	* update, and we do not want one event to overwrite the info.
	156	*/
	157	return tcmu_rbd_service_status_update(dev,
	158	rdev->lock_state == TCMUR_DEV_LOCK_WRITE_LOCKED ? true : false);
	159	}
135	160
136	161	static int tcmu_rbd_service_register(struct tcmu_device *dev)
137	162	{

169	194	goto free_image_id_buf;
170	195	}
171	196
172		ret = asprintf(&metadata_buf, "%s%c%s%c%s%c%s%c%s%c%s%c",
	197	ret = asprintf(&metadata_buf, "%s%c%s%c%s%c%s%c%s%c%s%c%s%c%s%c%s%c%s%c",
173	198	"pool_name", '\0', state->pool_name, '\0',
174	199	"image_name", '\0', state->image_name, '\0',
175		"image_id", '\0', image_id_buf, '\0');
	200	"image_id", '\0', image_id_buf, '\0',
	201	"daemon_type", '\0', "portal", '\0',
	202	"daemon_prefix", '\0', u.nodename, '\0');
176	203	if (ret < 0) {
177	204	tcmu_dev_err(dev, "Could not allocate metadata buf.\n");
178	205	ret = -ENOMEM;

184	211	if (ret < 0) {
185	212	tcmu_dev_err(dev, "Could not register service to cluster. (Err %d)\n",
186	213	ret);
187		}
188
	214	goto free_meta_buf;
	215	}
	216
	217	ret = tcmu_rbd_report_event(dev);
	218	if (ret < 0)
	219	tcmu_dev_err(dev, "Could not update status. (Err %d)\n", ret);
	220
	221	free_meta_buf:
189	222	free(metadata_buf);
190	223	free_daemon_buf:
191	224	free(daemon_buf);

211	244	#endif /* RBD_LOCK_ACQUIRE_SUPPORT */
212	245
213	246	#endif /* LIBRADOS_SUPPORTS_SERVICES */
	247
	248	#if defined LIBRADOS_SUPPORTS_GETADDRS \|\| defined RBD_LOCK_ACQUIRE_SUPPORT
	249	static void tcmu_rbd_rm_stale_entry_from_blacklist(struct tcmu_device dev, char addrs)
	250	{
	251	struct tcmu_rbd_state *state = tcmur_dev_get_private(dev);
	252	const char p, q, *end;
	253	char cmd, addr;
	254	int ret;
	255
	256	/*
	257	* Just skip extra chars before '[' if there has
	258	*/
	259	p = strchr(addrs, '[');
	260	if (!p)
	261	p = addrs;
	262
	263	/*
	264	* The addrs will a string like:
	265	* "[192.168.195.172:0/2203456141,192.168.195.172:0/4908756432]"
	266	* Or
	267	* "192.168.195.172:0/2203456141"
	268	*/
	269	while (1) {
	270	if (p == NULL \|\| *p == ']') {
	271	return; /* we are done here */
	272	} else if (p == '[' \|\| p == ',') {
	273	/* Skip "[" and white spaces */
	274	while (p != '\0' && !isalnum(p)) p++;
	275	if (*p == '\0') {
	276	tcmu_dev_warn(dev, "Get an invalid address '%s'!\n", addrs);
	277	return;
	278	}
	279
	280	end = strchr(p, ',');
	281	if (!end)
	282	end = strchr(p, ']');
	283
	284	if (!end) {
	285	tcmu_dev_warn(dev, "Get an invalid address '%s'!\n", addrs);
	286	return;
	287	}
	288
	289	q = end; /* The end should be ',' or ']' /
	290
	291	while (q != '\0' && !isalnum(q)) q--;
	292	if (*q == '\0') {
	293	tcmu_dev_warn(dev, "Get an invalid address '%s'!\n", addrs);
	294	return;
	295	}
	296
	297	addr = strndup(p, q - p + 1);
	298	p = end;
	299	} else {
	300	/* In case of "192.168.195.172:0/2203456141" */
	301	addr = strdup(p);
	302	p = NULL;
	303	}
	304
	305	ret = asprintf(&cmd,
	306	"{\"prefix\": \"osd blacklist\","
	307	"\"blacklistop\": \"rm\","
	308	"\"addr\": \"%s\"}",
	309	addr);
	310	free(addr);
	311	if (ret < 0) {
	312	tcmu_dev_warn(dev, "Could not allocate command. (Err %d)\n",
	313	ret);
	314	return;
	315	}
	316	ret = rados_mon_command(state->cluster, (const char**)&cmd, 1, NULL, 0,
	317	NULL, NULL, NULL, NULL);
	318	free(cmd);
	319	if (ret < 0) {
	320	tcmu_dev_err(dev, "Could not rm blacklist entry '%s'. (Err %d)\n",
	321	addr, ret);
	322	return;
	323	}
	324	}
	325	}
	326
	327	static int tcmu_rbd_rm_stale_entries_from_blacklist(struct tcmu_device *dev)
	328	{
	329	char *entry, tmp_entry;
	330	int ret = 0;
	331	int i;
	332
	333	pthread_mutex_lock(&blacklist_caches_lock);
	334	if (darray_empty(blacklist_caches))
	335	goto unlock;
	336
	337	/* Try to remove all the stale blacklist entities */
	338	darray_foreach(entry, blacklist_caches) {
	339	tcmu_dev_info(dev, "removing addrs: {%s}\n", *entry);
	340	tcmu_rbd_rm_stale_entry_from_blacklist(dev, *entry);
	341	}
	342
	343	unlock:
	344	for (i = darray_size(blacklist_caches) - 1; i >= 0; i--) {
	345	tmp_entry = darray_item(blacklist_caches, i);
	346	darray_remove(blacklist_caches, i);
	347	free(tmp_entry);
	348	}
	349
	350	pthread_mutex_unlock(&blacklist_caches_lock);
	351	return ret;
	352	}
	353	#endif // LIBRADOS_SUPPORTS_GETADDRS \|\| RBD_LOCK_ACQUIRE_SUPPORT
214	354
215	355	static char tcmu_rbd_find_quote(char string)
216	356	{

511	651
512	652	ret = rbd_is_exclusive_lock_owner(state->image, &is_owner);
513	653	if (ret < 0) {
514		tcmu_dev_err(dev, "Could not check lock ownership. Error: %s.\n",
515		strerror(-ret));
	654	if (ret == -ESHUTDOWN) {
	655	tcmu_dev_dbg(dev, "Client is blacklisted. Could not check lock ownership.\n");
	656	} else {
	657	tcmu_dev_err(dev, "Could not check lock ownership. Error: %s.\n",
	658	strerror(-ret));
	659	}
	660
516	661	if (ret == -ESHUTDOWN \|\| ret == -ETIMEDOUT)
517	662	return ret;
518	663

533	678
534	679	ret = tcmu_rbd_has_lock(dev);
535	680	if (ret == 1)
536		return TCMUR_DEV_LOCK_LOCKED;
	681	return TCMUR_DEV_LOCK_WRITE_LOCKED;
537	682	else if (ret == 0 \|\| ret == -ESHUTDOWN)
538	683	return TCMUR_DEV_LOCK_UNLOCKED;
539	684	else

736	881	static int tcmu_rbd_lock(struct tcmu_device *dev, uint16_t tag)
737	882	{
738	883	struct tcmu_rbd_state *state = tcmur_dev_get_private(dev);
	884	#if !defined LIBRADOS_SUPPORTS_GETADDRS && defined RBD_LOCK_ACQUIRE_SUPPORT
	885	rbd_lock_mode_t lock_mode;
	886	char owners1[1], owners2[1];
	887	size_t num_owners1 = 1, num_owners2 = 1;
	888	#endif
739	889	int ret;
740	890
741	891	ret = tcmu_rbd_has_lock(dev);

757	907	if (ret)
758	908	goto done;
759	909
	910	#if !defined LIBRADOS_SUPPORTS_GETADDRS && defined RBD_LOCK_ACQUIRE_SUPPORT
	911	ret = rbd_lock_get_owners(state->image, &lock_mode, owners1,
	912	&num_owners1);
	913	if ((!ret && !num_owners1) \|\| ret < 0) {
	914	tcmu_dev_warn(dev, "Could not get lock owners to store blacklist entry %d!\n",
	915	ret);
	916	} else {
	917	int is_owner;
	918
	919	/* To check whether we are still the lock owner */
	920	ret = rbd_is_exclusive_lock_owner(state->image, &is_owner);
	921	if (ret) {
	922	rbd_lock_get_owners_cleanup(owners1, num_owners1);
	923	tcmu_dev_warn(dev, "Could not check lock owners to store blacklist entry %d!\n",
	924	ret);
	925	goto no_owner;
	926	}
	927
	928	/* To get the lock owner again */
	929	ret = rbd_lock_get_owners(state->image, &lock_mode, owners2,
	930	&num_owners2);
	931	if ((!ret && !num_owners2) \|\| ret < 0) {
	932	tcmu_dev_warn(dev, "Could not get lock owners to store blacklist entry %d!\n",
	933	ret);
	934	/* Only we didn't lose the lock during the above check will we store the blacklist list */
	935	} else if (!strcmp(owners1[0], owners2[0]) && is_owner) {
	936	state->addrs = strdup(owners1[0]); // ignore the errors
	937	}
	938
	939	rbd_lock_get_owners_cleanup(owners1, num_owners1);
	940	rbd_lock_get_owners_cleanup(owners2, num_owners2);
	941	}
	942	no_owner:
	943	#endif
	944
760	945	set_lock_tag:
761	946	tcmu_dev_warn(dev, "Acquired exclusive lock.\n");
762	947	if (tag != TCMU_INVALID_LOCK_TAG)

805	990	free(state->pool_name);
806	991	if (state->id)
807	992	free(state->id);
	993	if (state->addrs)
	994	free(state->addrs);
808	995	free(state);
809	996	}
810	997

836	1023	char pool, name, *next_opt;
837	1024	char config, dev_cfg_dup;
838	1025	struct tcmu_rbd_state *state;
839		uint32_t max_blocks;
	1026	uint32_t max_blocks, unmap_gran;
840	1027	int ret;
	1028	char buf[128];
841	1029
842	1030	state = calloc(1, sizeof(*state));
843	1031	if (!state)

947	1135	max_blocks = (image_info.obj_size * 4) / tcmu_dev_get_block_size(dev);
948	1136	tcmu_dev_set_opt_xcopy_rw_len(dev, max_blocks);
949	1137	tcmu_dev_set_max_unmap_len(dev, max_blocks);
950		tcmu_dev_set_opt_unmap_gran(dev, image_info.obj_size /
951		tcmu_dev_get_block_size(dev), false);
	1138	ret = rados_conf_get(state->cluster, "rbd_discard_granularity_bytes", buf,
	1139	sizeof(buf));
	1140	if (!ret) {
	1141	tcmu_dev_dbg(dev, "rbd_discard_granularity_bytes: %s\n", buf);
	1142	unmap_gran = atoi(buf) / tcmu_dev_get_block_size(dev);
	1143	} else {
	1144	tcmu_dev_warn(dev,
	1145	"Failed to get 'rbd_discard_granularity_bytes', %d\n",
	1146	ret);
	1147	unmap_gran = image_info.obj_size / tcmu_dev_get_block_size(dev);
	1148	}
	1149	tcmu_dev_dbg(dev, "unmap_gran: %d\n", unmap_gran);
	1150	tcmu_dev_set_opt_unmap_gran(dev, unmap_gran, false);
	1151	tcmu_dev_set_unmap_gran_align(dev, unmap_gran);
952	1152	tcmu_dev_set_write_cache_enabled(dev, 0);
	1153
	1154	#if defined LIBRADOS_SUPPORTS_GETADDRS \|\| defined RBD_LOCK_ACQUIRE_SUPPORT
	1155	tcmu_rbd_rm_stale_entries_from_blacklist(dev);
	1156	#endif
	1157
	1158	#ifdef LIBRADOS_SUPPORTS_GETADDRS
	1159	/* Get current entry address for the image */
	1160	ret = rados_getaddrs(state->cluster, &state->addrs);
	1161	tcmu_dev_info(dev, "address: {%s}\n", state->addrs);
	1162	if (ret < 0)
	1163	return ret;
	1164	#endif
953	1165
954	1166	free(dev_cfg_dup);
955	1167	return 0;

968	1180	struct tcmu_rbd_state *state = tcmur_dev_get_private(dev);
969	1181
970	1182	tcmu_rbd_image_close(dev);
	1183
	1184	/*
	1185	* Since we are closing the device, but current device maybe
	1186	* already blacklisted by other tcmu nodes. Let's just save
	1187	* the entity addrs into the blacklist_caches, and let any
	1188	* other new device help remove it.
	1189	*/
	1190	if (state->addrs) {
	1191	pthread_mutex_lock(&blacklist_caches_lock);
	1192	darray_append(blacklist_caches, state->addrs);
	1193	pthread_mutex_unlock(&blacklist_caches_lock);
	1194	state->addrs = NULL;
	1195	}
	1196
971	1197	tcmu_rbd_state_free(state);
972	1198	}
973	1199
974	1200	static int tcmu_rbd_handle_blacklisted_cmd(struct tcmu_device *dev)
975		{
	1201	{
976	1202	tcmu_notify_lock_lost(dev);
977	1203	/*
978	1204	* This will happen during failback normally, because

990	1216	* to try a different OSD.
991	1217	*/
992	1218	static int tcmu_rbd_handle_timedout_cmd(struct tcmu_device *dev)
993		{
	1219	{
994	1220	tcmu_dev_err(dev, "Timing out cmd.\n");
995		tcmu_notify_conn_lost(dev);
	1221	tcmu_notify_cmd_timed_out(dev);
996	1222
997	1223	/*
998	1224	* TODO: For AA, we will want to kill the ceph tcp connections

1404	1630	}
1405	1631	#endif /* RBD_COMPARE_AND_WRITE_SUPPORT */
1406	1632
1407		/*
1408		* Return scsi status or TCMU_STS_NOT_HANDLED
1409		*/
1410		static int tcmu_rbd_handle_cmd(struct tcmu_device *dev,
1411		struct tcmur_cmd *tcmur_cmd)
1412		{
1413		struct tcmulib_cmd *cmd = tcmur_cmd->lib_cmd;
1414		uint8_t *cdb = cmd->cdb;
1415		int ret;
1416
1417		switch(cdb[0]) {
1418		#ifdef RBD_WRITE_SAME_SUPPORT
1419		case WRITE_SAME:
1420		case WRITE_SAME_16:
1421		ret = tcmur_handle_writesame(dev, tcmur_cmd,
1422		tcmu_rbd_aio_writesame);
1423		break;
1424		#endif
1425		#ifdef RBD_COMPARE_AND_WRITE_SUPPORT
1426		case COMPARE_AND_WRITE:
1427		ret = tcmur_handle_caw(dev, tcmur_cmd, tcmu_rbd_aio_caw);
1428		break;
1429		#endif
1430		default:
1431		ret = TCMU_STS_NOT_HANDLED;
1432		}
1433
1434		return ret;
1435		}
1436
1437	1633	static int tcmu_rbd_reconfig(struct tcmu_device *dev,
1438	1634	struct tcmulib_cfg_info *cfg)
1439	1635	{

1450	1646	default:
1451	1647	return -EOPNOTSUPP;
1452	1648	}
	1649	}
	1650
	1651	static int tcmu_rbd_init(void)
	1652	{
	1653	darray_init(blacklist_caches);
	1654	return 0;
	1655	}
	1656
	1657	static void tcmu_rbd_destroy(void)
	1658	{
	1659	char **entry;
	1660
	1661	tcmu_info("destroying the rbd handler\n");
	1662	pthread_mutex_lock(&blacklist_caches_lock);
	1663	if (darray_empty(blacklist_caches))
	1664	goto unlock;
	1665
	1666	/* Try to remove all the stale blacklist entities */
	1667	darray_foreach(entry, blacklist_caches)
	1668	free(*entry);
	1669
	1670	darray_free(blacklist_caches);
	1671
	1672	unlock:
	1673	pthread_mutex_unlock(&blacklist_caches_lock);
1453	1674	}
1454	1675
1455	1676	/*

1482	1703	.read = tcmu_rbd_read,
1483	1704	.write = tcmu_rbd_write,
1484	1705	.reconfig = tcmu_rbd_reconfig,
	1706	#ifdef LIBRADOS_SUPPORTS_SERVICES
	1707	.report_event = tcmu_rbd_report_event,
	1708	#endif
1485	1709	#ifdef LIBRBD_SUPPORTS_AIO_FLUSH
1486	1710	.flush = tcmu_rbd_flush,
1487	1711	#endif
1488	1712	#ifdef RBD_DISCARD_SUPPORT
1489	1713	.unmap = tcmu_rbd_unmap,
1490	1714	#endif
1491		.handle_cmd = tcmu_rbd_handle_cmd,
	1715	#ifdef RBD_WRITE_SAME_SUPPORT
	1716	.writesame = tcmu_rbd_aio_writesame,
	1717	#endif
	1718	#ifdef RBD_COMPARE_AND_WRITE_SUPPORT
	1719	.caw = tcmu_rbd_aio_caw,
	1720	#endif
1492	1721	#ifdef RBD_LOCK_ACQUIRE_SUPPORT
1493	1722	.lock = tcmu_rbd_lock,
1494	1723	.unlock = tcmu_rbd_unlock,
1495	1724	.get_lock_tag = tcmu_rbd_get_lock_tag,
1496	1725	.get_lock_state = tcmu_rbd_get_lock_state,
1497	1726	#endif
	1727	.init = tcmu_rbd_init,
	1728	.destroy = tcmu_rbd_destroy,
1498	1729	};
1499	1730
1500	1731	int handler_init(void)

-7

scsi.c less more

480	480	size_t iov_cnt)
481	481	{
482	482	if (!(cdb[1] & 0x01)) {
483		if (!cdb[2])
484		return tcmu_emulate_std_inquiry(port, cdb, iovec,
485		iov_cnt);
486		else
	483	if (cdb[2])
487	484	return TCMU_STS_INVALID_CDB;
488		} else {
489		return tcmu_emulate_evpd_inquiry(dev, port, cdb, iovec, iov_cnt);
490		}
	485	return tcmu_emulate_std_inquiry(port, cdb, iovec, iov_cnt);
	486	}
	487	return tcmu_emulate_evpd_inquiry(dev, port, cdb, iovec, iov_cnt);
491	488	}
492	489
493	490	int tcmu_emulate_test_unit_ready(

-4

target.c less more

21	21	#include "libtcmu_log.h"
22	22	#include "libtcmu_common.h"
23	23	#include "tcmur_device.h"
	24	#include "tcmur_work.h"
24	25	#include "target.h"
25	26	#include "alua.h"
26	27

211	212	* then sends IO only for it to fail due to the handler not
212	213	* being able to reach its backend).
213	214	*/
214		static void tgt_port_grp_recovery_thread_fn(void arg)
	215	static void tgt_port_grp_recovery_work_fn(void *arg)
215	216	{
216	217	struct tgt_port_grp *tpg = arg;
217	218	struct tcmur_device rdev, tmp_rdev;

272	273	}
273	274
274	275	free_tgt_port_grp(tpg);
275		return NULL;
276	276	}
277	277
278	278	int tcmu_add_dev_to_recovery_list(struct tcmu_device *dev)

326	326	ret = -ENOMEM;
327	327	goto done;
328	328	}
329		ret = pthread_create(&tpg->recovery_thread, NULL,
330		tgt_port_grp_recovery_thread_fn, tpg);
	329
	330	ret = tcmur_run_work(rdev->event_work, tpg, tgt_port_grp_recovery_work_fn);
331	331	if (ret) {
332	332	tcmu_dev_err(dev, "Could not start recovery thread. Err %d\n",
333	333	ret);

+21

-0

tcmu-runner.h less more

55	55
56	56	/* callback to finish/continue command processing */
57	57	void (done)(struct tcmu_device dev, struct tcmur_cmd *cmd, int ret);
	58	};
	59
	60	enum tcmur_event {
	61	TCMUR_EVT_LOCK_LOST,
	62	TCMUR_EVT_CONN_LOST,
	63	TCMUR_EVT_CMD_TIMED_OUT,
58	64	};
59	65
60	66	struct tcmulib_cfg_info;

144	150	int (flush)(struct tcmu_device dev, struct tcmur_cmd *cmd);
145	151	int (unmap)(struct tcmu_device dev, struct tcmur_cmd *cmd,
146	152	uint64_t off, uint64_t len);
	153	int (writesame)(struct tcmu_device dev, struct tcmur_cmd *cmd, uint64_t off,
	154	uint64_t len, struct iovec *iovec, size_t iov_cnt);
	155	int (caw)(struct tcmu_device dev, struct tcmur_cmd *cmd, uint64_t off,
	156	uint64_t len, struct iovec *iovec, size_t iov_cnt);
	157
	158	/*
	159	* Notify the handler of an event.
	160	*
	161	* Return 0 on success and a -Exyz error code on error.
	162	*/
	163	int (report_event)(struct tcmu_device dev);
147	164
148	165	/*
149	166	* If the lock is acquired and the tag is not TCMU_INVALID_LOCK_TAG,

180	197	* Update the logdir called by dynamic config thread.
181	198	*/
182	199	bool (*update_logdir)(void);
	200
	201	/* To init/destroy some global resrouces if needed */
	202	int (*init)(void);
	203	void (*destroy)(void);
183	204	};
184	205
185	206	void tcmur_cmd_complete(struct tcmu_device dev, void data, int rc);

-1

tcmu-runner.spec less more

34	34	Summary: A daemon that handles the userspace side of the LIO TCM-User backstore
35	35	Group: System Environment/Daemons
36	36	License: ASL 2.0 or LGPLv2+
37		Version: 1.5.2
	37	Version: 1.5.4
38	38	URL: https://github.com/open-iscsi/tcmu-runner
39	39
40	40	#%define _RC

-0

tcmur_aio.c less more

143	143	struct tcmur_device *rdev = tcmu_dev_get_private(dev);
144	144	struct tcmu_io_queue *io_wq = &rdev->work_queue;
145	145	int ret;
	146
	147	tcmu_set_thread_name("aio", dev);
146	148
147	149	while (1) {
148	150	struct tcmu_work *work;

+105

-100

tcmur_cmd_handler.c less more

364	364	pthread_mutex_lock(&state->lock);
365	365	state->refcount++;
366	366	pthread_mutex_unlock(&state->lock);
367
	367
368	368	ret = aio_request_schedule(dev, tcmur_ucmd, unmap_work_fn,
369	369	tcmur_cmd_complete);
370	370	if (ret != TCMU_STS_ASYNC_HANDLED)

372	372
373	373	nlbas -= lbas;
374	374	lba += lbas;
375
376	375	lbas = min(opt_unmap_gran, nlbas);
377
378	376	}
379	377
380	378	return ret;

681	679	uint8_t *cdb = cmd->cdb;
682	680	uint64_t lba = tcmu_cdb_get_lba(cdb);
683	681	uint64_t nlbas = tcmu_cdb_get_xfer_length(cdb);
	682	uint32_t align = tcmu_dev_get_unmap_gran_align(dev);
684	683	struct unmap_state *state;
685	684	int ret;
	685
	686	/* If not aligned then falls back to the writesame without unmap */
	687	if (lba % align \|\| nlbas % align) {
	688	tcmu_dev_dbg(dev,
	689	"Start lba: %"PRIu64" or nlbas: %"PRIu64" not aligned to %"PRIu32"\n",
	690	lba, nlbas, align);
	691	tcmu_dev_dbg(dev, "Falls back to writesame without unmap!\n");
	692	return TCMU_STS_NOT_HANDLED;
	693	}
686	694
687	695	tcmu_dev_dbg(dev, "Do UNMAP in WRITE_SAME cmd!\n");
688	696

700	708
701	709	unmap_put(dev, cmd, ret);
702	710	return TCMU_STS_ASYNC_HANDLED;
	711	}
	712
	713	static int tcmur_writesame_work_fn(struct tcmu_device dev, void data)
	714	{
	715	struct tcmur_cmd *tcmur_cmd = data;
	716	struct tcmulib_cmd *cmd = tcmur_cmd->lib_cmd;
	717	tcmur_writesame_fn_t write_same_fn = tcmur_cmd->cmd_state;
	718	uint8_t *cdb = cmd->cdb;
	719	uint64_t off = tcmu_cdb_to_byte(dev, cdb);
	720	uint64_t len = tcmu_lba_to_byte(dev, tcmu_cdb_get_xfer_length(cdb));
	721
	722	/*
	723	* Write contents of the logical block data(from the Data-Out Buffer)
	724	* to each LBA in the specified LBA range.
	725	*/
	726	return write_same_fn(dev, tcmur_cmd, off, len, cmd->iovec,
	727	cmd->iov_cnt);
703	728	}
704	729
705	730	static int handle_writesame(struct tcmu_device dev, struct tcmulib_cmd cmd)

715	740	struct write_same *write_same;
716	741	int i, ret;
717	742
	743	if (tcmu_dev_in_recovery(dev))
	744	return TCMU_STS_BUSY;
	745
	746	ret = alua_check_state(dev, cmd, false);
	747	if (ret)
	748	return ret;
	749
718	750	ret = handle_writesame_check(dev, cmd);
719	751	if (ret)
720	752	return ret;
721	753
722		if (rhandler->unmap && (cmd->cdb[1] & 0x08))
723		return handle_unmap_in_writesame(dev, cmd);
	754	if (rhandler->unmap && (cmd->cdb[1] & 0x08)) {
	755	ret = handle_unmap_in_writesame(dev, cmd);
	756	if (ret != TCMU_STS_NOT_HANDLED)
	757	return ret;
	758	}
	759
	760	if (rhandler->writesame) {
	761	tcmur_cmd->cmd_state = rhandler->writesame;
	762	tcmur_cmd->done = handle_generic_cbk;
	763	return aio_request_schedule(dev, tcmur_cmd,
	764	tcmur_writesame_work_fn,
	765	tcmur_cmd_complete);
	766	}
724	767
725	768	max_xfer_length = tcmu_dev_get_max_xfer_len(dev) * block_size;
726	769	length = round_up(length, max_xfer_length);

745	788	start_lba, write_lbas);
746	789
747	790	return aio_request_schedule(dev, tcmur_cmd, writesame_work_fn,
748		tcmur_cmd_complete);
749		}
750
751		static int tcmur_writesame_work_fn(struct tcmu_device dev, void data)
752		{
753		struct tcmur_cmd *tcmur_cmd = data;
754		struct tcmulib_cmd *cmd = tcmur_cmd->lib_cmd;
755		tcmur_writesame_fn_t write_same_fn = tcmur_cmd->cmd_state;
756		uint8_t *cdb = cmd->cdb;
757		uint64_t off = tcmu_cdb_to_byte(dev, cdb);
758		uint64_t len = tcmu_lba_to_byte(dev, tcmu_cdb_get_xfer_length(cdb));
759
760		/*
761		* Write contents of the logical block data(from the Data-Out Buffer)
762		* to each LBA in the specified LBA range.
763		*/
764		return write_same_fn(dev, tcmur_cmd, off, len, cmd->iovec,
765		cmd->iov_cnt);
766		}
767
768		int tcmur_handle_writesame(struct tcmu_device dev, struct tcmur_cmd tcmur_cmd,
769		tcmur_writesame_fn_t write_same_fn)
770		{
771		struct tcmur_handler *rhandler = tcmu_get_runner_handler(dev);
772		struct tcmulib_cmd *cmd = tcmur_cmd->lib_cmd;
773		int ret;
774
775		if (tcmu_dev_in_recovery(dev))
776		return TCMU_STS_BUSY;
777
778		ret = alua_check_state(dev, cmd);
779		if (ret)
780		return ret;
781
782		ret = handle_writesame_check(dev, cmd);
783		if (ret)
784		return ret;
785
786		if (rhandler->unmap && (cmd->cdb[1] & 0x08))
787		return handle_unmap_in_writesame(dev, cmd);
788
789		tcmur_cmd->cmd_state = write_same_fn;
790		tcmur_cmd->done = handle_generic_cbk;
791
792		return aio_request_schedule(dev, tcmur_cmd, tcmur_writesame_work_fn,
793	791	tcmur_cmd_complete);
794	792	}
795	793

1160	1158	{
1161	1159	int i, ret;
1162	1160
	1161	if (tdll % XCOPY_TARGET_DESC_LEN) {
	1162	tcmu_dev_err(udev,
	1163	"CSCD descriptor list length %u not a multiple of %u\n",
	1164	(unsigned int)tdll, XCOPY_TARGET_DESC_LEN);
	1165	return TCMU_STS_NOTSUPP_TGT_DESC_TYPE;
	1166	}
1163	1167	/* From spc4r36q,section 6.4.3.4 CSCD DESCRIPTOR LIST LENGTH field
1164	1168	* If the number of CSCD descriptors exceeds the allowed number, the copy
1165	1169	* manager shall terminate the command with CHECK CONDITION status, with

1172	1176	return TCMU_STS_TOO_MANY_TGT_DESC;
1173	1177	}
1174	1178
1175		for (i = 0; i < RCR_OP_MAX_TARGET_DESC_COUNT; i++) {
	1179	for (i = 0; tdll >= XCOPY_TARGET_DESC_LEN; i++) {
1176	1180	/*
1177	1181	* Only Identification Descriptor Target Descriptor support
1178	1182	* for now.

1183	1187	return ret;
1184	1188
1185	1189	tgt_desc += XCOPY_TARGET_DESC_LEN;
	1190	tdll -= XCOPY_TARGET_DESC_LEN;
1186	1191	} else {
1187	1192	tcmu_dev_err(udev, "Unsupport target descriptor type code 0x%x\n",
1188	1193	tgt_desc[0]);

1190	1195	}
1191	1196	}
1192	1197
	1198	ret = TCMU_STS_CP_TGT_DEV_NOTCONN;
1193	1199	if (xcopy->src_dev)
1194	1200	ret = xcopy_locate_udev(udev->ctx, xcopy->dst_tid_wwn,
1195	1201	&xcopy->dst_dev);

1307	1313	* data, after the last segment descriptor.
1308	1314	* */
1309	1315	inline_dl = be32toh((uint32_t )&par[12]);
	1316	if (inline_dl != 0) {
	1317	tcmu_dev_err(dev, "non-zero xcopy inline_dl %u unsupported\n",
	1318	inline_dl);
	1319	ret = TCMU_STS_INVALID_PARAM_LIST_LEN;
	1320	goto err;
	1321	}
1310	1322
1311	1323	/* From spc4r31, section 6.3.1 EXTENDED COPY command introduction
1312	1324	*

1348	1360	if (ret != TCMU_STS_OK)
1349	1361	goto err;
1350	1362
	1363	/*
	1364	* tcmu-runner can't determine whether the device(s) referred to in an
	1365	* XCOPY request should be accessible to the initiator via transport
	1366	* settings, ACLs, etc. XXX Consequently, we need to fail any
	1367	* cross-device requests for safety reasons.
	1368	*/
	1369	if (dev != xcopy->src_dev \|\| dev != xcopy->dst_dev) {
	1370	tcmu_dev_err(dev, "Cross-device XCOPY not supported\n");
	1371	ret = TCMU_STS_CP_TGT_DEV_NOTCONN;
	1372	goto err;
	1373	}
	1374
1351	1375	if (tcmu_dev_get_block_size(xcopy->src_dev) !=
1352	1376	tcmu_dev_get_block_size(xcopy->dst_dev)) {
1353	1377	tcmu_dev_err(dev, "The block size of src dev %u != dst dev %u\n",

1648	1672	return TCMU_STS_OK;
1649	1673	}
1650	1674
	1675	static int tcmur_caw_fn(struct tcmu_device dev, void data)
	1676	{
	1677	struct tcmur_cmd *tcmur_cmd = data;
	1678	struct tcmulib_cmd *cmd = tcmur_cmd->lib_cmd;
	1679	tcmur_caw_fn_t caw_fn = tcmur_cmd->cmd_state;
	1680	uint64_t off = tcmu_cdb_to_byte(dev, cmd->cdb);
	1681	size_t half = (tcmu_iovec_length(cmd->iovec, cmd->iov_cnt)) / 2;
	1682
	1683	return caw_fn(dev, tcmur_cmd, off, half, cmd->iovec, cmd->iov_cnt);
	1684	}
	1685
1651	1686	static int handle_caw(struct tcmu_device dev, struct tcmulib_cmd cmd)
1652	1687	{
	1688	struct tcmur_handler *rhandler = tcmu_get_runner_handler(dev);
1653	1689	struct tcmur_cmd *tcmur_cmd = cmd->hm_private;
1654	1690	size_t half = (tcmu_iovec_length(cmd->iovec, cmd->iov_cnt)) / 2;
1655	1691	struct tcmur_device *rdev = tcmu_dev_get_private(dev);
1656	1692	uint8_t sectors = cmd->cdb[13];
1657	1693	int ret;
	1694
	1695	if (tcmu_dev_in_recovery(dev))
	1696	return TCMU_STS_BUSY;
	1697
	1698	ret = alua_check_state(dev, cmd, false);
	1699	if (ret)
	1700	return ret;
1658	1701
1659	1702	/* From sbc4r12a section 5.3 COMPARE AND WRITE command
1660	1703	* A NUMBER OF LOGICAL BLOCKS field set to zero specifies that no

1672	1715	if (ret)
1673	1716	return ret;
1674	1717
	1718	if (rhandler->caw) {
	1719	tcmur_cmd->cmd_state = rhandler->caw;
	1720	tcmur_cmd->done = handle_generic_cbk;
	1721	return aio_request_schedule(dev, tcmur_cmd, tcmur_caw_fn,
	1722	tcmur_cmd_complete);
	1723	}
	1724
1675	1725	if (tcmur_cmd_state_init(tcmur_cmd, 0, half))
1676	1726	return TCMU_STS_NO_RESOURCE;
1677	1727

1687	1737	pthread_mutex_unlock(&rdev->caw_lock);
1688	1738	tcmur_cmd_state_free(tcmur_cmd);
1689	1739	return ret;
1690		}
1691
1692		static int tcmur_caw_fn(struct tcmu_device dev, void data)
1693		{
1694		struct tcmur_cmd *tcmur_cmd = data;
1695		struct tcmulib_cmd *cmd = tcmur_cmd->lib_cmd;
1696		tcmur_caw_fn_t caw_fn = tcmur_cmd->cmd_state;
1697		uint64_t off = tcmu_cdb_to_byte(dev, cmd->cdb);
1698		size_t half = (tcmu_iovec_length(cmd->iovec, cmd->iov_cnt)) / 2;
1699
1700		return caw_fn(dev, tcmur_cmd, off, half, cmd->iovec, cmd->iov_cnt);
1701		}
1702
1703		int tcmur_handle_caw(struct tcmu_device dev, struct tcmur_cmd tcmur_cmd,
1704		tcmur_caw_fn_t caw_fn)
1705		{
1706		struct tcmulib_cmd *cmd = tcmur_cmd->lib_cmd;
1707		uint8_t sectors = cmd->cdb[13];
1708		int ret;
1709
1710		if (tcmu_dev_in_recovery(dev))
1711		return TCMU_STS_BUSY;
1712
1713		/* From sbc4r12a section 5.3 COMPARE AND WRITE command
1714		* A NUMBER OF LOGICAL BLOCKS field set to zero specifies that no
1715		* read operations shall be performed, no logical block data shall
1716		* be transferred from the Data-Out Buffer, no compare operations
1717		* shall be performed, and no write operations shall be performed.
1718		* This condition shall not be considered an error.
1719		*/
1720		if (!sectors) {
1721		tcmu_dev_dbg(dev, "NUMBER OF LOGICAL BLOCKS is zero, just return ok.\n");
1722		return TCMU_STS_OK;
1723		}
1724
1725		ret = alua_check_state(dev, cmd);
1726		if (ret)
1727		return ret;
1728
1729		ret = handle_caw_check(dev, cmd);
1730		if (ret)
1731		return ret;
1732
1733		tcmur_cmd->cmd_state = caw_fn;
1734		tcmur_cmd->done = handle_generic_cbk;
1735
1736		return aio_request_schedule(dev, tcmur_cmd, tcmur_caw_fn,
1737		tcmur_cmd_complete);
1738	1740	}
1739	1741
1740	1742	/* async flush */

2117	2119	struct tcmur_handler *rhandler = tcmu_get_runner_handler(dev);
2118	2120	struct tcmur_device *rdev = tcmu_dev_get_private(dev);
2119	2121	uint8_t *cdb = cmd->cdb;
	2122	bool is_read = false;
2120	2123
2121	2124	track_aio_request_start(rdev);
2122	2125

2127	2130
2128	2131	/* Don't perform alua implicit transition if command is not supported */
2129	2132	switch(cdb[0]) {
	2133	/* Skip to grab the lock for reads */
2130	2134	case READ_6:
2131	2135	case READ_10:
2132	2136	case READ_12:
2133	2137	case READ_16:
	2138	is_read = true;
2134	2139	case WRITE_6:
2135	2140	case WRITE_10:
2136	2141	case WRITE_12:

2145	2150	case WRITE_SAME:
2146	2151	case WRITE_SAME_16:
2147	2152	case FORMAT_UNIT:
2148		ret = alua_check_state(dev, cmd);
	2153	ret = alua_check_state(dev, cmd, is_read);
2149	2154	if (ret)
2150	2155	goto untrack;
2151	2156	break;

-4

tcmur_cmd_handler.h less more

23	23	bool tcmur_handler_is_passthrough_only(struct tcmur_handler *rhandler);
24	24	void tcmur_tcmulib_cmd_complete(struct tcmu_device *dev,
25	25	struct tcmulib_cmd *cmd, int ret);
	26
26	27	typedef int (tcmur_writesame_fn_t)(struct tcmu_device dev,
27	28	struct tcmur_cmd *tcmur_cmd, uint64_t off,
28	29	uint64_t len, struct iovec *iov,
29	30	size_t iov_cnt);
30		int tcmur_handle_writesame(struct tcmu_device dev, struct tcmur_cmd tcmur_cmd,
31		tcmur_writesame_fn_t write_same_fn);
32	31
33	32	typedef int (tcmur_caw_fn_t)(struct tcmu_device dev,
34	33	struct tcmur_cmd *tcmur_cmd, uint64_t off,
35	34	uint64_t len, struct iovec *iov, size_t iov_cnt);
36		int tcmur_handle_caw(struct tcmu_device dev, struct tcmur_cmd tcmur_cmd,
37		tcmur_caw_fn_t caw_fn);
38	35
39	36	#endif /* __TCMUR_CMD_HANDLER_H */

+154

-86

tcmur_device.c less more

19	19	#include "tcmu-runner.h"
20	20	#include "tcmur_device.h"
21	21	#include "tcmur_cmd_handler.h"
	22	#include "tcmur_work.h"
22	23	#include "tcmu_runner_priv.h"
23	24	#include "target.h"
24	25

43	44	struct tcmur_handler *rhandler = tcmu_get_runner_handler(dev);
44	45	int ret, attempt = 0;
45	46	bool needs_close = false;
46		bool cancel_lock = false;
47	47
48	48	pthread_mutex_lock(&rdev->state_lock);
49	49	if (rdev->flags & TCMUR_DEV_FLAG_STOPPING) {
50	50	ret = 0;
51	51	goto done;
52	52	}
53
54		if (rdev->lock_state == TCMUR_DEV_LOCK_LOCKING &&
55		pthread_self() != rdev->lock_thread)
56		cancel_lock = true;
57	53	pthread_mutex_unlock(&rdev->state_lock);
58	54
59	55	/*

61	57	* async lock requests in progress that might be accessing
62	58	* the device.
63	59	*/
64		if (cancel_lock)
65		tcmu_cancel_lock_thread(dev);
66
67		/*
68		* Force a reacquisition of the lock when we have reopend the
69		* device, so it can update state. If we are being called from
70		* the lock code path then do not change state.
71		*/
72		pthread_mutex_lock(&rdev->state_lock);
73		if (rdev->lock_state != TCMUR_DEV_LOCK_LOCKING)
74		rdev->lock_state = TCMUR_DEV_LOCK_UNLOCKED;
75
	60	tcmur_flush_work(rdev->event_work);
	61
	62	pthread_mutex_lock(&rdev->state_lock);
76	63	if (rdev->flags & TCMUR_DEV_FLAG_IS_OPEN)
77	64	needs_close = true;
78	65	rdev->flags &= ~TCMUR_DEV_FLAG_IS_OPEN;

164	151	}
165	152	pthread_mutex_unlock(&rdev->state_lock);
166	153	tcmu_dev_dbg(dev, "Recovery thread wait done\n");
	154	}
	155
	156	static void __tcmu_report_event(void *data)
	157	{
	158	struct tcmu_device *dev = data;
	159	struct tcmur_device *rdev = tcmu_dev_get_private(dev);
	160	struct tcmur_handler *rhandler = tcmu_get_runner_handler(dev);
	161	int ret;
	162
	163	/*
	164	* For cmd timeouts and unbalanced systems we will get a burst so wait
	165	* a second to batch up the updates.
	166	*/
	167	sleep(1);
	168
	169	pthread_mutex_lock(&rdev->state_lock);
	170	ret = rhandler->report_event(dev);
	171	if (ret)
	172	tcmu_dev_err(dev, "Could not report events. Error %d.\n", ret);
	173	pthread_mutex_unlock(&rdev->state_lock);
	174	}
	175
	176	static void tcmu_report_event(struct tcmu_device *dev)
	177	{
	178	struct tcmur_handler *rhandler = tcmu_get_runner_handler(dev);
	179	struct tcmur_device *rdev = tcmu_dev_get_private(dev);
	180	int ret;
	181
	182	if (!rhandler->report_event)
	183	return;
	184
	185	ret = tcmur_run_work(rdev->event_work, dev, __tcmu_report_event);
	186	if (!ret)
	187	return;
	188
	189	if (ret != -EBUSY)
	190	tcmu_dev_err(dev, "Could not execute event work. Error %d", ret);
	191	}
	192
	193	static bool __tcmu_notify_conn_lost(struct tcmu_device *dev)
	194	{
	195	struct tcmur_device *rdev = tcmu_dev_get_private(dev);
	196
	197	/*
	198	* Although there are 2 checks for STOPPING in __tcmu_reopen_dev
	199	* which is called a little later by the recovery thread, STOPPING
	200	* checking is still needed here.
	201	*
	202	* In device removal, tcmu_get_alua_grps will never get access to
	203	* configfs dir resource which is holded by kernel in configfs_rmdir,
	204	* thus tcmulib_cmd->done() will never get a chance to clear
	205	* tracked_aio_ops. This will cause a deadlock in dev_removed
	206	* which is polling tracked_aio_ops.
	207	*/
	208	if ((rdev->flags & TCMUR_DEV_FLAG_STOPPING) \|\|
	209	(rdev->flags & TCMUR_DEV_FLAG_IN_RECOVERY))
	210	return false;
	211
	212	tcmu_dev_err(dev, "Handler connection lost (lock state %d)\n",
	213	rdev->lock_state);
	214
	215	if (!tcmu_add_dev_to_recovery_list(dev)) {
	216	rdev->flags \|= TCMUR_DEV_FLAG_IN_RECOVERY;
	217	rdev->conn_lost_cnt++;
	218	return true;
	219	}
	220
	221	return false;
167	222	}
168	223
169	224	/**

178	233	void tcmu_notify_conn_lost(struct tcmu_device *dev)
179	234	{
180	235	struct tcmur_device *rdev = tcmu_dev_get_private(dev);
181
182		pthread_mutex_lock(&rdev->state_lock);
183
184		/*
185		* Although there are 2 checks for STOPPING in __tcmu_reopen_dev
186		* which is called a little later by the recovery thread, STOPPING
187		* checking is still needed here.
188		*
189		* In device removal, tcmu_get_alua_grps will never get access to
190		* configfs dir resource which is holded by kernel in configfs_rmdir,
191		* thus tcmulib_cmd->done() will never get a chance to clear
192		* tracked_aio_ops. This will cause a deadlock in dev_removed
193		* which is polling tracked_aio_ops.
194		*/
195		if ((rdev->flags & TCMUR_DEV_FLAG_STOPPING) \|\|
196		(rdev->flags & TCMUR_DEV_FLAG_IN_RECOVERY))
197		goto unlock;
198
199		tcmu_dev_err(dev, "Handler connection lost (lock state %d)\n",
200		rdev->lock_state);
201
202		if (!tcmu_add_dev_to_recovery_list(dev))
203		rdev->flags \|= TCMUR_DEV_FLAG_IN_RECOVERY;
204		unlock:
205		pthread_mutex_unlock(&rdev->state_lock);
	236	bool report;
	237
	238	pthread_mutex_lock(&rdev->state_lock);
	239	report =__tcmu_notify_conn_lost(dev);
	240	pthread_mutex_unlock(&rdev->state_lock);
	241
	242	if (report)
	243	tcmu_report_event(dev);
	244	}
	245
	246	static void __tcmu_notify_lock_lost(struct tcmu_device *dev)
	247	{
	248	struct tcmur_device *rdev = tcmu_dev_get_private(dev);
	249
	250	rdev->lock_lost = true;
	251	rdev->lock_state = TCMUR_DEV_LOCK_UNLOCKED;
	252	rdev->lock_lost_cnt++;
	253
	254	tcmu_report_event(dev);
206	255	}
207	256
208	257	/**

223	272	* We could be getting stale IO completions. If we are trying to
224	273	* reaquire the lock do not change state.
225	274	*/
226		if (rdev->lock_state != TCMUR_DEV_LOCK_LOCKING) {
227		rdev->lock_lost = true;
228		rdev->lock_state = TCMUR_DEV_LOCK_UNLOCKED;
229		}
230		pthread_mutex_unlock(&rdev->state_lock);
231		}
232
233		int tcmu_cancel_lock_thread(struct tcmu_device *dev)
234		{
	275	if (rdev->lock_state != TCMUR_DEV_LOCK_WRITE_LOCKING) {
	276	__tcmu_notify_lock_lost(dev);
	277	}
	278	pthread_mutex_unlock(&rdev->state_lock);
	279	}
	280
	281	void tcmu_release_dev_lock(struct tcmu_device *dev)
	282	{
	283	struct tcmur_handler *rhandler = tcmu_get_runner_handler(dev);
235	284	struct tcmur_device *rdev = tcmu_dev_get_private(dev);
236	285	int ret;
237	286
238	287	pthread_mutex_lock(&rdev->state_lock);
239		if (rdev->lock_state != TCMUR_DEV_LOCK_LOCKING) {
240		pthread_mutex_unlock(&rdev->state_lock);
241		return 0;
242		}
243		/*
244		* It looks like lock calls are not cancelable, so
245		* we wait here to avoid crashes.
246		*/
247		tcmu_dev_dbg(rdev->dev, "waiting for lock thread to exit\n");
248		ret = pthread_cond_wait(&rdev->lock_cond, &rdev->state_lock);
249		pthread_mutex_unlock(&rdev->state_lock);
250
251		return ret;
252		}
253
254		void tcmu_release_dev_lock(struct tcmu_device *dev)
255		{
256		struct tcmur_handler *rhandler = tcmu_get_runner_handler(dev);
257		struct tcmur_device *rdev = tcmu_dev_get_private(dev);
258		int ret;
259
260		pthread_mutex_lock(&rdev->state_lock);
261		if (rdev->lock_state != TCMUR_DEV_LOCK_LOCKED) {
	288	if (rdev->lock_state != TCMUR_DEV_LOCK_WRITE_LOCKED) {
262	289	pthread_mutex_unlock(&rdev->state_lock);
263	290	return;
264	291	}

375	402	goto done;
376	403	}
377	404
	405	/*
	406	* Since we are here the lock state must be one of:
	407	* for implicit:
	408	* TCMUR_DEV_LOCK_READ_LOCKING
	409	* TCMUR_DEV_LOCK_WRITE_LOCKING
	410	*
	411	* for explicit:
	412	* TCMUR_DEV_LOCK_UNLOCKED
	413	* TCMUR_DEV_LOCK_UNKNOWN
	414	*/
	415
378	416	reopen = false;
379	417	pthread_mutex_lock(&rdev->state_lock);
380		if (rdev->lock_lost \|\| !(rdev->flags & TCMUR_DEV_FLAG_IS_OPEN)) {
	418	if (rdev->lock_lost \|\| !(rdev->flags & TCMUR_DEV_FLAG_IS_OPEN))
381	419	reopen = true;
382		}
383	420	pthread_mutex_unlock(&rdev->state_lock);
384	421
385	422	retry:

397	434	goto drop_conn;
398	435	}
399	436	}
	437
	438	pthread_mutex_lock(&rdev->state_lock);
	439	if (rdev->lock_state == TCMUR_DEV_LOCK_READ_LOCKING) {
	440	pthread_mutex_unlock(&rdev->state_lock);
	441	ret = TCMU_STS_OK;
	442	goto done;
	443	}
	444	pthread_mutex_unlock(&rdev->state_lock);
400	445
401	446	ret = rhandler->lock(dev, tag);
402	447	if (ret == TCMU_STS_FENCED) {

431	476
432	477	/* TODO: set UA based on bgly's patches */
433	478	pthread_mutex_lock(&rdev->state_lock);
434		if (ret == TCMU_STS_OK)
435		rdev->lock_state = TCMUR_DEV_LOCK_LOCKED;
436		else
	479	if (ret != TCMU_STS_OK) {
437	480	rdev->lock_state = TCMUR_DEV_LOCK_UNLOCKED;
438
439		tcmu_dev_dbg(dev, "lock call done. lock state %d\n", rdev->lock_state);
	481	tcmu_dev_info(dev, "Lock acquisition unsuccessful\n");
	482	} else {
	483	if (rdev->lock_state == TCMUR_DEV_LOCK_READ_LOCKING) {
	484	rdev->lock_state = TCMUR_DEV_LOCK_READ_LOCKED;
	485	tcmu_dev_info(dev, "Read lock acquisition successful\n");
	486	} else if (rdev->lock_state == TCMUR_DEV_LOCK_WRITE_LOCKING) {
	487	rdev->lock_state = TCMUR_DEV_LOCK_WRITE_LOCKED;
	488	tcmu_dev_info(dev, "Write lock acquisition successful\n");
	489	} else {
	490	/*
	491	* For explicit transition it will always acquire the write lock.
	492	*/
	493	rdev->lock_state = TCMUR_DEV_LOCK_WRITE_LOCKED;
	494	tcmu_dev_info(dev, "Write lock acquisition successful\n");
	495	}
	496	}
	497
440	498	tcmu_cfgfs_dev_exec_action(dev, "block_dev", 0);
441	499
442		pthread_cond_signal(&rdev->lock_cond);
443	500	pthread_mutex_unlock(&rdev->state_lock);
444	501
445	502	return ret;

465	522	state = rhandler->get_lock_state(dev);
466	523	pthread_mutex_lock(&rdev->state_lock);
467	524	check_state:
468		if (rdev->lock_state == TCMUR_DEV_LOCK_LOCKED &&
469		state != TCMUR_DEV_LOCK_LOCKED) {
	525	if (rdev->lock_state == TCMUR_DEV_LOCK_WRITE_LOCKED &&
	526	state != TCMUR_DEV_LOCK_WRITE_LOCKED) {
470	527	tcmu_dev_dbg(dev, "Updated out of sync lock state.\n");
471		rdev->lock_state = TCMUR_DEV_LOCK_UNLOCKED;
472		rdev->lock_lost = true;
	528	__tcmu_notify_lock_lost(dev);
473	529	}
474	530	pthread_mutex_unlock(&rdev->state_lock);
475	531	}

487	543
488	544	return rdev->hm_private;
489	545	}
	546
	547	void tcmu_notify_cmd_timed_out(struct tcmu_device *dev)
	548	{
	549	struct tcmur_device *rdev = tcmu_dev_get_private(dev);
	550
	551	pthread_mutex_lock(&rdev->state_lock);
	552	rdev->cmd_timed_out_cnt++;
	553	__tcmu_notify_conn_lost(dev);
	554	pthread_mutex_unlock(&rdev->state_lock);
	555
	556	tcmu_report_event(dev);
	557	}

+14

-7

tcmur_device.h less more

31	31	};
32	32
33	33	enum {
	34	TCMUR_DEV_LOCK_UNKNOWN,
34	35	TCMUR_DEV_LOCK_UNLOCKED,
35		TCMUR_DEV_LOCK_LOCKED,
36		TCMUR_DEV_LOCK_LOCKING,
37		TCMUR_DEV_LOCK_UNKNOWN,
	36	TCMUR_DEV_LOCK_READ_LOCKING,
	37	TCMUR_DEV_LOCK_READ_LOCKED,
	38	TCMUR_DEV_LOCK_WRITE_LOCKING,
	39	TCMUR_DEV_LOCK_WRITE_LOCKED,
38	40	};
	41
	42	struct tcmur_work;
39	43
40	44	struct tcmur_device {
41	45	struct tcmu_device *dev;

47	51	uint32_t flags;
48	52	uint8_t failover_type;
49	53
50		pthread_t recovery_thread;
51	54	struct list_node recovery_entry;
	55
	56	/* tcmur_event counters */
	57	uint64_t lock_lost_cnt;
	58	uint64_t conn_lost_cnt;
	59	uint64_t cmd_timed_out_cnt;
	60	struct tcmur_work *event_work;
52	61
53	62	bool lock_lost;
54	63	uint8_t lock_state;
55		pthread_t lock_thread;
56		pthread_cond_t lock_cond;
57	64
58	65	/* General lock for lock state, thread, dev state, etc */
59	66	pthread_mutex_t state_lock;

79	86
80	87	bool tcmu_dev_in_recovery(struct tcmu_device *dev);
81	88	void tcmu_cancel_recovery(struct tcmu_device *dev);
82		int tcmu_cancel_lock_thread(struct tcmu_device *dev);
83	89
84	90	void tcmu_notify_conn_lost(struct tcmu_device *dev);
85	91	void tcmu_notify_lock_lost(struct tcmu_device *dev);
	92	void tcmu_notify_cmd_timed_out(struct tcmu_device *dev);
86	93
87	94	int __tcmu_reopen_dev(struct tcmu_device *dev, int retries);
88	95	int tcmu_reopen_dev(struct tcmu_device *dev, int retries);

+131

-0

tcmur_work.c less more

	0	/*
	1	* Copyright (c) 2020 Red Hat, Inc.
	2	*
	3	* This file is licensed to you under your choice of the GNU Lesser
	4	* General Public License, version 2.1 or any later version (LGPLv2.1 or
	5	* later), or the Apache License 2.0.
	6	*/
	7	#define _GNU_SOURCE
	8	#include <stdlib.h>
	9	#include <stdint.h>
	10	#include <errno.h>
	11	#include <pthread.h>
	12
	13	#include "libtcmu.h"
	14	#include "libtcmu_log.h"
	15	#include "tcmur_device.h"
	16	#include "tcmur_work.h"
	17
	18	struct tcmur_work *tcmur_create_work(void)
	19	{
	20	struct tcmur_work *work;
	21
	22	work = calloc(1, sizeof(*work));
	23	if (!work)
	24	return NULL;
	25
	26	if (pthread_mutex_init(&work->lock, NULL))
	27	goto free_work;
	28
	29	if (pthread_cond_init(&work->cond, NULL))
	30	goto destroy_mutex;
	31
	32	return work;
	33
	34	destroy_mutex:
	35	pthread_mutex_destroy(&work->lock);
	36	free_work:
	37	free(work);
	38	return NULL;
	39	}
	40
	41	static void __tcmur_flush_work(struct tcmur_work *work)
	42	{
	43	char pname[TCMU_THREAD_NAME_LEN];
	44
	45	if (pthread_getname_np(pthread_self(), pname, TCMU_THREAD_NAME_LEN))
	46	return;
	47
	48	/*
	49	* The event work thread may need to do a handler reopen
	50	* call and try to flush itself. Just ignore.
	51	*/
	52	if (!strcmp(pname, "ework-thread"))
	53	return;
	54
	55	/*
	56	* Some handlers will crash if we do a cancel so we just wait.
	57	*/
	58	tcmu_dbg("waiting for %d work thread to complete\n", work->refcnt);
	59	if (work->refcnt)
	60	pthread_cond_wait(&work->cond, &work->lock);
	61	}
	62
	63	void tcmur_flush_work(struct tcmur_work *work)
	64	{
	65	pthread_mutex_lock(&work->lock);
	66	__tcmur_flush_work(work);
	67	pthread_mutex_unlock(&work->lock);
	68	}
	69
	70	struct private {
	71	void *data;
	72	void (work_fn)(void );
	73	struct tcmur_work *work;
	74	};
	75
	76	static void tcmur_work_fn(void data)
	77	{
	78	struct private *p = data;
	79
	80	tcmu_set_thread_name("ework-thread", NULL);
	81
	82	p->work_fn(p->data);
	83
	84	pthread_mutex_lock(&p->work->lock);
	85	if (--p->work->refcnt == 0)
	86	pthread_cond_signal(&p->work->cond);
	87	pthread_mutex_unlock(&p->work->lock);
	88
	89	free(p);
	90	return NULL;
	91	}
	92
	93	int tcmur_run_work(struct tcmur_work work, void data, void (work_fn)(void ))
	94	{
	95	pthread_attr_t attr;
	96	pthread_t thread;
	97	struct private *p;
	98	int ret;
	99
	100	p = malloc(sizeof(struct private));
	101	if (!p)
	102	return -ENOMEM;
	103
	104	p->data = data;
	105	p->work_fn = work_fn;
	106	p->work = work;
	107
	108	pthread_attr_init(&attr);
	109	pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
	110
	111	pthread_mutex_lock(&work->lock);
	112	ret = pthread_create(&thread, &attr, tcmur_work_fn, p);
	113	if (!ret)
	114	work->refcnt++;
	115	pthread_mutex_unlock(&work->lock);
	116
	117	pthread_attr_destroy(&attr);
	118
	119	if (ret)
	120	free(p);
	121	return ret;
	122	}
	123
	124	void tcmur_destroy_work(struct tcmur_work *work)
	125	{
	126	tcmur_flush_work(work);
	127	pthread_mutex_destroy(&work->lock);
	128	pthread_cond_destroy(&work->cond);
	129	free(work);
	130	}

+30

-0

tcmur_work.h less more

	0	/*
	1	* Copyright (c) 2020 Red Hat, Inc.
	2	*
	3	* This file is licensed to you under your choice of the GNU Lesser
	4	* General Public License, version 2.1 or any later version (LGPLv2.1 or
	5	* later), or the Apache License 2.0.
	6	*/
	7
	8	#ifndef __TCMU_WORK_H
	9	#define __TCMU_WORK_H
	10
	11	#include <pthread.h>
	12
	13	#include "ccan/list/list.h"
	14
	15	struct tcmu_device;
	16
	17	struct tcmur_work {
	18	pthread_mutex_t lock;
	19	pthread_cond_t cond;
	20	int refcnt;
	21	};
	22
	23	struct tcmur_work *tcmur_create_work(void);
	24	void tcmur_destroy_work(struct tcmur_work *work);
	25	int tcmur_run_work(struct tcmur_work work, void data,
	26	void (work_fn)(void ));
	27	void tcmur_flush_work(struct tcmur_work *work);
	28
	29	#endif

-0

version.h.in less more

3	3	#define DEFAULT_HANDLER_PATH "@tcmu-runner_HANDLER_PATH@"
4	4
5	5	#define GFAPI_VERSION760 @GFAPI_VERSION760@
	6
	7	#define GFAPI_VERSION766 @GFAPI_VERSION766@