DAOS-6159 test: Run fault injection tests in parallel and prepare for IL testing.

src/client/array/dc_array.c

@@ -515,10 +515,8 @@ write_md_cb(tse_task_t *task, void *data)
 	}
 
 	D_ALLOC_PTR(params);
-	if (params == NULL) {
-		D_ERROR("Failed memory allocation\n");
+	if (params == NULL)
 		return -DER_NOMEM;
-	}
 
 	params->md_vals[0] = AKEY_MAGIC_V;
 	params->md_vals[1] = args->cell_size;
@@ -760,7 +758,8 @@ dc_array_open(tse_task_t *task)
 	rc = daos_task_create(DAOS_OPC_OBJ_OPEN, tse_task2sched(task),
 			      0, NULL, &open_task);
 	if (rc != 0) {
-		D_ERROR("Failed to open object_open task\n");
+		D_ERROR("Failed to open object_open task "DF_RC"\n",
+			DP_RC(rc));
 		D_GOTO(err_ptask, rc);
 	}
 
@@ -775,14 +774,16 @@ dc_array_open(tse_task_t *task)
 		/** The upper task completes when the open task completes */
 		rc = tse_task_register_deps(task, 1, &open_task);
 		if (rc != 0) {
-			D_ERROR("Failed to register dependency\n");
+			D_ERROR("Failed to register dependency "DF_RC"\n",
+				DP_RC(rc));
 			D_GOTO(err_put1, rc);
 		}
 
 		rc = tse_task_register_comp_cb(task, open_handle_cb, &args,
 					       sizeof(args));
 		if (rc != 0) {
-			D_ERROR("Failed to register completion cb\n");
+			D_ERROR("Failed to register completion cb "DF_RC"\n",
+				DP_RC(rc));
 			D_GOTO(err_put1, rc);
 		}
 
@@ -820,10 +821,9 @@ dc_array_open(tse_task_t *task)
 	 * the open handle.
 	 */
 	D_ALLOC_PTR(params);
-	if (params == NULL) {
-		D_ERROR("Failed memory allocation\n");
+	if (params == NULL)
 		D_GOTO(err_put2, rc = -DER_NOMEM);
-	}
+
 
 	rc = tse_task_register_comp_cb(task, free_md_params_cb, &params,
 				       sizeof(params));
@@ -876,7 +876,8 @@ dc_array_close(tse_task_t *task)
 	rc = daos_task_create(DAOS_OPC_OBJ_CLOSE, tse_task2sched(task),
 			      0, NULL, &close_task);
 	if (rc != 0) {
-		D_ERROR("Failed to create object_close task\n");
+		D_ERROR("Failed to create object_close task "DF_RC"\n",
+			DP_RC(rc));
 		D_GOTO(err_put1, rc);
 	}
 	close_args = daos_task_get_args(close_task);
@@ -885,15 +886,17 @@ dc_array_close(tse_task_t *task)
 	/** The upper task completes when the close task completes */
 	rc = tse_task_register_deps(task, 1, &close_task);
 	if (rc != 0) {
-		D_ERROR("Failed to register dependency\n");
+		D_ERROR("Failed to register dependency "DF_RC"\n",
+			DP_RC(rc));
 		D_GOTO(err_put2, rc);
 	}
 
 	/** Add a completion CB on the upper task to free the array */
 	rc = tse_task_register_cbs(task, NULL, NULL, 0, free_handle_cb,
 				   &array, sizeof(array));
 	if (rc != 0) {
-		D_ERROR("Failed to register completion cb\n");
+		D_ERROR("Failed to register completion cb "DF_RC"\n",
+			DP_RC(rc));
 		D_GOTO(err_put2, rc);
 	}
 
@@ -1247,7 +1250,7 @@ set_short_read_cb(tse_task_t *task, void *data)
 	int			rc = task->dt_result;
 
 	if (rc != 0) {
-		D_ERROR("Failed to get array size (%d)\n", rc);
+		D_ERROR("Failed to get array size "DF_RC"\n", DP_RC(rc));
 		return rc;
 	}
 
@@ -1298,7 +1301,7 @@ check_short_read_cb(tse_task_t *task, void *data)
 	int			rc = task->dt_result;
 
 	if (rc != 0) {
-		D_ERROR("Array Read Failed (%d)\n", rc);
+		D_ERROR("Array Read Failed "DF_RC"\n", DP_RC(rc));
 		return rc;
 	}
 
@@ -1427,6 +1430,7 @@ dc_array_io(daos_handle_t array_oh, daos_handle_t th,
 	daos_size_t	num_records;
 	daos_off_t	record_i;
 	struct io_params *head = NULL;
+	bool		head_cb_registered = false;
 	daos_size_t	num_ios;
 	d_list_t	io_task_list;
 	daos_size_t	tot_num_records = 0;
@@ -1520,10 +1524,8 @@ dc_array_io(daos_handle_t array_oh, daos_handle_t th,
 
 		/** allocate params for this dkey io */
 		D_ALLOC_PTR(params);
-		if (params == NULL) {
-			D_ERROR("Failed memory allocation\n");
+		if (params == NULL)
 			D_GOTO(err_stask, rc = -DER_NOMEM);
-		}
 		params->dkey_val = dkey_val;
 
 		/*
@@ -1693,7 +1695,8 @@ dc_array_io(daos_handle_t array_oh, daos_handle_t th,
 			rc = create_sgl(user_sgl, array->cell_size,
 					dkey_records, &cur_off, &cur_i, sgl);
 			if (rc != 0) {
-				D_ERROR("Failed to create sgl\n");
+				D_ERROR("Failed to create sgl "DF_RC"\n",
+					DP_RC(rc));
 				D_GOTO(err_stask, rc);
 			}
 		}
@@ -1708,8 +1711,8 @@ dc_array_io(daos_handle_t array_oh, daos_handle_t th,
 					      tse_task2sched(task),
 					      0, NULL, &io_task);
 			if (rc != 0) {
-				D_ERROR("Fetch dkey "DF_U64" failed (%d)\n",
-					params->dkey_val, rc);
+				D_ERROR("Fetch dkey "DF_U64" failed "DF_RC"\n",
+					params->dkey_val, DP_RC(rc));
 				D_GOTO(err_iotask, rc);
 			}
 			io_arg = daos_task_get_args(io_task);
@@ -1768,6 +1771,7 @@ dc_array_io(daos_handle_t array_oh, daos_handle_t th,
 				       sizeof(head));
 	if (rc)
 		D_GOTO(err_iotask, rc);
+	head_cb_registered = true;
 
 	/*
 	 * If this is a byte array, schedule the get_size task with a prep
@@ -1781,10 +1785,8 @@ dc_array_io(daos_handle_t array_oh, daos_handle_t th,
 			struct hole_params	*sparams;
 
 			D_ALLOC_PTR(sparams);
-			if (sparams == NULL) {
-				D_ERROR("Failed memory allocation\n");
+			if (sparams == NULL)
 				D_GOTO(err_iotask, rc = -DER_NOMEM);
-			}
 
 			sparams->io_list	= head;
 			sparams->records_req    = tot_num_records;
@@ -1823,7 +1825,7 @@ dc_array_io(daos_handle_t array_oh, daos_handle_t th,
 	if (op_type == DAOS_OPC_ARRAY_READ && array->byte_array)
 		tse_task_complete(stask, rc);
 err_task:
-	if (head)
+	if (head && !head_cb_registered)
 		tse_task_register_comp_cb(task, free_io_params_cb, &head,
 					  sizeof(head));
 	if (array)
@@ -2032,10 +2034,8 @@ punch_key(daos_handle_t oh, daos_handle_t th, daos_size_t dkey_val,
 	int			rc;
 
 	D_ALLOC_PTR(params);
-	if (params == NULL) {
-		D_ERROR("Failed memory allocation\n");
+	if (params == NULL)
 		return -DER_NOMEM;
-	}
 
 	params->dkey_val = dkey_val;
 	dkey = &params->dkey;
@@ -2095,10 +2095,8 @@ punch_extent(daos_handle_t oh, daos_handle_t th, daos_size_t dkey_val,
 		record_i + 1, num_records, dkey_val);
 
 	D_ALLOC_PTR(params);
-	if (params == NULL) {
-		D_ERROR("Failed memory allocation\n");
+	if (params == NULL)
 		return -DER_NOMEM;
-	}
 
 	iod = &params->iod;
 	sgl = NULL;
@@ -2251,10 +2249,8 @@ check_record(daos_handle_t oh, daos_handle_t th, daos_size_t dkey_val,
 	int			rc;
 
 	D_ALLOC_PTR(params);
-	if (params == NULL) {
-		D_ERROR("Failed memory allocation\n");
+	if (params == NULL)
 		return -DER_NOMEM;
-	}
 
 	iod = &params->iod;
 	sgl = NULL;
@@ -2325,10 +2321,8 @@ add_record(daos_handle_t oh, daos_handle_t th, struct set_size_props *props)
 	int			rc;
 
 	D_ALLOC_PTR(params);
-	if (params == NULL) {
-		D_ERROR("Failed memory allocation\n");
+	if (params == NULL)
 		return -DER_NOMEM;
-	}
 
 	iod = &params->iod;
 	sgl = &params->sgl;

src/client/dfuse/il/int_posix.c

@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2017-2020 Intel Corporation.
+ * (C) Copyright 2017-2021 Intel Corporation.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -93,7 +93,29 @@ static const char * const bypass_status[] = {
 /* Unwind after close or error on container.  Closes container handle
  * and also pool handle if last container is closed.
  *
+ * ioil_shrink_pool() is only used in ioil_fini() where stale pools
+ * have been left open, for example if there are problems on close.
  */
+
+static void
+ioil_shrink_pool(struct ioil_pool *pool)
+{
+
+	if (daos_handle_is_valid(pool->iop_poh)) {
+		int rc;
+
+		rc = daos_pool_disconnect(pool->iop_poh, NULL);
+		if (rc != 0) {
+			D_ERROR("daos_pool_disconnect() failed, " DF_RC "\n",
+				DP_RC(rc));
+			return;
+		}
+		pool->iop_poh = DAOS_HDL_INVAL;
+	}
+	d_list_del(&pool->iop_pools);
+	D_FREE(pool);
+}
+
 static void
 ioil_shrink(struct ioil_cont *cont)
 {
@@ -105,15 +127,21 @@ ioil_shrink(struct ioil_cont *cont)
 
 	if (cont->ioc_dfs != NULL) {
 		rc = dfs_umount(cont->ioc_dfs);
-		if (rc != 0)
+		if (rc != 0) {
 			D_ERROR("dfs_umount() failed, %d\n", rc);
+			return;
+		}
+		cont->ioc_dfs = NULL;
 	}
 
-	if (!daos_handle_is_inval(cont->ioc_coh)) {
+	if (daos_handle_is_valid(cont->ioc_coh)) {
 		rc = daos_cont_close(cont->ioc_coh, NULL);
-		if (rc != 0)
+		if (rc != 0) {
 			D_ERROR("daos_cont_close() failed, " DF_RC "\n",
 				DP_RC(rc));
+			return;
+		}
+		cont->ioc_coh = DAOS_HDL_INVAL;
 	}
 
 	pool = cont->ioc_pool;
@@ -123,13 +151,7 @@ ioil_shrink(struct ioil_cont *cont)
 	if (!d_list_empty(&pool->iop_container_head))
 		return;
 
-	rc = daos_pool_disconnect(pool->iop_poh, NULL);
-	if (rc != 0)
-		D_ERROR("daos_pool_disconnect() failed, " DF_RC "\n",
-			DP_RC(rc));
-
-	d_list_del(&pool->iop_pools);
-	D_FREE(pool);
+	ioil_shrink_pool(pool);
 }
 
 static void
@@ -157,7 +179,8 @@ ioil_initialize_fd_table(int max_fds)
 			 entry_array_close);
 	if (rc != 0)
 		DFUSE_LOG_ERROR("Could not allocate file descriptor table"
-				", disabling kernel bypass: rc = %d", rc);
+				", disabling kernel bypass: rc = " DF_RC,
+				DP_RC(rc));
 	return rc;
 }
 
@@ -244,7 +267,10 @@ ioil_init(void)
 
 	D_INIT_LIST_HEAD(&ioil_iog.iog_pools_head);
 
-	daos_debug_init(DAOS_LOG_DEFAULT);
+	rc = daos_debug_init(DAOS_LOG_DEFAULT);
+	if (rc) {
+		ioil_iog.iog_no_daos = true;
+	}
 
 	DFUSE_TRA_ROOT(&ioil_iog, "il");
 
@@ -253,15 +279,14 @@ ioil_init(void)
 	if (rc != 0) {
 		DFUSE_LOG_ERROR("Could not get process file descriptor limit"
 				", disabling kernel bypass");
-		printf("Failed\n");
 		return;
 	}
 
 	rc = ioil_initialize_fd_table(rlimit.rlim_max);
 	if (rc != 0) {
-		DFUSE_LOG_ERROR("Could not create fd_table, rc = %d,"
-				", disabling kernel bypass", rc);
-		printf("Failed.\n");
+		DFUSE_LOG_ERROR("Could not create fd_table, "
+				"disabling kernel bypass, rc = "DF_RC,
+				DP_RC(rc));
 		return;
 	}
 
@@ -283,6 +308,7 @@ ioil_fini(void)
 	DFUSE_TRA_DOWN(&ioil_iog);
 	vector_destroy(&fd_table);
 
+	/* Tidy up any remaining open connections */
 	d_list_for_each_entry_safe(pool, pnext,
 				   &ioil_iog.iog_pools_head, iop_pools) {
 		d_list_for_each_entry_safe(cont, cnext,
@@ -292,6 +318,12 @@ ioil_fini(void)
 		}
 	}
 
+	/* Tidy up any pools which do not have open containers */
+	d_list_for_each_entry_safe(pool, pnext,
+				   &ioil_iog.iog_pools_head, iop_pools) {
+		ioil_shrink_pool(pool);
+	}
+
 	if (ioil_iog.iog_daos_init)
 		daos_fini();
 	ioil_iog.iog_daos_init = false;

src/common/fail_loc.c

@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2016-2020 Intel Corporation.
+ * (C) Copyright 2016-2021 Intel Corporation.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -181,9 +181,16 @@ daos_fail_init(void)
 	if (rc != 0 && rc != -DER_NOSYS)
 		return rc;
 
+	/* Log, but no not propagate error on registering the fault as this
+	 * leads to deadlocks.
+	 */
 	rc = d_fault_attr_set(DAOS_FAIL_UNIT_TEST_GROUP, attr);
-	if (rc)
+	if (rc != 0) {
+		D_ERROR("Failed to set fault attr, "DF_RC"\n",
+			DP_RC(rc));
 		d_fault_inject_fini();
+	}
+	rc = 0;
 
 	return rc;
 }