eventdev: introduce event pre-scheduling

Event pre-scheduling improves scheduling performance by assigning events
to event ports in advance when dequeues are issued.
The dequeue operation initiates the pre-schedule operation, which completes
in parallel without affecting the dequeued event flow contexts and
dequeue latency.

Event devices can indicate pre-scheduling capabilities using
`RTE_EVENT_DEV_CAP_EVENT_PRESCHEDULE` and
`RTE_EVENT_DEV_CAP_EVENT_PRESCHEDULE_ADAPTIVE` via the event device info
function `info.event_dev_cap`.

Applications can select the pre-schedule type and configure it through
`rte_event_dev_config.preschedule_type` during `rte_event_dev_configure`.

The supported pre-schedule types are:
 * `RTE_EVENT_PRESCHEDULE_NONE` - No pre-scheduling.
 * `RTE_EVENT_PRESCHEDULE` - Always issue a pre-schedule on dequeue.
 * `RTE_EVENT_PRESCHEDULE_ADAPTIVE` - Delay issuing pre-schedule until
   there are no forward progress constraints with the held flow contexts.

Signed-off-by: Pavan Nikhilesh <[email protected]>
Acked-by: Jerin Jacob <[email protected]>

app/test/test_eventdev.c

@@ -1251,6 +1251,112 @@ test_eventdev_profile_switch(void)
 	return TEST_SUCCESS;
 }
 
+static int
+preschedule_test(enum rte_event_dev_preschedule_type preschedule_type, const char *preschedule_name)
+{
+#define NB_EVENTS     1024
+	uint64_t start, total;
+	struct rte_event ev;
+	int rc, cnt;
+
+	ev.event_type = RTE_EVENT_TYPE_CPU;
+	ev.queue_id = 0;
+	ev.op = RTE_EVENT_OP_NEW;
+	ev.u64 = 0xBADF00D0;
+
+	for (cnt = 0; cnt < NB_EVENTS; cnt++) {
+		ev.flow_id = cnt;
+		rc = rte_event_enqueue_burst(TEST_DEV_ID, 0, &ev, 1);
+		TEST_ASSERT(rc == 1, "Failed to enqueue event");
+	}
+
+	RTE_SET_USED(preschedule_type);
+	total = 0;
+	while (cnt) {
+		start = rte_rdtsc_precise();
+		rc = rte_event_dequeue_burst(TEST_DEV_ID, 0, &ev, 1, 0);
+		if (rc) {
+			total += rte_rdtsc_precise() - start;
+			cnt--;
+		}
+	}
+	printf("Preschedule type : %s, avg cycles %" PRIu64 "\n", preschedule_name,
+	       total / NB_EVENTS);
+
+	return TEST_SUCCESS;
+}
+
+static int
+preschedule_configure(enum rte_event_dev_preschedule_type type, struct rte_event_dev_info *info)
+{
+	struct rte_event_dev_config dev_conf;
+	struct rte_event_queue_conf qcfg;
+	struct rte_event_port_conf pcfg;
+	int rc;
+
+	devconf_set_default_sane_values(&dev_conf, info);
+	dev_conf.nb_event_ports = 1;
+	dev_conf.nb_event_queues = 1;
+	dev_conf.preschedule_type = type;
+
+	rc = rte_event_dev_configure(TEST_DEV_ID, &dev_conf);
+	TEST_ASSERT_SUCCESS(rc, "Failed to configure eventdev");
+
+	rc = rte_event_port_default_conf_get(TEST_DEV_ID, 0, &pcfg);
+	TEST_ASSERT_SUCCESS(rc, "Failed to get port0 default config");
+	rc = rte_event_port_setup(TEST_DEV_ID, 0, &pcfg);
+	TEST_ASSERT_SUCCESS(rc, "Failed to setup port0");
+
+	rc = rte_event_queue_default_conf_get(TEST_DEV_ID, 0, &qcfg);
+	TEST_ASSERT_SUCCESS(rc, "Failed to get queue0 default config");
+	rc = rte_event_queue_setup(TEST_DEV_ID, 0, &qcfg);
+	TEST_ASSERT_SUCCESS(rc, "Failed to setup queue0");
+
+	rc = rte_event_port_link(TEST_DEV_ID, 0, NULL, NULL, 0);
+	TEST_ASSERT(rc == (int)dev_conf.nb_event_queues, "Failed to link port, device %d",
+		    TEST_DEV_ID);
+
+	rc = rte_event_dev_start(TEST_DEV_ID);
+	TEST_ASSERT_SUCCESS(rc, "Failed to start event device");
+
+	return 0;
+}
+
+static int
+test_eventdev_preschedule_configure(void)
+{
+	struct rte_event_dev_info info;
+	int rc;
+
+	rte_event_dev_info_get(TEST_DEV_ID, &info);
+
+	if ((info.event_dev_cap & RTE_EVENT_DEV_CAP_EVENT_PRESCHEDULE) == 0)
+		return TEST_SKIPPED;
+
+	rc = preschedule_configure(RTE_EVENT_PRESCHEDULE_NONE, &info);
+	TEST_ASSERT_SUCCESS(rc, "Failed to configure eventdev");
+	rc = preschedule_test(RTE_EVENT_PRESCHEDULE_NONE, "RTE_EVENT_PRESCHEDULE_NONE");
+	TEST_ASSERT_SUCCESS(rc, "Failed to test preschedule RTE_EVENT_PRESCHEDULE_NONE");
+
+	rte_event_dev_stop(TEST_DEV_ID);
+	rc = preschedule_configure(RTE_EVENT_PRESCHEDULE, &info);
+	TEST_ASSERT_SUCCESS(rc, "Failed to configure eventdev");
+	rc = preschedule_test(RTE_EVENT_PRESCHEDULE, "RTE_EVENT_PRESCHEDULE");
+	TEST_ASSERT_SUCCESS(rc, "Failed to test preschedule RTE_EVENT_PRESCHEDULE");
+
+	if (info.event_dev_cap & RTE_EVENT_DEV_CAP_EVENT_PRESCHEDULE_ADAPTIVE) {
+		rte_event_dev_stop(TEST_DEV_ID);
+		rc = preschedule_configure(RTE_EVENT_PRESCHEDULE_ADAPTIVE, &info);
+		TEST_ASSERT_SUCCESS(rc, "Failed to configure eventdev");
+		rc = preschedule_test(RTE_EVENT_PRESCHEDULE_ADAPTIVE,
+				      "RTE_EVENT_PRESCHEDULE_ADAPTIVE");
+		TEST_ASSERT_SUCCESS(rc,
+				    "Failed to test preschedule RTE_EVENT_PRESCHEDULE_ADAPTIVE");
+	}
+
+	return TEST_SUCCESS;
+}
+
 static int
 test_eventdev_close(void)
 {
@@ -1311,6 +1417,8 @@ static struct unit_test_suite eventdev_common_testsuite  = {
 			test_eventdev_start_stop),
 		TEST_CASE_ST(eventdev_configure_setup, eventdev_stop_device,
 			test_eventdev_profile_switch),
+		TEST_CASE_ST(eventdev_configure_setup, NULL,
+			test_eventdev_preschedule_configure),
 		TEST_CASE_ST(eventdev_setup_device, eventdev_stop_device,
 			test_eventdev_link),
 		TEST_CASE_ST(eventdev_setup_device, eventdev_stop_device,

doc/guides/eventdevs/features/default.ini

@@ -23,6 +23,7 @@ maintenance_free           =
 runtime_queue_attr         =
 profile_links              =
 independent_enq            =
+preschedule                =
 
 ;
 ; Features of a default Ethernet Rx adapter.

doc/guides/prog_guide/eventdev/eventdev.rst

@@ -357,6 +357,31 @@ Worker path:
        // Process the event received.
    }
 
+Event Pre-scheduling
+~~~~~~~~~~~~~~~~~~~~
+
+Event pre-scheduling improves scheduling performance
+by assigning events to event ports in advance when dequeues are issued.
+The ``rte_event_dequeue_burst`` operation initiates the pre-schedule operation,
+which completes in parallel
+without affecting the dequeued event flow contexts and dequeue latency.
+On the next dequeue operation, the pre-scheduled events are dequeued
+and pre-schedule is initiated again.
+
+An application can use event pre-scheduling if the event device supports it
+at either device level or at a individual port level.
+The application must check pre-schedule capability
+by checking if ``rte_event_dev_info.event_dev_cap`` has the bit
+``RTE_EVENT_DEV_CAP_PRESCHEDULE`` or ``RTE_EVENT_DEV_CAP_PRESCHEDULE_ADAPTIVE`` set,
+if present pre-scheduling can be enabled at device configuration time
+by setting appropriate pre-schedule type in ``rte_event_dev_config.preschedule``.
+
+The following pre-schedule types are supported:
+ * ``RTE_EVENT_PRESCHEDULE_NONE`` - No pre-scheduling.
+ * ``RTE_EVENT_PRESCHEDULE`` - Always issue a pre-schedule when dequeue is issued.
+ * ``RTE_EVENT_PRESCHEDULE_ADAPTIVE`` - Issue pre-schedule when dequeue is issued
+   and there are no forward progress constraints.
+
 Starting the EventDev
 ~~~~~~~~~~~~~~~~~~~~~
 

doc/guides/rel_notes/release_24_11.rst

@@ -82,6 +82,14 @@ New Features
 
   The new statistics are useful for debugging and profiling.
 
+* **Added event device pre-scheduling support.**
+
+  Added support for pre-scheduling of events to event ports
+  to improve scheduling performance and latency.
+
+  * Added ``rte_event_dev_config::preschedule_type``
+    to configure the device level pre-scheduling type.
+
 * **Updated event device library for independent enqueue feature.**
 
   Added support for independent enqueue feature.
@@ -148,6 +156,8 @@ ABI Changes
   has been increased from 8 to 253 (which is the maximum possible with Unix domain socket).
   This allows for more queues when using software devices such as TAP and XDP.
 
+* eventdev: Added ``preschedule_type`` field to ``rte_event_dev_config`` structure.
+
 
 Known Issues
 ------------

lib/eventdev/rte_eventdev.h

@@ -461,6 +461,30 @@ struct rte_event;
  * only applies to ports that have enabled independent enqueue feature.
  */
 
+#define RTE_EVENT_DEV_CAP_EVENT_PRESCHEDULE (1ULL << 17)
+/**< Event device supports event pre-scheduling.
+ *
+ * When this capability is available, the application can enable event pre-scheduling on the event
+ * device to pre-schedule events to a event port when `rte_event_dequeue_burst()`
+ * is issued.
+ * The pre-schedule process starts with the `rte_event_dequeue_burst()` call and the
+ * pre-scheduled events are returned on the next `rte_event_dequeue_burst()` call.
+ *
+ * @see rte_event_dev_configure()
+ */
+
+#define RTE_EVENT_DEV_CAP_EVENT_PRESCHEDULE_ADAPTIVE (1ULL << 18)
+/**< Event device supports adaptive event pre-scheduling.
+ *
+ * When this capability is available, the application can enable adaptive pre-scheduling
+ * on the event device where the events are pre-scheduled when there are no forward
+ * progress constraints with the currently held flow contexts.
+ * The pre-schedule process starts with the `rte_event_dequeue_burst()` call and the
+ * pre-scheduled events are returned on the next `rte_event_dequeue_burst()` call.
+ *
+ * @see rte_event_dev_configure()
+ */
+
 /* Event device priority levels */
 #define RTE_EVENT_DEV_PRIORITY_HIGHEST   0
 /**< Highest priority level for events and queues.
@@ -695,6 +719,26 @@ rte_event_dev_attr_get(uint8_t dev_id, uint32_t attr_id,
  *  @see rte_event_dequeue_timeout_ticks(), rte_event_dequeue_burst()
  */
 
+/** Event device pre-schedule type enumeration. */
+enum rte_event_dev_preschedule_type {
+	RTE_EVENT_PRESCHEDULE_NONE,
+	/**< Disable pre-schedule across the event device or on a given event port.
+	 * @ref rte_event_dev_config.preschedule_type
+	 */
+	RTE_EVENT_PRESCHEDULE,
+	/**< Enable pre-schedule always across the event device or a given event port.
+	 * @ref rte_event_dev_config.preschedule_type
+	 * @see RTE_EVENT_DEV_CAP_EVENT_PRESCHEDULE
+	 */
+	RTE_EVENT_PRESCHEDULE_ADAPTIVE,
+	/**< Enable adaptive pre-schedule across the event device or a given event port.
+	 * Delay issuing pre-schedule until there are no forward progress constraints with
+	 * the held flow contexts.
+	 * @ref rte_event_dev_config.preschedule_type
+	 * @see RTE_EVENT_DEV_CAP_EVENT_PRESCHEDULE_ADAPTIVE
+	 */
+};
+
 /** Event device configuration structure */
 struct rte_event_dev_config {
 	uint32_t dequeue_timeout_ns;
@@ -767,6 +811,11 @@ struct rte_event_dev_config {
 	 * optimized for single-link usage, this field is a hint for how many
 	 * to allocate; otherwise, regular event ports and queues will be used.
 	 */
+	enum rte_event_dev_preschedule_type preschedule_type;
+	/**< Event pre-schedule type to use across the event device, if supported.
+	 * @see RTE_EVENT_DEV_CAP_EVENT_PRESCHEDULE
+	 * @see RTE_EVENT_DEV_CAP_EVENT_PRESCHEDULE_ADAPTIVE
+	 */
 };
 
 /**