lib: sbi: Optimize sbi_tlb and sbi_ipi

This patch comes from:
[PATCH v4 0/4] Miscellaneous about sbi_tlb and sbi_ipi
http://lists.infradead.org/pipermail/opensbi/2023-February/004402.html

include/sbi/sbi_ipi.h

@@ -41,6 +41,10 @@ struct sbi_ipi_event_ops {
 	 * Update callback to save/enqueue data for remote HART
 	 * Note: This is an optional callback and it is called just before
 	 * triggering IPI to remote HART.
+	 * @return 0  success
+	 * @return -1 break IPI, done on local hart
+	 * @return -2 need retry
+
 	 */
 	int (* update)(struct sbi_scratch *scratch,
 			struct sbi_scratch *remote_scratch,

lib/sbi/sbi_ipi.c

@@ -15,6 +15,7 @@
 #include <sbi/sbi_domain.h>
 #include <sbi/sbi_error.h>
 #include <sbi/sbi_hart.h>
+#include <sbi/sbi_hartmask.h>
 #include <sbi/sbi_hsm.h>
 #include <sbi/sbi_init.h>
 #include <sbi/sbi_ipi.h>
@@ -31,7 +32,7 @@ static unsigned long ipi_data_off;
 static const struct sbi_ipi_device *ipi_dev = NULL;
 static const struct sbi_ipi_event_ops *ipi_ops_array[SBI_IPI_EVENT_MAX];
 
-static int sbi_ipi_send(struct sbi_scratch *scratch, u32 remote_hartid,
+static int sbi_ipi_update(struct sbi_scratch *scratch, u32 remote_hartid,
 			u32 event, void *data)
 {
 	int ret;
@@ -69,6 +70,18 @@ static int sbi_ipi_send(struct sbi_scratch *scratch, u32 remote_hartid,
 
 	sbi_pmu_ctr_incr_fw(SBI_PMU_FW_IPI_SENT);
 
+	return 0;
+}
+
+static int sbi_ipi_sync(struct sbi_scratch *scratch, u32 event)
+{
+	const struct sbi_ipi_event_ops *ipi_ops;
+
+	if ((SBI_IPI_EVENT_MAX <= event) ||
+			!ipi_ops_array[event])
+		return SBI_EINVAL;
+	ipi_ops = ipi_ops_array[event];
+
 	if (ipi_ops->sync)
 		ipi_ops->sync(scratch);
 
@@ -82,8 +95,10 @@ static int sbi_ipi_send(struct sbi_scratch *scratch, u32 remote_hartid,
  */
 int sbi_ipi_send_many(ulong hmask, ulong hbase, u32 event, void *data)
 {
-	int rc;
+	int rc, done;
 	ulong i, m;
+	struct sbi_hartmask target_mask = {0};
+	struct sbi_hartmask retry_mask = {0};
 	struct sbi_domain *dom = sbi_domain_thishart_ptr();
 	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
 
@@ -93,23 +108,36 @@ int sbi_ipi_send_many(ulong hmask, ulong hbase, u32 event, void *data)
 			return rc;
 		m &= hmask;
 
-		/* Send IPIs */
 		for (i = hbase; m; i++, m >>= 1) {
 			if (m & 1UL)
-				sbi_ipi_send(scratch, i, event, data);
+				sbi_hartmask_set_hart(i, &target_mask);
 		}
 	} else {
 		hbase = 0;
 		while (!sbi_hsm_hart_interruptible_mask(dom, hbase, &m)) {
-			/* Send IPIs */
 			for (i = hbase; m; i++, m >>= 1) {
 				if (m & 1UL)
-					sbi_ipi_send(scratch, i, event, data);
+					sbi_hartmask_set_hart(i, &target_mask);
 			}
 			hbase += BITS_PER_LONG;
 		}
 	}
 
+	retry_mask = target_mask;
+	do {
+		done = true;
+		sbi_hartmask_for_each_hart(i, &retry_mask) {
+			rc = sbi_ipi_update(scratch, i, event, data);
+			if (rc == -2)
+				done = false;
+			else
+				sbi_hartmask_clear_hart(i, &retry_mask);
+		}
+	} while (!done);
+
+	/* sync IPIs */
+	sbi_ipi_sync(scratch, event);
+
 	return 0;
 }
 

lib/sbi/sbi_tlb.c

@@ -211,11 +211,11 @@ static void tlb_pmu_incr_fw_ctr(struct sbi_tlb_info *data)
 		sbi_pmu_ctr_incr_fw(SBI_PMU_FW_HFENCE_VVMA_ASID_SENT);
 }
 
-static void tlb_entry_process(struct sbi_tlb_info *tinfo)
+static void tlb_process_helper(struct sbi_tlb_info *tinfo)
 {
 	u32 rhartid;
 	struct sbi_scratch *rscratch = NULL;
-	unsigned long *rtlb_sync = NULL;
+	atomic_t *rtlb_sync = NULL;
 
 	tinfo->local_fn(tinfo);
 
@@ -225,47 +225,40 @@ static void tlb_entry_process(struct sbi_tlb_info *tinfo)
 			continue;
 
 		rtlb_sync = sbi_scratch_offset_ptr(rscratch, tlb_sync_off);
-		while (atomic_raw_xchg_ulong(rtlb_sync, 1)) ;
+		atomic_sub_return(rtlb_sync, 1);
 	}
 }
 
-static void tlb_process_count(struct sbi_scratch *scratch, int count)
+static int tlb_process_once(struct sbi_scratch *scratch)
 {
 	struct sbi_tlb_info tinfo;
-	unsigned int deq_count = 0;
 	struct sbi_fifo *tlb_fifo =
 			sbi_scratch_offset_ptr(scratch, tlb_fifo_off);
 
-	while (!sbi_fifo_dequeue(tlb_fifo, &tinfo)) {
+	if (!sbi_fifo_dequeue(tlb_fifo, &tinfo)) {
-		tlb_entry_process(&tinfo);
+		tlb_process_helper(&tinfo);
-		deq_count++;
+		return 0;
-		if (deq_count > count)
-			break;
-
 	}
+
+	return -1;
 }
 
 static void tlb_process(struct sbi_scratch *scratch)
 {
-	struct sbi_tlb_info tinfo;
+	while (!tlb_process_once(scratch));
-	struct sbi_fifo *tlb_fifo =
-			sbi_scratch_offset_ptr(scratch, tlb_fifo_off);
-
-	while (!sbi_fifo_dequeue(tlb_fifo, &tinfo))
-		tlb_entry_process(&tinfo);
 }
 
 static void tlb_sync(struct sbi_scratch *scratch)
 {
-	unsigned long *tlb_sync =
+	atomic_t *tlb_sync =
-			sbi_scratch_offset_ptr(scratch, tlb_sync_off);
+		sbi_scratch_offset_ptr(scratch, tlb_sync_off);
 
-	while (!atomic_raw_xchg_ulong(tlb_sync, 0)) {
+	while (atomic_read(tlb_sync) > 0) {
 		/*
 		 * While we are waiting for remote hart to set the sync,
 		 * consume fifo requests to avoid deadlock.
 		 */
-		tlb_process_count(scratch, 1);
+		tlb_process_once(scratch);
 	}
 
 	return;
@@ -343,6 +336,7 @@ static int tlb_update(struct sbi_scratch *scratch,
 			  u32 remote_hartid, void *data)
 {
 	int ret;
+	atomic_t *tlb_sync;
 	struct sbi_fifo *tlb_fifo_r;
 	struct sbi_tlb_info *tinfo = data;
 	u32 curr_hartid = current_hartid();
@@ -369,11 +363,7 @@ static int tlb_update(struct sbi_scratch *scratch,
 	tlb_fifo_r = sbi_scratch_offset_ptr(remote_scratch, tlb_fifo_off);
 
 	ret = sbi_fifo_inplace_update(tlb_fifo_r, data, tlb_update_cb);
-	if (ret != SBI_FIFO_UNCHANGED) {
+	if (ret == SBI_FIFO_UNCHANGED && sbi_fifo_enqueue(tlb_fifo_r, data) < 0) {
-		return 1;
-	}
-
-	while (sbi_fifo_enqueue(tlb_fifo_r, data) < 0) {
 		/**
 		 * For now, Busy loop until there is space in the fifo.
 		 * There may be case where target hart is also
@@ -382,11 +372,15 @@ static int tlb_update(struct sbi_scratch *scratch,
 		 * TODO: Introduce a wait/wakeup event mechanism to handle
 		 * this properly.
 		 */
-		tlb_process_count(scratch, 1);
+		tlb_process_once(scratch);
 		sbi_dprintf("hart%d: hart%d tlb fifo full\n",
 			    curr_hartid, remote_hartid);
+		return -2;
 	}
 
+	tlb_sync = sbi_scratch_offset_ptr(scratch, tlb_sync_off);
+	atomic_add_return(tlb_sync, 1);
+
 	return 0;
 }
 
@@ -413,7 +407,7 @@ int sbi_tlb_init(struct sbi_scratch *scratch, bool cold_boot)
 {
 	int ret;
 	void *tlb_mem;
-	unsigned long *tlb_sync;
+	atomic_t *tlb_sync;
 	struct sbi_fifo *tlb_q;
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 
@@ -455,7 +449,7 @@ int sbi_tlb_init(struct sbi_scratch *scratch, bool cold_boot)
 	tlb_q = sbi_scratch_offset_ptr(scratch, tlb_fifo_off);
 	tlb_mem = sbi_scratch_offset_ptr(scratch, tlb_fifo_mem_off);
 
-	*tlb_sync = 0;
+	tlb_sync->counter = 0;
 
 	sbi_fifo_init(tlb_q, tlb_mem,
 		      SBI_TLB_FIFO_NUM_ENTRIES, SBI_TLB_INFO_SIZE);