diff --git a/include/sbi/sbi_ipi.h b/include/sbi/sbi_ipi.h index f384e748..f9a45c0c 100644 --- a/include/sbi/sbi_ipi.h +++ b/include/sbi/sbi_ipi.h @@ -41,6 +41,10 @@ struct sbi_ipi_event_ops { * Update callback to save/enqueue data for remote HART * Note: This is an optional callback and it is called just before * triggering IPI to remote HART. + * @return 0 success + * @return -1 break IPI, done on local hart + * @return -2 need retry + */ int (* update)(struct sbi_scratch *scratch, struct sbi_scratch *remote_scratch, diff --git a/lib/sbi/sbi_ipi.c b/lib/sbi/sbi_ipi.c index b9f62056..0530f06e 100644 --- a/lib/sbi/sbi_ipi.c +++ b/lib/sbi/sbi_ipi.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -31,7 +32,7 @@ static unsigned long ipi_data_off; static const struct sbi_ipi_device *ipi_dev = NULL; static const struct sbi_ipi_event_ops *ipi_ops_array[SBI_IPI_EVENT_MAX]; -static int sbi_ipi_send(struct sbi_scratch *scratch, u32 remote_hartid, +static int sbi_ipi_update(struct sbi_scratch *scratch, u32 remote_hartid, u32 event, void *data) { int ret; @@ -69,6 +70,18 @@ static int sbi_ipi_send(struct sbi_scratch *scratch, u32 remote_hartid, sbi_pmu_ctr_incr_fw(SBI_PMU_FW_IPI_SENT); + return 0; +} + +static int sbi_ipi_sync(struct sbi_scratch *scratch, u32 event) +{ + const struct sbi_ipi_event_ops *ipi_ops; + + if ((SBI_IPI_EVENT_MAX <= event) || + !ipi_ops_array[event]) + return SBI_EINVAL; + ipi_ops = ipi_ops_array[event]; + if (ipi_ops->sync) ipi_ops->sync(scratch); @@ -82,8 +95,10 @@ static int sbi_ipi_send(struct sbi_scratch *scratch, u32 remote_hartid, */ int sbi_ipi_send_many(ulong hmask, ulong hbase, u32 event, void *data) { - int rc; + int rc, done; ulong i, m; + struct sbi_hartmask target_mask = {0}; + struct sbi_hartmask retry_mask = {0}; struct sbi_domain *dom = sbi_domain_thishart_ptr(); struct sbi_scratch *scratch = sbi_scratch_thishart_ptr(); @@ -93,23 +108,36 @@ int sbi_ipi_send_many(ulong hmask, ulong hbase, u32 event, void *data) return rc; m &= hmask; - /* Send IPIs */ for (i = hbase; m; i++, m >>= 1) { if (m & 1UL) - sbi_ipi_send(scratch, i, event, data); + sbi_hartmask_set_hart(i, &target_mask); } } else { hbase = 0; while (!sbi_hsm_hart_interruptible_mask(dom, hbase, &m)) { - /* Send IPIs */ for (i = hbase; m; i++, m >>= 1) { if (m & 1UL) - sbi_ipi_send(scratch, i, event, data); + sbi_hartmask_set_hart(i, &target_mask); } hbase += BITS_PER_LONG; } } + retry_mask = target_mask; + do { + done = true; + sbi_hartmask_for_each_hart(i, &retry_mask) { + rc = sbi_ipi_update(scratch, i, event, data); + if (rc == -2) + done = false; + else + sbi_hartmask_clear_hart(i, &retry_mask); + } + } while (!done); + + /* sync IPIs */ + sbi_ipi_sync(scratch, event); + return 0; } diff --git a/lib/sbi/sbi_tlb.c b/lib/sbi/sbi_tlb.c index 4c142ea3..f3bac823 100644 --- a/lib/sbi/sbi_tlb.c +++ b/lib/sbi/sbi_tlb.c @@ -211,11 +211,11 @@ static void tlb_pmu_incr_fw_ctr(struct sbi_tlb_info *data) sbi_pmu_ctr_incr_fw(SBI_PMU_FW_HFENCE_VVMA_ASID_SENT); } -static void tlb_entry_process(struct sbi_tlb_info *tinfo) +static void tlb_process_helper(struct sbi_tlb_info *tinfo) { u32 rhartid; struct sbi_scratch *rscratch = NULL; - unsigned long *rtlb_sync = NULL; + atomic_t *rtlb_sync = NULL; tinfo->local_fn(tinfo); @@ -225,47 +225,40 @@ static void tlb_entry_process(struct sbi_tlb_info *tinfo) continue; rtlb_sync = sbi_scratch_offset_ptr(rscratch, tlb_sync_off); - while (atomic_raw_xchg_ulong(rtlb_sync, 1)) ; + atomic_sub_return(rtlb_sync, 1); } } -static void tlb_process_count(struct sbi_scratch *scratch, int count) +static int tlb_process_once(struct sbi_scratch *scratch) { struct sbi_tlb_info tinfo; - unsigned int deq_count = 0; struct sbi_fifo *tlb_fifo = sbi_scratch_offset_ptr(scratch, tlb_fifo_off); - while (!sbi_fifo_dequeue(tlb_fifo, &tinfo)) { - tlb_entry_process(&tinfo); - deq_count++; - if (deq_count > count) - break; - + if (!sbi_fifo_dequeue(tlb_fifo, &tinfo)) { + tlb_process_helper(&tinfo); + return 0; } + + return -1; } static void tlb_process(struct sbi_scratch *scratch) { - struct sbi_tlb_info tinfo; - struct sbi_fifo *tlb_fifo = - sbi_scratch_offset_ptr(scratch, tlb_fifo_off); - - while (!sbi_fifo_dequeue(tlb_fifo, &tinfo)) - tlb_entry_process(&tinfo); + while (!tlb_process_once(scratch)); } static void tlb_sync(struct sbi_scratch *scratch) { - unsigned long *tlb_sync = - sbi_scratch_offset_ptr(scratch, tlb_sync_off); + atomic_t *tlb_sync = + sbi_scratch_offset_ptr(scratch, tlb_sync_off); - while (!atomic_raw_xchg_ulong(tlb_sync, 0)) { + while (atomic_read(tlb_sync) > 0) { /* * While we are waiting for remote hart to set the sync, * consume fifo requests to avoid deadlock. */ - tlb_process_count(scratch, 1); + tlb_process_once(scratch); } return; @@ -343,6 +336,7 @@ static int tlb_update(struct sbi_scratch *scratch, u32 remote_hartid, void *data) { int ret; + atomic_t *tlb_sync; struct sbi_fifo *tlb_fifo_r; struct sbi_tlb_info *tinfo = data; u32 curr_hartid = current_hartid(); @@ -369,11 +363,7 @@ static int tlb_update(struct sbi_scratch *scratch, tlb_fifo_r = sbi_scratch_offset_ptr(remote_scratch, tlb_fifo_off); ret = sbi_fifo_inplace_update(tlb_fifo_r, data, tlb_update_cb); - if (ret != SBI_FIFO_UNCHANGED) { - return 1; - } - - while (sbi_fifo_enqueue(tlb_fifo_r, data) < 0) { + if (ret == SBI_FIFO_UNCHANGED && sbi_fifo_enqueue(tlb_fifo_r, data) < 0) { /** * For now, Busy loop until there is space in the fifo. * There may be case where target hart is also @@ -382,11 +372,15 @@ static int tlb_update(struct sbi_scratch *scratch, * TODO: Introduce a wait/wakeup event mechanism to handle * this properly. */ - tlb_process_count(scratch, 1); + tlb_process_once(scratch); sbi_dprintf("hart%d: hart%d tlb fifo full\n", curr_hartid, remote_hartid); + return -2; } + tlb_sync = sbi_scratch_offset_ptr(scratch, tlb_sync_off); + atomic_add_return(tlb_sync, 1); + return 0; } @@ -413,7 +407,7 @@ int sbi_tlb_init(struct sbi_scratch *scratch, bool cold_boot) { int ret; void *tlb_mem; - unsigned long *tlb_sync; + atomic_t *tlb_sync; struct sbi_fifo *tlb_q; const struct sbi_platform *plat = sbi_platform_ptr(scratch); @@ -455,7 +449,7 @@ int sbi_tlb_init(struct sbi_scratch *scratch, bool cold_boot) tlb_q = sbi_scratch_offset_ptr(scratch, tlb_fifo_off); tlb_mem = sbi_scratch_offset_ptr(scratch, tlb_fifo_mem_off); - *tlb_sync = 0; + tlb_sync->counter = 0; sbi_fifo_init(tlb_q, tlb_mem, SBI_TLB_FIFO_NUM_ENTRIES, SBI_TLB_INFO_SIZE);