diff --git a/README.md b/README.md
index feb0e71..dbf497f 100755
--- a/README.md
+++ b/README.md
@@ -1,3 +1,32 @@
+# ezrknn-toolkit2
+This repo tries to make RKNN LLM usage easier
+
+## Requirements
+Keep in mind this repo is focused for:
+- High-end Rockchip SoCs, mainly the RK3588
+- Linux, not Android
+- Linux kernels from Rockchip (as of writing 5.10 and 6.1 from Rockchip should work, if your board has one of these it will very likely be Rockchip's kernel)
+
+## Quick Install
+Run:
+
+```bash
+# TODO
+```
+
+## Test
+Run (cd is required):
+
+```bash
+# TODO
+```
+
+# Original README starts below
+
+
+
+
+
# Description
RKLLM software stack can help users to quickly deploy AI models to Rockchip chips. The overall framework is as follows:
@@ -108,4 +137,4 @@ https://github.com/airockchip/rknn-toolkit2
- Add support for models such as Llama3, Gemma2, and MiniCPM3.
- Resolve catastrophic forgetting issue when the number of tokens exceeds max_context.
-for older version, please refer [CHANGELOG](CHANGELOG.md)
\ No newline at end of file
+for older version, please refer [CHANGELOG](CHANGELOG.md)
diff --git a/rknpu-driver/driver-0.9.6/Kconfig b/rknpu-driver/driver-0.9.6/Kconfig
deleted file mode 100644
index c3343ee..0000000
--- a/rknpu-driver/driver-0.9.6/Kconfig
+++ /dev/null
@@ -1,60 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-menu "RKNPU"
- depends on ARCH_ROCKCHIP
-
-config ROCKCHIP_RKNPU
- tristate "ROCKCHIP_RKNPU"
- depends on DRM || DMABUF_HEAPS_ROCKCHIP_CMA_HEAP
- help
- rknpu module.
-
-if ROCKCHIP_RKNPU
-
-config ROCKCHIP_RKNPU_DEBUG_FS
- bool "RKNPU debugfs"
- depends on DEBUG_FS
- default y
- help
- Enable debugfs to debug RKNPU usage.
-
-config ROCKCHIP_RKNPU_PROC_FS
- bool "RKNPU procfs"
- depends on PROC_FS
- help
- Enable procfs to debug RKNPU usage.
-
-config ROCKCHIP_RKNPU_FENCE
- bool "RKNPU fence"
- depends on SYNC_FILE
- help
- Enable fence support for RKNPU.
-
-config ROCKCHIP_RKNPU_SRAM
- bool "RKNPU SRAM"
- depends on NO_GKI
- help
- Enable RKNPU SRAM support
-
-choice
- prompt "RKNPU memory manager"
- default ROCKCHIP_RKNPU_DRM_GEM
- help
- Select RKNPU memory manager
-
-config ROCKCHIP_RKNPU_DRM_GEM
- bool "RKNPU DRM GEM"
- depends on DRM
- help
- Enable RKNPU memory manager by DRM GEM.
-
-config ROCKCHIP_RKNPU_DMA_HEAP
- bool "RKNPU DMA heap"
- depends on DMABUF_HEAPS_ROCKCHIP_CMA_HEAP
- help
- Enable RKNPU memory manager by DMA Heap.
-
-endchoice
-
-endif
-
-endmenu
diff --git a/rknpu-driver/driver-0.9.6/Makefile b/rknpu-driver/driver-0.9.6/Makefile
deleted file mode 100644
index d73563d..0000000
--- a/rknpu-driver/driver-0.9.6/Makefile
+++ /dev/null
@@ -1,17 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_ROCKCHIP_RKNPU) += rknpu.o
-
-ccflags-y += -I$(srctree)/$(src)/include
-ccflags-y += -I$(src)/include
-ccflags-y += -Werror
-
-rknpu-y += rknpu_drv.o
-rknpu-y += rknpu_reset.o
-rknpu-y += rknpu_job.o
-rknpu-y += rknpu_debugger.o
-rknpu-y += rknpu_iommu.o
-rknpu-$(CONFIG_PM_DEVFREQ) += rknpu_devfreq.o
-rknpu-$(CONFIG_ROCKCHIP_RKNPU_SRAM) += rknpu_mm.o
-rknpu-$(CONFIG_ROCKCHIP_RKNPU_FENCE) += rknpu_fence.o
-rknpu-$(CONFIG_ROCKCHIP_RKNPU_DRM_GEM) += rknpu_gem.o
-rknpu-$(CONFIG_ROCKCHIP_RKNPU_DMA_HEAP) += rknpu_mem.o
diff --git a/rknpu-driver/driver-0.9.6/include/rknpu_debugger.h b/rknpu-driver/driver-0.9.6/include/rknpu_debugger.h
deleted file mode 100644
index 3f4420d..0000000
--- a/rknpu-driver/driver-0.9.6/include/rknpu_debugger.h
+++ /dev/null
@@ -1,88 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (C) Rockchip Electronics Co.Ltd
- * Author: Felix Zeng
- */
-
-#ifndef __LINUX_RKNPU_DEBUGGER_H_
-#define __LINUX_RKNPU_DEBUGGER_H_
-
-#include
-
-/*
- * struct rknpu_debugger - rknpu debugger information
- *
- * This structure represents a debugger to be created by the rknpu driver
- * or core.
- */
-struct rknpu_debugger {
-#ifdef CONFIG_ROCKCHIP_RKNPU_DEBUG_FS
- /* Directory of debugfs file */
- struct dentry *debugfs_dir;
- struct list_head debugfs_entry_list;
- struct mutex debugfs_lock;
-#endif
-#ifdef CONFIG_ROCKCHIP_RKNPU_PROC_FS
- /* Directory of procfs file */
- struct proc_dir_entry *procfs_dir;
- struct list_head procfs_entry_list;
- struct mutex procfs_lock;
-#endif
-};
-
-/*
- * struct rknpu_debugger_list - debugfs/procfs info list entry
- *
- * This structure represents a debugfs/procfs file to be created by the npu
- * driver or core.
- */
-struct rknpu_debugger_list {
- /* File name */
- const char *name;
- /*
- * Show callback. &seq_file->private will be set to the &struct
- * rknpu_debugger_node corresponding to the instance of this info
- * on a given &struct rknpu_debugger.
- */
- int (*show)(struct seq_file *seq, void *data);
- /*
- * Write callback. &seq_file->private will be set to the &struct
- * rknpu_debugger_node corresponding to the instance of this info
- * on a given &struct rknpu_debugger.
- */
- ssize_t (*write)(struct file *file, const char __user *ubuf, size_t len,
- loff_t *offp);
- /* Procfs/Debugfs private data. */
- void *data;
-};
-
-/*
- * struct rknpu_debugger_node - Nodes for debugfs/procfs
- *
- * This structure represents each instance of procfs/debugfs created from the
- * template.
- */
-struct rknpu_debugger_node {
- struct rknpu_debugger *debugger;
-
- /* template for this node. */
- const struct rknpu_debugger_list *info_ent;
-
- /* Each Procfs/Debugfs file. */
-#ifdef CONFIG_ROCKCHIP_RKNPU_DEBUG_FS
- struct dentry *dent;
-#endif
-
-#ifdef CONFIG_ROCKCHIP_RKNPU_PROC_FS
- struct proc_dir_entry *pent;
-#endif
-
- struct list_head list;
-};
-
-struct rknpu_device;
-
-int rknpu_debugger_init(struct rknpu_device *rknpu_dev);
-int rknpu_debugger_remove(struct rknpu_device *rknpu_dev);
-
-#endif /* __LINUX_RKNPU_FENCE_H_ */
diff --git a/rknpu-driver/driver-0.9.6/include/rknpu_devfreq.h b/rknpu-driver/driver-0.9.6/include/rknpu_devfreq.h
deleted file mode 100644
index 3edeb1f..0000000
--- a/rknpu-driver/driver-0.9.6/include/rknpu_devfreq.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (C) Rockchip Electronics Co.Ltd
- * Author: Finley Xiao
- */
-
-#ifndef __LINUX_RKNPU_DEVFREQ_H
-#define __LINUX_RKNPU_DEVFREQ_H
-
-#ifdef CONFIG_PM_DEVFREQ
-void rknpu_devfreq_lock(struct rknpu_device *rknpu_dev);
-void rknpu_devfreq_unlock(struct rknpu_device *rknpu_dev);
-int rknpu_devfreq_init(struct rknpu_device *rknpu_dev);
-void rknpu_devfreq_remove(struct rknpu_device *rknpu_dev);
-int rknpu_devfreq_runtime_suspend(struct device *dev);
-int rknpu_devfreq_runtime_resume(struct device *dev);
-#else
-static inline int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
-{
- return -EOPNOTSUPP;
-}
-
-static inline void rknpu_devfreq_remove(struct rknpu_device *rknpu_dev)
-{
-}
-
-static inline void rknpu_devfreq_lock(struct rknpu_device *rknpu_dev)
-{
-}
-
-static inline void rknpu_devfreq_unlock(struct rknpu_device *rknpu_dev)
-{
-}
-
-static inline int rknpu_devfreq_runtime_suspend(struct device *dev)
-{
- return -EOPNOTSUPP;
-}
-
-static inline int rknpu_devfreq_runtime_resume(struct device *dev)
-{
- return -EOPNOTSUPP;
-}
-#endif /* CONFIG_PM_DEVFREQ */
-
-#endif /* __LINUX_RKNPU_DEVFREQ_H_ */
diff --git a/rknpu-driver/driver-0.9.6/include/rknpu_drv.h b/rknpu-driver/driver-0.9.6/include/rknpu_drv.h
deleted file mode 100644
index dbcb1b6..0000000
--- a/rknpu-driver/driver-0.9.6/include/rknpu_drv.h
+++ /dev/null
@@ -1,183 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (C) Rockchip Electronics Co.Ltd
- * Author: Felix Zeng
- */
-
-#ifndef __LINUX_RKNPU_DRV_H_
-#define __LINUX_RKNPU_DRV_H_
-
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-
-#include
-#include
-#include
-
-#include "rknpu_job.h"
-#include "rknpu_fence.h"
-#include "rknpu_debugger.h"
-#include "rknpu_mm.h"
-
-#define DRIVER_NAME "rknpu"
-#define DRIVER_DESC "RKNPU driver"
-#define DRIVER_DATE "20240322"
-#define DRIVER_MAJOR 0
-#define DRIVER_MINOR 9
-#define DRIVER_PATCHLEVEL 6
-
-#define LOG_TAG "RKNPU"
-
-/* sample interval: 1000ms */
-#define RKNPU_LOAD_INTERVAL 1000000000
-
-#define LOG_INFO(fmt, args...) pr_info(LOG_TAG ": " fmt, ##args)
-#if KERNEL_VERSION(5, 5, 0) <= LINUX_VERSION_CODE
-#define LOG_WARN(fmt, args...) pr_warn(LOG_TAG ": " fmt, ##args)
-#else
-#define LOG_WARN(fmt, args...) pr_warning(LOG_TAG ": " fmt, ##args)
-#endif
-#define LOG_DEBUG(fmt, args...) pr_devel(LOG_TAG ": " fmt, ##args)
-#define LOG_ERROR(fmt, args...) pr_err(LOG_TAG ": " fmt, ##args)
-
-#define LOG_DEV_INFO(dev, fmt, args...) dev_info(dev, LOG_TAG ": " fmt, ##args)
-#define LOG_DEV_WARN(dev, fmt, args...) dev_warn(dev, LOG_TAG ": " fmt, ##args)
-#define LOG_DEV_DEBUG(dev, fmt, args...) dev_dbg(dev, LOG_TAG ": " fmt, ##args)
-#define LOG_DEV_ERROR(dev, fmt, args...) dev_err(dev, LOG_TAG ": " fmt, ##args)
-
-#define RKNPU_MAX_IOMMU_DOMAIN_NUM 16
-
-struct rknpu_irqs_data {
- const char *name;
- irqreturn_t (*irq_hdl)(int irq, void *ctx);
-};
-
-struct rknpu_amount_data {
- uint16_t offset_clr_all;
- uint16_t offset_dt_wr;
- uint16_t offset_dt_rd;
- uint16_t offset_wt_rd;
-};
-
-struct rknpu_config {
- __u32 bw_priority_addr;
- __u32 bw_priority_length;
- __u64 dma_mask;
- __u32 pc_data_amount_scale;
- __u32 pc_task_number_bits;
- __u32 pc_task_number_mask;
- __u32 pc_task_status_offset;
- __u32 pc_dma_ctrl;
- const struct rknpu_irqs_data *irqs;
- int num_irqs;
- __u64 nbuf_phyaddr;
- __u64 nbuf_size;
- __u64 max_submit_number;
- __u32 core_mask;
- const struct rknpu_amount_data *amount_top;
- const struct rknpu_amount_data *amount_core;
-};
-
-struct rknpu_timer {
- ktime_t busy_time;
- ktime_t total_busy_time;
-};
-
-struct rknpu_subcore_data {
- struct list_head todo_list;
- wait_queue_head_t job_done_wq;
- struct rknpu_job *job;
- int64_t task_num;
- struct rknpu_timer timer;
-};
-
-/**
- * RKNPU device
- *
- * @base: IO mapped base address for device
- * @dev: Device instance
- * @drm_dev: DRM device instance
- */
-struct rknpu_device {
- void __iomem *base[RKNPU_MAX_CORES];
- struct device *dev;
-#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
- struct device *fake_dev;
- struct drm_device *drm_dev;
-#endif
-#ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
- struct miscdevice miscdev;
- struct rk_dma_heap *heap;
-#endif
- atomic_t sequence;
- spinlock_t lock;
- spinlock_t irq_lock;
- struct mutex power_lock;
- struct mutex reset_lock;
- struct mutex domain_lock;
- struct rknpu_subcore_data subcore_datas[RKNPU_MAX_CORES];
- const struct rknpu_config *config;
- void __iomem *bw_priority_base;
- struct rknpu_fence_context *fence_ctx;
- bool iommu_en;
- struct reset_control **srsts;
- int num_srsts;
- struct clk_bulk_data *clks;
- int num_clks;
- struct regulator *vdd;
- struct regulator *mem;
- struct monitor_dev_info *mdev_info;
- struct ipa_power_model_data *model_data;
- struct thermal_cooling_device *devfreq_cooling;
- struct devfreq *devfreq;
- unsigned long ondemand_freq;
- struct rockchip_opp_info opp_info;
- unsigned long current_freq;
- unsigned long current_volt;
- int bypass_irq_handler;
- int bypass_soft_reset;
- bool soft_reseting;
- struct device *genpd_dev_npu0;
- struct device *genpd_dev_npu1;
- struct device *genpd_dev_npu2;
- bool multiple_domains;
- atomic_t power_refcount;
- atomic_t cmdline_power_refcount;
- struct delayed_work power_off_work;
- struct workqueue_struct *power_off_wq;
- struct rknpu_debugger debugger;
- struct hrtimer timer;
- ktime_t kt;
- phys_addr_t sram_start;
- phys_addr_t sram_end;
- phys_addr_t nbuf_start;
- phys_addr_t nbuf_end;
- uint32_t sram_size;
- uint32_t nbuf_size;
- void __iomem *sram_base_io;
- void __iomem *nbuf_base_io;
- struct rknpu_mm *sram_mm;
- unsigned long power_put_delay;
- struct iommu_group *iommu_group;
- int iommu_domain_num;
- int iommu_domain_id;
- struct iommu_domain *iommu_domains[RKNPU_MAX_IOMMU_DOMAIN_NUM];
-};
-
-struct rknpu_session {
- struct rknpu_device *rknpu_dev;
- struct list_head list;
-};
-
-int rknpu_power_get(struct rknpu_device *rknpu_dev);
-int rknpu_power_put(struct rknpu_device *rknpu_dev);
-
-#endif /* __LINUX_RKNPU_DRV_H_ */
diff --git a/rknpu-driver/driver-0.9.6/include/rknpu_fence.h b/rknpu-driver/driver-0.9.6/include/rknpu_fence.h
deleted file mode 100644
index 164f6de..0000000
--- a/rknpu-driver/driver-0.9.6/include/rknpu_fence.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (C) Rockchip Electronics Co.Ltd
- * Author: Felix Zeng
- */
-
-#ifndef __LINUX_RKNPU_FENCE_H_
-#define __LINUX_RKNPU_FENCE_H_
-
-#include "rknpu_job.h"
-
-struct rknpu_fence_context {
- unsigned int context;
- unsigned int seqno;
- spinlock_t spinlock;
-};
-
-int rknpu_fence_context_alloc(struct rknpu_device *rknpu_dev);
-
-int rknpu_fence_alloc(struct rknpu_job *job);
-
-int rknpu_fence_get_fd(struct rknpu_job *job);
-
-#endif /* __LINUX_RKNPU_FENCE_H_ */
diff --git a/rknpu-driver/driver-0.9.6/include/rknpu_gem.h b/rknpu-driver/driver-0.9.6/include/rknpu_gem.h
deleted file mode 100644
index 903c0fa..0000000
--- a/rknpu-driver/driver-0.9.6/include/rknpu_gem.h
+++ /dev/null
@@ -1,215 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (C) Rockchip Electronics Co.Ltd
- * Author: Felix Zeng
- */
-
-#ifndef __LINUX_RKNPU_GEM_H
-#define __LINUX_RKNPU_GEM_H
-
-#include
-#include
-
-#include
-#include
-#include
-#include
-
-#if KERNEL_VERSION(4, 14, 0) > LINUX_VERSION_CODE
-#include
-#endif
-
-#include "rknpu_mm.h"
-
-#define to_rknpu_obj(x) container_of(x, struct rknpu_gem_object, base)
-
-/*
- * rknpu drm buffer structure.
- *
- * @base: a gem object.
- * - a new handle to this gem object would be created
- * by drm_gem_handle_create().
- * @flags: indicate memory type to allocated buffer and cache attribute.
- * @size: size requested from user, in bytes and this size is aligned
- * in page unit.
- * @cookie: cookie returned by dma_alloc_attrs
- * @kv_addr: kernel virtual address to allocated memory region.
- * @dma_addr: bus address(accessed by dma) to allocated memory region.
- * - this address could be physical address without IOMMU and
- * device address with IOMMU.
- * @pages: Array of backing pages.
- * @sgt: Imported sg_table.
- *
- * P.S. this object would be transferred to user as kms_bo.handle so
- * user can access the buffer through kms_bo.handle.
- */
-struct rknpu_gem_object {
- struct drm_gem_object base;
- unsigned int flags;
- unsigned long size;
- unsigned long sram_size;
- unsigned long nbuf_size;
- struct rknpu_mm_obj *sram_obj;
- dma_addr_t iova_start;
- unsigned long iova_size;
- void *cookie;
- void __iomem *kv_addr;
- dma_addr_t dma_addr;
- unsigned long dma_attrs;
- unsigned long num_pages;
- struct page **pages;
- struct sg_table *sgt;
- struct drm_mm_node mm_node;
- int iommu_domain_id;
-};
-
-enum rknpu_cache_type {
- RKNPU_CACHE_SRAM = 1 << 0,
- RKNPU_CACHE_NBUF = 1 << 1,
-};
-
-/* create a new buffer with gem object */
-struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *dev,
- unsigned int flags,
- unsigned long size,
- unsigned long sram_size,
- int iommu_domain_id);
-
-/* destroy a buffer with gem object */
-void rknpu_gem_object_destroy(struct rknpu_gem_object *rknpu_obj);
-
-/* request gem object creation and buffer allocation as the size */
-int rknpu_gem_create_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file_priv);
-
-/* get fake-offset of gem object that can be used with mmap. */
-int rknpu_gem_map_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file_priv);
-
-int rknpu_gem_destroy_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file_priv);
-
-/*
- * get rknpu drm object,
- * gem object reference count would be increased.
- */
-static inline void rknpu_gem_object_get(struct drm_gem_object *obj)
-{
-#if KERNEL_VERSION(4, 13, 0) < LINUX_VERSION_CODE
- drm_gem_object_get(obj);
-#else
- drm_gem_object_reference(obj);
-#endif
-}
-
-/*
- * put rknpu drm object acquired from rknpu_gem_object_find() or rknpu_gem_object_get(),
- * gem object reference count would be decreased.
- */
-static inline void rknpu_gem_object_put(struct drm_gem_object *obj)
-{
-#if KERNEL_VERSION(5, 9, 0) <= LINUX_VERSION_CODE
- drm_gem_object_put(obj);
-#elif KERNEL_VERSION(4, 13, 0) < LINUX_VERSION_CODE
- drm_gem_object_put_unlocked(obj);
-#else
- drm_gem_object_unreference_unlocked(obj);
-#endif
-}
-
-/*
- * get rknpu drm object from gem handle, this function could be used for
- * other drivers such as 2d/3d acceleration drivers.
- * with this function call, gem object reference count would be increased.
- */
-static inline struct rknpu_gem_object *
-rknpu_gem_object_find(struct drm_file *filp, unsigned int handle)
-{
- struct drm_gem_object *obj;
-
- obj = drm_gem_object_lookup(filp, handle);
- if (!obj) {
- // DRM_ERROR("failed to lookup gem object.\n");
- return NULL;
- }
-
- rknpu_gem_object_put(obj);
-
- return to_rknpu_obj(obj);
-}
-
-/* get buffer information to memory region allocated by gem. */
-int rknpu_gem_get_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file_priv);
-
-/* free gem object. */
-void rknpu_gem_free_object(struct drm_gem_object *obj);
-
-/* create memory region for drm framebuffer. */
-int rknpu_gem_dumb_create(struct drm_file *file_priv, struct drm_device *dev,
- struct drm_mode_create_dumb *args);
-
-#if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE
-/* map memory region for drm framebuffer to user space. */
-int rknpu_gem_dumb_map_offset(struct drm_file *file_priv,
- struct drm_device *dev, uint32_t handle,
- uint64_t *offset);
-#endif
-
-/* page fault handler and mmap fault address(virtual) to physical memory. */
-#if KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE
-vm_fault_t rknpu_gem_fault(struct vm_fault *vmf);
-#elif KERNEL_VERSION(4, 14, 0) <= LINUX_VERSION_CODE
-int rknpu_gem_fault(struct vm_fault *vmf);
-#else
-int rknpu_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
-#endif
-
-int rknpu_gem_mmap_obj(struct drm_gem_object *obj, struct vm_area_struct *vma);
-
-/* set vm_flags and we can change the vm attribute to other one at here. */
-int rknpu_gem_mmap(struct file *filp, struct vm_area_struct *vma);
-
-/* low-level interface prime helpers */
-#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
-struct drm_gem_object *rknpu_gem_prime_import(struct drm_device *dev,
- struct dma_buf *dma_buf);
-#endif
-struct sg_table *rknpu_gem_prime_get_sg_table(struct drm_gem_object *obj);
-struct drm_gem_object *
-rknpu_gem_prime_import_sg_table(struct drm_device *dev,
- struct dma_buf_attachment *attach,
- struct sg_table *sgt);
-#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
-void *rknpu_gem_prime_vmap(struct drm_gem_object *obj);
-void rknpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);
-#else
-int rknpu_gem_prime_vmap(struct drm_gem_object *obj, struct iosys_map *map);
-void rknpu_gem_prime_vunmap(struct drm_gem_object *obj, struct iosys_map *map);
-#endif
-int rknpu_gem_prime_mmap(struct drm_gem_object *obj,
- struct vm_area_struct *vma);
-
-int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file_priv);
-
-static inline void *rknpu_gem_alloc_page(size_t nr_pages)
-{
-#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
- return kvmalloc_array(nr_pages, sizeof(struct page *),
- GFP_KERNEL | __GFP_ZERO);
-#else
- return drm_calloc_large(nr_pages, sizeof(struct page *));
-#endif
-}
-
-static inline void rknpu_gem_free_page(void *pages)
-{
-#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
- kvfree(pages);
-#else
- drm_free_large(pages);
-#endif
-}
-
-#endif
diff --git a/rknpu-driver/driver-0.9.6/include/rknpu_ioctl.h b/rknpu-driver/driver-0.9.6/include/rknpu_ioctl.h
deleted file mode 100644
index 76724f3..0000000
--- a/rknpu-driver/driver-0.9.6/include/rknpu_ioctl.h
+++ /dev/null
@@ -1,327 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (C) Rockchip Electronics Co.Ltd
- * Author: Felix Zeng
- */
-
-#ifndef __LINUX_RKNPU_IOCTL_H
-#define __LINUX_RKNPU_IOCTL_H
-
-#include
-#include
-
-#if !defined(__KERNEL__)
-#define __user
-#endif
-
-#ifndef __packed
-#define __packed __attribute__((packed))
-#endif
-
-#define RKNPU_OFFSET_VERSION 0x0
-#define RKNPU_OFFSET_VERSION_NUM 0x4
-#define RKNPU_OFFSET_PC_OP_EN 0x8
-#define RKNPU_OFFSET_PC_DATA_ADDR 0x10
-#define RKNPU_OFFSET_PC_DATA_AMOUNT 0x14
-#define RKNPU_OFFSET_PC_TASK_CONTROL 0x30
-#define RKNPU_OFFSET_PC_DMA_BASE_ADDR 0x34
-
-#define RKNPU_OFFSET_INT_MASK 0x20
-#define RKNPU_OFFSET_INT_CLEAR 0x24
-#define RKNPU_OFFSET_INT_STATUS 0x28
-#define RKNPU_OFFSET_INT_RAW_STATUS 0x2c
-
-#define RKNPU_OFFSET_ENABLE_MASK 0xf008
-
-#define RKNPU_INT_CLEAR 0x1ffff
-
-#define RKNPU_PC_DATA_EXTRA_AMOUNT 4
-
-#define RKNPU_STR_HELPER(x) #x
-
-#define RKNPU_GET_DRV_VERSION_STRING(MAJOR, MINOR, PATCHLEVEL) \
- RKNPU_STR_HELPER(MAJOR) \
- "." RKNPU_STR_HELPER(MINOR) "." RKNPU_STR_HELPER(PATCHLEVEL)
-#define RKNPU_GET_DRV_VERSION_CODE(MAJOR, MINOR, PATCHLEVEL) \
- (MAJOR * 10000 + MINOR * 100 + PATCHLEVEL)
-#define RKNPU_GET_DRV_VERSION_MAJOR(CODE) (CODE / 10000)
-#define RKNPU_GET_DRV_VERSION_MINOR(CODE) ((CODE % 10000) / 100)
-#define RKNPU_GET_DRV_VERSION_PATCHLEVEL(CODE) (CODE % 100)
-
-/* memory type definitions. */
-enum e_rknpu_mem_type {
- /* physically continuous memory and used as default. */
- RKNPU_MEM_CONTIGUOUS = 0 << 0,
- /* physically non-continuous memory. */
- RKNPU_MEM_NON_CONTIGUOUS = 1 << 0,
- /* non-cacheable mapping and used as default. */
- RKNPU_MEM_NON_CACHEABLE = 0 << 1,
- /* cacheable mapping. */
- RKNPU_MEM_CACHEABLE = 1 << 1,
- /* write-combine mapping. */
- RKNPU_MEM_WRITE_COMBINE = 1 << 2,
- /* dma attr kernel mapping */
- RKNPU_MEM_KERNEL_MAPPING = 1 << 3,
- /* iommu mapping */
- RKNPU_MEM_IOMMU = 1 << 4,
- /* zero mapping */
- RKNPU_MEM_ZEROING = 1 << 5,
- /* allocate secure buffer */
- RKNPU_MEM_SECURE = 1 << 6,
- /* allocate from dma32 zone */
- RKNPU_MEM_DMA32 = 1 << 7,
- /* request SRAM */
- RKNPU_MEM_TRY_ALLOC_SRAM = 1 << 8,
- /* request NBUF */
- RKNPU_MEM_TRY_ALLOC_NBUF = 1 << 9,
- RKNPU_MEM_MASK = RKNPU_MEM_NON_CONTIGUOUS | RKNPU_MEM_CACHEABLE |
- RKNPU_MEM_WRITE_COMBINE | RKNPU_MEM_KERNEL_MAPPING |
- RKNPU_MEM_IOMMU | RKNPU_MEM_ZEROING |
- RKNPU_MEM_SECURE | RKNPU_MEM_DMA32 |
- RKNPU_MEM_TRY_ALLOC_SRAM | RKNPU_MEM_TRY_ALLOC_NBUF
-};
-
-/* sync mode definitions. */
-enum e_rknpu_mem_sync_mode {
- RKNPU_MEM_SYNC_TO_DEVICE = 1 << 0,
- RKNPU_MEM_SYNC_FROM_DEVICE = 1 << 1,
- RKNPU_MEM_SYNC_MASK =
- RKNPU_MEM_SYNC_TO_DEVICE | RKNPU_MEM_SYNC_FROM_DEVICE
-};
-
-/* job mode definitions. */
-enum e_rknpu_job_mode {
- RKNPU_JOB_SLAVE = 0 << 0,
- RKNPU_JOB_PC = 1 << 0,
- RKNPU_JOB_BLOCK = 0 << 1,
- RKNPU_JOB_NONBLOCK = 1 << 1,
- RKNPU_JOB_PINGPONG = 1 << 2,
- RKNPU_JOB_FENCE_IN = 1 << 3,
- RKNPU_JOB_FENCE_OUT = 1 << 4,
- RKNPU_JOB_MASK = RKNPU_JOB_PC | RKNPU_JOB_NONBLOCK |
- RKNPU_JOB_PINGPONG | RKNPU_JOB_FENCE_IN |
- RKNPU_JOB_FENCE_OUT
-};
-
-/* action definitions */
-enum e_rknpu_action {
- RKNPU_GET_HW_VERSION = 0,
- RKNPU_GET_DRV_VERSION = 1,
- RKNPU_GET_FREQ = 2,
- RKNPU_SET_FREQ = 3,
- RKNPU_GET_VOLT = 4,
- RKNPU_SET_VOLT = 5,
- RKNPU_ACT_RESET = 6,
- RKNPU_GET_BW_PRIORITY = 7,
- RKNPU_SET_BW_PRIORITY = 8,
- RKNPU_GET_BW_EXPECT = 9,
- RKNPU_SET_BW_EXPECT = 10,
- RKNPU_GET_BW_TW = 11,
- RKNPU_SET_BW_TW = 12,
- RKNPU_ACT_CLR_TOTAL_RW_AMOUNT = 13,
- RKNPU_GET_DT_WR_AMOUNT = 14,
- RKNPU_GET_DT_RD_AMOUNT = 15,
- RKNPU_GET_WT_RD_AMOUNT = 16,
- RKNPU_GET_TOTAL_RW_AMOUNT = 17,
- RKNPU_GET_IOMMU_EN = 18,
- RKNPU_SET_PROC_NICE = 19,
- RKNPU_POWER_ON = 20,
- RKNPU_POWER_OFF = 21,
- RKNPU_GET_TOTAL_SRAM_SIZE = 22,
- RKNPU_GET_FREE_SRAM_SIZE = 23,
- RKNPU_GET_IOMMU_DOMAIN_ID = 24,
- RKNPU_SET_IOMMU_DOMAIN_ID = 25,
-};
-
-/**
- * User-desired buffer creation information structure.
- *
- * @handle: The handle of the created GEM object.
- * @flags: user request for setting memory type or cache attributes.
- * @size: user-desired memory allocation size.
- * - this size value would be page-aligned internally.
- * @obj_addr: address of RKNPU memory object.
- * @dma_addr: dma address that access by rknpu.
- * @sram_size: user-desired sram memory allocation size.
- * - this size value would be page-aligned internally.
- * @iommu_domain_id: iommu domain id
- * @reserved: just padding to be 64-bit aligned.
- */
-struct rknpu_mem_create {
- __u32 handle;
- __u32 flags;
- __u64 size;
- __u64 obj_addr;
- __u64 dma_addr;
- __u64 sram_size;
- __s32 iommu_domain_id;
- __u32 reserved;
-};
-
-/**
- * A structure for getting a fake-offset that can be used with mmap.
- *
- * @handle: handle of gem object.
- * @reserved: just padding to be 64-bit aligned.
- * @offset: a fake-offset of gem object.
- */
-struct rknpu_mem_map {
- __u32 handle;
- __u32 reserved;
- __u64 offset;
-};
-
-/**
- * For destroying DMA buffer
- *
- * @handle: handle of the buffer.
- * @reserved: reserved for padding.
- * @obj_addr: rknpu_mem_object addr.
- */
-struct rknpu_mem_destroy {
- __u32 handle;
- __u32 reserved;
- __u64 obj_addr;
-};
-
-/**
- * For synchronizing DMA buffer
- *
- * @flags: user request for setting memory type or cache attributes.
- * @reserved: reserved for padding.
- * @obj_addr: address of RKNPU memory object.
- * @offset: offset in bytes from start address of buffer.
- * @size: size of memory region.
- *
- */
-struct rknpu_mem_sync {
- __u32 flags;
- __u32 reserved;
- __u64 obj_addr;
- __u64 offset;
- __u64 size;
-};
-
-/**
- * struct rknpu_task structure for task information
- *
- * @flags: flags for task
- * @op_idx: operator index
- * @enable_mask: enable mask
- * @int_mask: interrupt mask
- * @int_clear: interrupt clear
- * @int_status: interrupt status
- * @regcfg_amount: register config number
- * @regcfg_offset: offset for register config
- * @regcmd_addr: address for register command
- *
- */
-struct rknpu_task {
- __u32 flags;
- __u32 op_idx;
- __u32 enable_mask;
- __u32 int_mask;
- __u32 int_clear;
- __u32 int_status;
- __u32 regcfg_amount;
- __u32 regcfg_offset;
- __u64 regcmd_addr;
-} __packed;
-
-/**
- * struct rknpu_subcore_task structure for subcore task index
- *
- * @task_start: task start index
- * @task_number: task number
- *
- */
-struct rknpu_subcore_task {
- __u32 task_start;
- __u32 task_number;
-};
-
-/**
- * struct rknpu_submit structure for job submit
- *
- * @flags: flags for job submit
- * @timeout: submit timeout
- * @task_start: task start index
- * @task_number: task number
- * @task_counter: task counter
- * @priority: submit priority
- * @task_obj_addr: address of task object
- * @iommu_domain_id: iommu domain id
- * @reserved: just padding to be 64-bit aligned.
- * @task_base_addr: task base address
- * @hw_elapse_time: hardware elapse time
- * @core_mask: core mask of rknpu
- * @fence_fd: dma fence fd
- * @subcore_task: subcore task
- *
- */
-struct rknpu_submit {
- __u32 flags;
- __u32 timeout;
- __u32 task_start;
- __u32 task_number;
- __u32 task_counter;
- __s32 priority;
- __u64 task_obj_addr;
- __u32 iommu_domain_id;
- __u32 reserved;
- __u64 task_base_addr;
- __s64 hw_elapse_time;
- __u32 core_mask;
- __s32 fence_fd;
- struct rknpu_subcore_task subcore_task[5];
-};
-
-/**
- * struct rknpu_task structure for action (GET, SET or ACT)
- *
- * @flags: flags for action
- * @value: GET or SET value
- *
- */
-struct rknpu_action {
- __u32 flags;
- __u32 value;
-};
-
-#define RKNPU_ACTION 0x00
-#define RKNPU_SUBMIT 0x01
-#define RKNPU_MEM_CREATE 0x02
-#define RKNPU_MEM_MAP 0x03
-#define RKNPU_MEM_DESTROY 0x04
-#define RKNPU_MEM_SYNC 0x05
-
-#define RKNPU_IOC_MAGIC 'r'
-#define RKNPU_IOW(nr, type) _IOW(RKNPU_IOC_MAGIC, nr, type)
-#define RKNPU_IOR(nr, type) _IOR(RKNPU_IOC_MAGIC, nr, type)
-#define RKNPU_IOWR(nr, type) _IOWR(RKNPU_IOC_MAGIC, nr, type)
-
-#include
-
-#define DRM_IOCTL_RKNPU_ACTION \
- DRM_IOWR(DRM_COMMAND_BASE + RKNPU_ACTION, struct rknpu_action)
-#define DRM_IOCTL_RKNPU_SUBMIT \
- DRM_IOWR(DRM_COMMAND_BASE + RKNPU_SUBMIT, struct rknpu_submit)
-#define DRM_IOCTL_RKNPU_MEM_CREATE \
- DRM_IOWR(DRM_COMMAND_BASE + RKNPU_MEM_CREATE, struct rknpu_mem_create)
-#define DRM_IOCTL_RKNPU_MEM_MAP \
- DRM_IOWR(DRM_COMMAND_BASE + RKNPU_MEM_MAP, struct rknpu_mem_map)
-#define DRM_IOCTL_RKNPU_MEM_DESTROY \
- DRM_IOWR(DRM_COMMAND_BASE + RKNPU_MEM_DESTROY, struct rknpu_mem_destroy)
-#define DRM_IOCTL_RKNPU_MEM_SYNC \
- DRM_IOWR(DRM_COMMAND_BASE + RKNPU_MEM_SYNC, struct rknpu_mem_sync)
-
-#define IOCTL_RKNPU_ACTION RKNPU_IOWR(RKNPU_ACTION, struct rknpu_action)
-#define IOCTL_RKNPU_SUBMIT RKNPU_IOWR(RKNPU_SUBMIT, struct rknpu_submit)
-#define IOCTL_RKNPU_MEM_CREATE \
- RKNPU_IOWR(RKNPU_MEM_CREATE, struct rknpu_mem_create)
-#define IOCTL_RKNPU_MEM_MAP RKNPU_IOWR(RKNPU_MEM_MAP, struct rknpu_mem_map)
-#define IOCTL_RKNPU_MEM_DESTROY \
- RKNPU_IOWR(RKNPU_MEM_DESTROY, struct rknpu_mem_destroy)
-#define IOCTL_RKNPU_MEM_SYNC RKNPU_IOWR(RKNPU_MEM_SYNC, struct rknpu_mem_sync)
-
-#endif
diff --git a/rknpu-driver/driver-0.9.6/include/rknpu_iommu.h b/rknpu-driver/driver-0.9.6/include/rknpu_iommu.h
deleted file mode 100644
index 43d36db..0000000
--- a/rknpu-driver/driver-0.9.6/include/rknpu_iommu.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (C) Rockchip Electronics Co.Ltd
- * Author: Felix Zeng
- */
-
-#ifndef __LINUX_RKNPU_IOMMU_H
-#define __LINUX_RKNPU_IOMMU_H
-
-#include
-#include
-#include
-#include
-#include
-
-#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
-#include
-#endif
-
-#include "rknpu_drv.h"
-
-enum iommu_dma_cookie_type {
- IOMMU_DMA_IOVA_COOKIE,
- IOMMU_DMA_MSI_COOKIE,
-};
-
-struct rknpu_iommu_dma_cookie {
- enum iommu_dma_cookie_type type;
-
- /* Full allocator for IOMMU_DMA_IOVA_COOKIE */
- struct iova_domain iovad;
-};
-
-dma_addr_t rknpu_iommu_dma_alloc_iova(struct iommu_domain *domain, size_t size,
- u64 dma_limit, struct device *dev);
-
-void rknpu_iommu_dma_free_iova(struct rknpu_iommu_dma_cookie *cookie,
- dma_addr_t iova, size_t size);
-
-int rknpu_iommu_init_domain(struct rknpu_device *rknpu_dev);
-int rknpu_iommu_switch_domain(struct rknpu_device *rknpu_dev, int domain_id);
-void rknpu_iommu_free_domains(struct rknpu_device *rknpu_dev);
-
-#if KERNEL_VERSION(5, 10, 0) < LINUX_VERSION_CODE
-int iommu_get_dma_cookie(struct iommu_domain *domain);
-#endif
-
-#endif
diff --git a/rknpu-driver/driver-0.9.6/include/rknpu_job.h b/rknpu-driver/driver-0.9.6/include/rknpu_job.h
deleted file mode 100644
index 46eb304..0000000
--- a/rknpu-driver/driver-0.9.6/include/rknpu_job.h
+++ /dev/null
@@ -1,81 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (C) Rockchip Electronics Co.Ltd
- * Author: Felix Zeng
- */
-
-#ifndef __LINUX_RKNPU_JOB_H_
-#define __LINUX_RKNPU_JOB_H_
-
-#include
-#include
-#include
-
-#include
-
-#include "rknpu_ioctl.h"
-
-#define RKNPU_MAX_CORES 3
-
-#define RKNPU_JOB_DONE (1 << 0)
-#define RKNPU_JOB_ASYNC (1 << 1)
-#define RKNPU_JOB_DETACHED (1 << 2)
-
-#define RKNPU_CORE_AUTO_MASK 0x00
-#define RKNPU_CORE0_MASK 0x01
-#define RKNPU_CORE1_MASK 0x02
-#define RKNPU_CORE2_MASK 0x04
-
-struct rknpu_job {
- struct rknpu_device *rknpu_dev;
- struct list_head head[RKNPU_MAX_CORES];
- struct work_struct cleanup_work;
- bool irq_entry[RKNPU_MAX_CORES];
- unsigned int flags;
- int ret;
- struct rknpu_submit *args;
- bool args_owner;
- struct rknpu_task *first_task;
- struct rknpu_task *last_task;
- uint32_t int_mask[RKNPU_MAX_CORES];
- uint32_t int_status[RKNPU_MAX_CORES];
- struct dma_fence *fence;
- ktime_t timestamp;
- uint32_t use_core_num;
- atomic_t run_count;
- atomic_t interrupt_count;
- ktime_t hw_commit_time;
- ktime_t hw_recoder_time;
- ktime_t hw_elapse_time;
- atomic_t submit_count[RKNPU_MAX_CORES];
- int iommu_domain_id;
-};
-
-irqreturn_t rknpu_core0_irq_handler(int irq, void *data);
-irqreturn_t rknpu_core1_irq_handler(int irq, void *data);
-irqreturn_t rknpu_core2_irq_handler(int irq, void *data);
-
-#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
-int rknpu_submit_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file_priv);
-#endif
-#ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
-int rknpu_submit_ioctl(struct rknpu_device *rknpu_dev, unsigned long data);
-#endif
-
-int rknpu_get_hw_version(struct rknpu_device *rknpu_dev, uint32_t *version);
-
-int rknpu_get_bw_priority(struct rknpu_device *rknpu_dev, uint32_t *priority,
- uint32_t *expect, uint32_t *tw);
-
-int rknpu_set_bw_priority(struct rknpu_device *rknpu_dev, uint32_t priority,
- uint32_t expect, uint32_t tw);
-
-int rknpu_clear_rw_amount(struct rknpu_device *rknpu_dev);
-
-int rknpu_get_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *dt_wr,
- uint32_t *dt_rd, uint32_t *wd_rd);
-
-int rknpu_get_total_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *amount);
-
-#endif /* __LINUX_RKNPU_JOB_H_ */
diff --git a/rknpu-driver/driver-0.9.6/include/rknpu_mem.h b/rknpu-driver/driver-0.9.6/include/rknpu_mem.h
deleted file mode 100644
index 9cc5d1d..0000000
--- a/rknpu-driver/driver-0.9.6/include/rknpu_mem.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (C) Rockchip Electronics Co.Ltd
- * Author: Felix Zeng
- */
-
-#ifndef __LINUX_RKNPU_MEM_H
-#define __LINUX_RKNPU_MEM_H
-
-#include
-#include
-
-/*
- * rknpu DMA buffer structure.
- *
- * @flags: indicate memory type to allocated buffer and cache attribute.
- * @size: size requested from user, in bytes and this size is aligned
- * in page unit.
- * @kv_addr: kernel virtual address to allocated memory region.
- * @dma_addr: bus address(accessed by dma) to allocated memory region.
- * - this address could be physical address without IOMMU and
- * device address with IOMMU.
- * @pages: Array of backing pages.
- * @sgt: Imported sg_table.
- * @dmabuf: buffer for this attachment.
- * @owner: Is this memory internally allocated.
- */
-struct rknpu_mem_object {
- unsigned long flags;
- unsigned long size;
- void __iomem *kv_addr;
- dma_addr_t dma_addr;
- struct page **pages;
- struct sg_table *sgt;
- struct dma_buf *dmabuf;
- struct list_head head;
- unsigned int owner;
-};
-
-int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, struct file *file,
- unsigned int cmd, unsigned long data);
-int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, struct file *file,
- unsigned long data);
-int rknpu_mem_sync_ioctl(struct rknpu_device *rknpu_dev, unsigned long data);
-
-#endif
diff --git a/rknpu-driver/driver-0.9.6/include/rknpu_mm.h b/rknpu-driver/driver-0.9.6/include/rknpu_mm.h
deleted file mode 100644
index 73ae6d7..0000000
--- a/rknpu-driver/driver-0.9.6/include/rknpu_mm.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (C) Rockchip Electronics Co.Ltd
- * Author: Felix Zeng
- */
-
-#ifndef __LINUX_RKNPU_MM_H
-#define __LINUX_RKNPU_MM_H
-
-#include
-#include
-#include
-#include
-
-#include "rknpu_drv.h"
-
-struct rknpu_mm {
- void *bitmap;
- struct mutex lock;
- unsigned int chunk_size;
- unsigned int total_chunks;
- unsigned int free_chunks;
-};
-
-struct rknpu_mm_obj {
- uint32_t range_start;
- uint32_t range_end;
-};
-
-int rknpu_mm_create(unsigned int mem_size, unsigned int chunk_size,
- struct rknpu_mm **mm);
-
-void rknpu_mm_destroy(struct rknpu_mm *mm);
-
-int rknpu_mm_alloc(struct rknpu_mm *mm, unsigned int size,
- struct rknpu_mm_obj **mm_obj);
-
-int rknpu_mm_free(struct rknpu_mm *mm, struct rknpu_mm_obj *mm_obj);
-
-int rknpu_mm_dump(struct seq_file *m, void *data);
-
-#endif
diff --git a/rknpu-driver/driver-0.9.6/include/rknpu_reset.h b/rknpu-driver/driver-0.9.6/include/rknpu_reset.h
deleted file mode 100644
index b80e29b..0000000
--- a/rknpu-driver/driver-0.9.6/include/rknpu_reset.h
+++ /dev/null
@@ -1,18 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (C) Rockchip Electronics Co.Ltd
- * Author: Felix Zeng
- */
-
-#ifndef __LINUX_RKNPU_RESET_H
-#define __LINUX_RKNPU_RESET_H
-
-#include
-
-#include "rknpu_drv.h"
-
-int rknpu_reset_get(struct rknpu_device *rknpu_dev);
-
-int rknpu_soft_reset(struct rknpu_device *rknpu_dev);
-
-#endif
diff --git a/rknpu-driver/driver-0.9.6/rknpu.mod.c b/rknpu-driver/driver-0.9.6/rknpu.mod.c
deleted file mode 100644
index 041c0da..0000000
--- a/rknpu-driver/driver-0.9.6/rknpu.mod.c
+++ /dev/null
@@ -1,31 +0,0 @@
-#include
-#define INCLUDE_VERMAGIC
-#include
-#include
-#include
-
-BUILD_SALT;
-
-MODULE_INFO(vermagic, VERMAGIC_STRING);
-MODULE_INFO(name, KBUILD_MODNAME);
-
-__visible struct module __this_module
-__section(".gnu.linkonce.this_module") = {
- .name = KBUILD_MODNAME,
- .init = init_module,
-#ifdef CONFIG_MODULE_UNLOAD
- .exit = cleanup_module,
-#endif
- .arch = MODULE_ARCH_INIT,
-};
-
-MODULE_INFO(intree, "Y");
-
-#ifdef CONFIG_RETPOLINE
-MODULE_INFO(retpoline, "Y");
-#endif
-
-MODULE_INFO(depends, "");
-
-
-MODULE_INFO(srcversion, "3F8407365EF7935915D4CC3");
diff --git a/rknpu-driver/driver-0.9.6/rknpu_debugger.c b/rknpu-driver/driver-0.9.6/rknpu_debugger.c
deleted file mode 100644
index 0cfec7f..0000000
--- a/rknpu-driver/driver-0.9.6/rknpu_debugger.c
+++ /dev/null
@@ -1,605 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) Rockchip Electronics Co.Ltd
- * Author: Felix Zeng
- */
-
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-
-#ifndef FPGA_PLATFORM
-#ifdef CONFIG_PM_DEVFREQ
-#include <../drivers/devfreq/governor.h>
-#endif
-#endif
-
-#include "rknpu_drv.h"
-#include "rknpu_mm.h"
-#include "rknpu_reset.h"
-#include "rknpu_debugger.h"
-
-#define RKNPU_DEBUGGER_ROOT_NAME "rknpu"
-
-#if defined(CONFIG_ROCKCHIP_RKNPU_DEBUG_FS) || \
- defined(CONFIG_ROCKCHIP_RKNPU_PROC_FS)
-static int rknpu_version_show(struct seq_file *m, void *data)
-{
- seq_printf(m, "%s: v%d.%d.%d\n", DRIVER_DESC, DRIVER_MAJOR,
- DRIVER_MINOR, DRIVER_PATCHLEVEL);
-
- return 0;
-}
-
-static int rknpu_load_show(struct seq_file *m, void *data)
-{
- struct rknpu_debugger_node *node = m->private;
- struct rknpu_debugger *debugger = node->debugger;
- struct rknpu_device *rknpu_dev =
- container_of(debugger, struct rknpu_device, debugger);
- struct rknpu_subcore_data *subcore_data = NULL;
- unsigned long flags;
- int i;
- int load;
- uint64_t total_busy_time, div_value;
-
- seq_puts(m, "NPU load: ");
- for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
- subcore_data = &rknpu_dev->subcore_datas[i];
-
- if (rknpu_dev->config->num_irqs > 1)
- seq_printf(m, " Core%d: ", i);
-
- spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
-
- total_busy_time = subcore_data->timer.total_busy_time;
-
- spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
-
- div_value = (RKNPU_LOAD_INTERVAL / 100);
- do_div(total_busy_time, div_value);
- load = total_busy_time > 100 ? 100 : total_busy_time;
-
- if (rknpu_dev->config->num_irqs > 1)
- seq_printf(m, "%2.d%%,", load);
- else
- seq_printf(m, "%2.d%%", load);
- }
- seq_puts(m, "\n");
-
- return 0;
-}
-
-static int rknpu_power_show(struct seq_file *m, void *data)
-{
- struct rknpu_debugger_node *node = m->private;
- struct rknpu_debugger *debugger = node->debugger;
- struct rknpu_device *rknpu_dev =
- container_of(debugger, struct rknpu_device, debugger);
-
- if (atomic_read(&rknpu_dev->power_refcount) > 0)
- seq_puts(m, "on\n");
- else
- seq_puts(m, "off\n");
-
- return 0;
-}
-
-static ssize_t rknpu_power_set(struct file *file, const char __user *ubuf,
- size_t len, loff_t *offp)
-{
- struct seq_file *priv = file->private_data;
- struct rknpu_debugger_node *node = priv->private;
- struct rknpu_debugger *debugger = node->debugger;
- struct rknpu_device *rknpu_dev =
- container_of(debugger, struct rknpu_device, debugger);
- char buf[8];
-
- if (len > sizeof(buf) - 1)
- return -EINVAL;
- if (copy_from_user(buf, ubuf, len))
- return -EFAULT;
- buf[len - 1] = '\0';
-
- if (strcmp(buf, "on") == 0) {
- atomic_inc(&rknpu_dev->cmdline_power_refcount);
- rknpu_power_get(rknpu_dev);
- LOG_INFO("rknpu power is on!");
- } else if (strcmp(buf, "off") == 0) {
- if (atomic_read(&rknpu_dev->power_refcount) > 0 &&
- atomic_dec_if_positive(
- &rknpu_dev->cmdline_power_refcount) >= 0) {
- atomic_sub(
- atomic_read(&rknpu_dev->cmdline_power_refcount),
- &rknpu_dev->power_refcount);
- atomic_set(&rknpu_dev->cmdline_power_refcount, 0);
- rknpu_power_put(rknpu_dev);
- }
- if (atomic_read(&rknpu_dev->power_refcount) <= 0)
- LOG_INFO("rknpu power is off!");
- } else {
- LOG_ERROR("rknpu power node params is invalid!");
- }
-
- return len;
-}
-
-static int rknpu_power_put_delay_show(struct seq_file *m, void *data)
-{
- struct rknpu_debugger_node *node = m->private;
- struct rknpu_debugger *debugger = node->debugger;
- struct rknpu_device *rknpu_dev =
- container_of(debugger, struct rknpu_device, debugger);
-
- seq_printf(m, "%lu\n", rknpu_dev->power_put_delay);
-
- return 0;
-}
-
-static ssize_t rknpu_power_put_delay_set(struct file *file,
- const char __user *ubuf, size_t len,
- loff_t *offp)
-{
- struct seq_file *priv = file->private_data;
- struct rknpu_debugger_node *node = priv->private;
- struct rknpu_debugger *debugger = node->debugger;
- struct rknpu_device *rknpu_dev =
- container_of(debugger, struct rknpu_device, debugger);
- char buf[16];
- unsigned long power_put_delay = 0;
- int ret = 0;
-
- if (len > sizeof(buf) - 1)
- return -EINVAL;
- if (copy_from_user(buf, ubuf, len))
- return -EFAULT;
- buf[len - 1] = '\0';
-
- ret = kstrtoul(buf, 10, &power_put_delay);
- if (ret) {
- LOG_ERROR("failed to parse power put delay string: %s\n", buf);
- return -EFAULT;
- }
-
- rknpu_dev->power_put_delay = power_put_delay;
-
- LOG_INFO("set rknpu power put delay time %lums\n",
- rknpu_dev->power_put_delay);
-
- return len;
-}
-
-static int rknpu_freq_show(struct seq_file *m, void *data)
-{
- struct rknpu_debugger_node *node = m->private;
- struct rknpu_debugger *debugger = node->debugger;
- struct rknpu_device *rknpu_dev =
- container_of(debugger, struct rknpu_device, debugger);
- unsigned long current_freq = 0;
-
- rknpu_power_get(rknpu_dev);
-
- current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
-
- rknpu_power_put(rknpu_dev);
-
- seq_printf(m, "%lu\n", current_freq);
-
- return 0;
-}
-
-#ifdef CONFIG_PM_DEVFREQ
-static ssize_t rknpu_freq_set(struct file *file, const char __user *ubuf,
- size_t len, loff_t *offp)
-{
- struct seq_file *priv = file->private_data;
- struct rknpu_debugger_node *node = priv->private;
- struct rknpu_debugger *debugger = node->debugger;
- struct rknpu_device *rknpu_dev =
- container_of(debugger, struct rknpu_device, debugger);
- unsigned long current_freq = 0;
- char buf[16];
- unsigned long freq = 0;
- int ret = 0;
-
- if (len > sizeof(buf) - 1)
- return -EINVAL;
- if (copy_from_user(buf, ubuf, len))
- return -EFAULT;
- buf[len - 1] = '\0';
-
- ret = kstrtoul(buf, 10, &freq);
- if (ret) {
- LOG_ERROR("failed to parse freq string: %s\n", buf);
- return -EFAULT;
- }
-
- if (!rknpu_dev->devfreq)
- return -EFAULT;
-
- rknpu_power_get(rknpu_dev);
-
- current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
- if (freq != current_freq) {
- rknpu_dev->ondemand_freq = freq;
- mutex_lock(&rknpu_dev->devfreq->lock);
- update_devfreq(rknpu_dev->devfreq);
- mutex_unlock(&rknpu_dev->devfreq->lock);
- }
-
- rknpu_power_put(rknpu_dev);
-
- return len;
-}
-#else
-static ssize_t rknpu_freq_set(struct file *file, const char __user *ubuf,
- size_t len, loff_t *offp)
-{
- return -EFAULT;
-}
-#endif
-
-static int rknpu_volt_show(struct seq_file *m, void *data)
-{
- struct rknpu_debugger_node *node = m->private;
- struct rknpu_debugger *debugger = node->debugger;
- struct rknpu_device *rknpu_dev =
- container_of(debugger, struct rknpu_device, debugger);
- unsigned long current_volt = 0;
-
- current_volt = regulator_get_voltage(rknpu_dev->vdd);
-
- seq_printf(m, "%lu\n", current_volt);
-
- return 0;
-}
-
-static int rknpu_reset_show(struct seq_file *m, void *data)
-{
- struct rknpu_debugger_node *node = m->private;
- struct rknpu_debugger *debugger = node->debugger;
- struct rknpu_device *rknpu_dev =
- container_of(debugger, struct rknpu_device, debugger);
-
- if (!rknpu_dev->bypass_soft_reset)
- seq_puts(m, "on\n");
- else
- seq_puts(m, "off\n");
-
- return 0;
-}
-
-static ssize_t rknpu_reset_set(struct file *file, const char __user *ubuf,
- size_t len, loff_t *offp)
-{
- struct seq_file *priv = file->private_data;
- struct rknpu_debugger_node *node = priv->private;
- struct rknpu_debugger *debugger = node->debugger;
- struct rknpu_device *rknpu_dev =
- container_of(debugger, struct rknpu_device, debugger);
- char buf[8];
-
- if (len > sizeof(buf) - 1)
- return -EINVAL;
- if (copy_from_user(buf, ubuf, len))
- return -EFAULT;
- buf[len - 1] = '\0';
-
- if (strcmp(buf, "1") == 0 &&
- atomic_read(&rknpu_dev->power_refcount) > 0)
- rknpu_soft_reset(rknpu_dev);
- else if (strcmp(buf, "on") == 0)
- rknpu_dev->bypass_soft_reset = 0;
- else if (strcmp(buf, "off") == 0)
- rknpu_dev->bypass_soft_reset = 1;
-
- return len;
-}
-
-static struct rknpu_debugger_list rknpu_debugger_root_list[] = {
- { "version", rknpu_version_show, NULL, NULL },
- { "load", rknpu_load_show, NULL, NULL },
- { "power", rknpu_power_show, rknpu_power_set, NULL },
- { "freq", rknpu_freq_show, rknpu_freq_set, NULL },
- { "volt", rknpu_volt_show, NULL, NULL },
- { "delayms", rknpu_power_put_delay_show, rknpu_power_put_delay_set,
- NULL },
- { "reset", rknpu_reset_show, rknpu_reset_set, NULL },
-#ifdef CONFIG_ROCKCHIP_RKNPU_SRAM
- { "mm", rknpu_mm_dump, NULL, NULL },
-#endif
-};
-
-static ssize_t rknpu_debugger_write(struct file *file, const char __user *ubuf,
- size_t len, loff_t *offp)
-{
- struct seq_file *priv = file->private_data;
- struct rknpu_debugger_node *node = priv->private;
-
- if (node->info_ent->write)
- return node->info_ent->write(file, ubuf, len, offp);
- else
- return len;
-}
-
-static int rknpu_debugfs_open(struct inode *inode, struct file *file)
-{
- struct rknpu_debugger_node *node = inode->i_private;
-
- return single_open(file, node->info_ent->show, node);
-}
-
-static const struct file_operations rknpu_debugfs_fops = {
- .owner = THIS_MODULE,
- .open = rknpu_debugfs_open,
- .read = seq_read,
- .llseek = seq_lseek,
- .release = single_release,
- .write = rknpu_debugger_write,
-};
-#endif /* #if defined(CONFIG_ROCKCHIP_RKNPU_DEBUG_FS) || defined(CONFIG_ROCKCHIP_RKNPU_PROC_FS) */
-
-#ifdef CONFIG_ROCKCHIP_RKNPU_DEBUG_FS
-static int rknpu_debugfs_remove_files(struct rknpu_debugger *debugger)
-{
- struct rknpu_debugger_node *pos, *q;
- struct list_head *entry_list;
-
- mutex_lock(&debugger->debugfs_lock);
-
- /* Delete debugfs entry list */
- entry_list = &debugger->debugfs_entry_list;
- list_for_each_entry_safe(pos, q, entry_list, list) {
- if (pos->dent == NULL)
- continue;
- list_del(&pos->list);
- kfree(pos);
- pos = NULL;
- }
-
- /* Delete all debugfs node in this directory */
- debugfs_remove_recursive(debugger->debugfs_dir);
- debugger->debugfs_dir = NULL;
-
- mutex_unlock(&debugger->debugfs_lock);
-
- return 0;
-}
-
-static int rknpu_debugfs_create_files(const struct rknpu_debugger_list *files,
- int count, struct dentry *root,
- struct rknpu_debugger *debugger)
-{
- int i;
- struct dentry *ent;
- struct rknpu_debugger_node *tmp;
-
- for (i = 0; i < count; i++) {
- tmp = kmalloc(sizeof(struct rknpu_debugger_node), GFP_KERNEL);
- if (tmp == NULL) {
- LOG_ERROR(
- "Cannot alloc node path /sys/kernel/debug/%pd/%s\n",
- root, files[i].name);
- goto MALLOC_FAIL;
- }
-
- tmp->info_ent = &files[i];
- tmp->debugger = debugger;
-
- ent = debugfs_create_file(files[i].name, S_IFREG | S_IRUGO,
- root, tmp, &rknpu_debugfs_fops);
- if (!ent) {
- LOG_ERROR("Cannot create /sys/kernel/debug/%pd/%s\n",
- root, files[i].name);
- goto CREATE_FAIL;
- }
-
- tmp->dent = ent;
-
- mutex_lock(&debugger->debugfs_lock);
- list_add_tail(&tmp->list, &debugger->debugfs_entry_list);
- mutex_unlock(&debugger->debugfs_lock);
- }
-
- return 0;
-
-CREATE_FAIL:
- kfree(tmp);
-MALLOC_FAIL:
- rknpu_debugfs_remove_files(debugger);
-
- return -1;
-}
-
-static int rknpu_debugfs_remove(struct rknpu_debugger *debugger)
-{
- rknpu_debugfs_remove_files(debugger);
-
- return 0;
-}
-
-static int rknpu_debugfs_init(struct rknpu_debugger *debugger)
-{
- int ret;
-
- debugger->debugfs_dir =
- debugfs_create_dir(RKNPU_DEBUGGER_ROOT_NAME, NULL);
- if (IS_ERR_OR_NULL(debugger->debugfs_dir)) {
- LOG_ERROR("failed on mkdir /sys/kernel/debug/%s\n",
- RKNPU_DEBUGGER_ROOT_NAME);
- debugger->debugfs_dir = NULL;
- return -EIO;
- }
-
- ret = rknpu_debugfs_create_files(rknpu_debugger_root_list,
- ARRAY_SIZE(rknpu_debugger_root_list),
- debugger->debugfs_dir, debugger);
- if (ret) {
- LOG_ERROR(
- "Could not install rknpu_debugger_root_list debugfs\n");
- goto CREATE_FAIL;
- }
-
- return 0;
-
-CREATE_FAIL:
- rknpu_debugfs_remove(debugger);
-
- return ret;
-}
-#endif /* #ifdef CONFIG_ROCKCHIP_RKNPU_DEBUG_FS */
-
-#ifdef CONFIG_ROCKCHIP_RKNPU_PROC_FS
-static int rknpu_procfs_open(struct inode *inode, struct file *file)
-{
-#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
- struct rknpu_debugger_node *node = PDE_DATA(inode);
-#else
- struct rknpu_debugger_node *node = pde_data(inode);
-#endif
-
- return single_open(file, node->info_ent->show, node);
-}
-
-static const struct proc_ops rknpu_procfs_fops = {
- .proc_open = rknpu_procfs_open,
- .proc_read = seq_read,
- .proc_lseek = seq_lseek,
- .proc_release = single_release,
- .proc_write = rknpu_debugger_write,
-};
-
-static int rknpu_procfs_remove_files(struct rknpu_debugger *debugger)
-{
- struct rknpu_debugger_node *pos, *q;
- struct list_head *entry_list;
-
- mutex_lock(&debugger->procfs_lock);
-
- /* Delete procfs entry list */
- entry_list = &debugger->procfs_entry_list;
- list_for_each_entry_safe(pos, q, entry_list, list) {
- if (pos->pent == NULL)
- continue;
- list_del(&pos->list);
- kfree(pos);
- pos = NULL;
- }
-
- /* Delete all procfs node in this directory */
- proc_remove(debugger->procfs_dir);
- debugger->procfs_dir = NULL;
-
- mutex_unlock(&debugger->procfs_lock);
-
- return 0;
-}
-
-static int rknpu_procfs_create_files(const struct rknpu_debugger_list *files,
- int count, struct proc_dir_entry *root,
- struct rknpu_debugger *debugger)
-{
- int i;
- struct proc_dir_entry *ent;
- struct rknpu_debugger_node *tmp;
-
- for (i = 0; i < count; i++) {
- tmp = kmalloc(sizeof(struct rknpu_debugger_node), GFP_KERNEL);
- if (tmp == NULL) {
- LOG_ERROR("Cannot alloc node path for /proc/%s/%s\n",
- RKNPU_DEBUGGER_ROOT_NAME, files[i].name);
- goto MALLOC_FAIL;
- }
-
- tmp->info_ent = &files[i];
- tmp->debugger = debugger;
-
- ent = proc_create_data(files[i].name, S_IFREG | S_IRUGO, root,
- &rknpu_procfs_fops, tmp);
- if (!ent) {
- LOG_ERROR("Cannot create /proc/%s/%s\n",
- RKNPU_DEBUGGER_ROOT_NAME, files[i].name);
- goto CREATE_FAIL;
- }
-
- tmp->pent = ent;
-
- mutex_lock(&debugger->procfs_lock);
- list_add_tail(&tmp->list, &debugger->procfs_entry_list);
- mutex_unlock(&debugger->procfs_lock);
- }
-
- return 0;
-
-CREATE_FAIL:
- kfree(tmp);
-MALLOC_FAIL:
- rknpu_procfs_remove_files(debugger);
- return -1;
-}
-
-static int rknpu_procfs_remove(struct rknpu_debugger *debugger)
-{
- rknpu_procfs_remove_files(debugger);
-
- return 0;
-}
-
-static int rknpu_procfs_init(struct rknpu_debugger *debugger)
-{
- int ret;
-
- debugger->procfs_dir = proc_mkdir(RKNPU_DEBUGGER_ROOT_NAME, NULL);
- if (IS_ERR_OR_NULL(debugger->procfs_dir)) {
- pr_err("failed on mkdir /proc/%s\n", RKNPU_DEBUGGER_ROOT_NAME);
- debugger->procfs_dir = NULL;
- return -EIO;
- }
-
- ret = rknpu_procfs_create_files(rknpu_debugger_root_list,
- ARRAY_SIZE(rknpu_debugger_root_list),
- debugger->procfs_dir, debugger);
- if (ret) {
- pr_err("Could not install rknpu_debugger_root_list procfs\n");
- goto CREATE_FAIL;
- }
-
- return 0;
-
-CREATE_FAIL:
- rknpu_procfs_remove(debugger);
-
- return ret;
-}
-#endif /* #ifdef CONFIG_ROCKCHIP_RKNPU_PROC_FS */
-
-int rknpu_debugger_init(struct rknpu_device *rknpu_dev)
-{
-#ifdef CONFIG_ROCKCHIP_RKNPU_DEBUG_FS
- mutex_init(&rknpu_dev->debugger.debugfs_lock);
- INIT_LIST_HEAD(&rknpu_dev->debugger.debugfs_entry_list);
- rknpu_debugfs_init(&rknpu_dev->debugger);
-#endif
-#ifdef CONFIG_ROCKCHIP_RKNPU_PROC_FS
- mutex_init(&rknpu_dev->debugger.procfs_lock);
- INIT_LIST_HEAD(&rknpu_dev->debugger.procfs_entry_list);
- rknpu_procfs_init(&rknpu_dev->debugger);
-#endif
- return 0;
-}
-
-int rknpu_debugger_remove(struct rknpu_device *rknpu_dev)
-{
-#ifdef CONFIG_ROCKCHIP_RKNPU_DEBUG_FS
- rknpu_debugfs_remove(&rknpu_dev->debugger);
-#endif
-#ifdef CONFIG_ROCKCHIP_RKNPU_PROC_FS
- rknpu_procfs_remove(&rknpu_dev->debugger);
-#endif
- return 0;
-}
diff --git a/rknpu-driver/driver-0.9.6/rknpu_devfreq.c b/rknpu-driver/driver-0.9.6/rknpu_devfreq.c
deleted file mode 100644
index 1fe98d5..0000000
--- a/rknpu-driver/driver-0.9.6/rknpu_devfreq.c
+++ /dev/null
@@ -1,795 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) Rockchip Electronics Co.Ltd
- * Author: Finley Xiao
- */
-
-#include
-#include
-#include
-#include
-#include
-#include <../drivers/devfreq/governor.h>
-#include "rknpu_drv.h"
-#include "rknpu_devfreq.h"
-
-#define POWER_DOWN_FREQ 200000000
-
-static int npu_devfreq_target(struct device *dev, unsigned long *freq,
- u32 flags);
-
-static struct monitor_dev_profile npu_mdevp = {
- .type = MONITOR_TYPE_DEV,
- .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust,
- .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust,
-#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
- .check_rate_volt = rockchip_monitor_check_rate_volt,
-#else
- .update_volt = rockchip_monitor_check_rate_volt,
-#endif
-};
-
-static int npu_devfreq_get_dev_status(struct device *dev,
- struct devfreq_dev_status *stat)
-{
- return 0;
-}
-
-static int npu_devfreq_get_cur_freq(struct device *dev, unsigned long *freq)
-{
- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
-
- *freq = rknpu_dev->current_freq;
-
- return 0;
-}
-
-static struct devfreq_dev_profile npu_devfreq_profile = {
- .polling_ms = 50,
- .target = npu_devfreq_target,
- .get_dev_status = npu_devfreq_get_dev_status,
- .get_cur_freq = npu_devfreq_get_cur_freq,
-};
-
-static int devfreq_rknpu_ondemand_func(struct devfreq *df, unsigned long *freq)
-{
- struct rknpu_device *rknpu_dev = df->data;
-
- if (rknpu_dev && rknpu_dev->ondemand_freq)
- *freq = rknpu_dev->ondemand_freq;
- else
- *freq = df->previous_freq;
-
- return 0;
-}
-
-static int devfreq_rknpu_ondemand_handler(struct devfreq *devfreq,
- unsigned int event, void *data)
-{
- return 0;
-}
-
-static struct devfreq_governor devfreq_rknpu_ondemand = {
- .name = "rknpu_ondemand",
- .get_target_freq = devfreq_rknpu_ondemand_func,
- .event_handler = devfreq_rknpu_ondemand_handler,
-};
-
-static int rk3576_npu_set_read_margin(struct device *dev,
- struct rockchip_opp_info *opp_info,
- u32 rm)
-{
- if (!opp_info->grf || !opp_info->volt_rm_tbl)
- return 0;
-
- if (rm == opp_info->current_rm || rm == UINT_MAX)
- return 0;
-
- LOG_DEV_DEBUG(dev, "set rm to %d\n", rm);
-
- regmap_write(opp_info->grf, 0x08, 0x001c0000 | (rm << 2));
- regmap_write(opp_info->grf, 0x0c, 0x003c0000 | (rm << 2));
- regmap_write(opp_info->grf, 0x10, 0x001c0000 | (rm << 2));
-
- return 0;
-}
-
-static int rk3588_npu_get_soc_info(struct device *dev, struct device_node *np,
- int *bin, int *process)
-{
- int ret = 0;
- u8 value = 0;
-
- if (!bin)
- return 0;
-
- if (of_property_match_string(np, "nvmem-cell-names",
- "specification_serial_number") >= 0) {
- ret = rockchip_nvmem_cell_read_u8(
- np, "specification_serial_number", &value);
- if (ret) {
- LOG_DEV_ERROR(
- dev,
- "Failed to get specification_serial_number\n");
- return ret;
- }
- /* RK3588M */
- if (value == 0xd)
- *bin = 1;
- /* RK3588J */
- else if (value == 0xa)
- *bin = 2;
- }
- if (*bin < 0)
- *bin = 0;
- LOG_DEV_INFO(dev, "bin=%d\n", *bin);
-
- return ret;
-}
-
-#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
-static int rk3588_npu_set_soc_info(struct device *dev, struct device_node *np,
- struct rockchip_opp_info *opp_info)
-{
- int bin = opp_info->bin;
-
- if (opp_info->volt_sel < 0)
- return 0;
- if (bin < 0)
- bin = 0;
-
- if (!of_property_read_bool(np, "rockchip,supported-hw"))
- return 0;
-
- /* SoC Version */
- opp_info->supported_hw[0] = BIT(bin);
- /* Speed Grade */
- opp_info->supported_hw[1] = BIT(opp_info->volt_sel);
-
- return 0;
-}
-#else
-static int rk3588_npu_set_soc_info(struct device *dev, struct device_node *np,
- int bin, int process, int volt_sel)
-{
- struct opp_table *opp_table;
- u32 supported_hw[2];
-
- if (volt_sel < 0)
- return 0;
- if (bin < 0)
- bin = 0;
-
- if (!of_property_read_bool(np, "rockchip,supported-hw"))
- return 0;
-
- /* SoC Version */
- supported_hw[0] = BIT(bin);
- /* Speed Grade */
- supported_hw[1] = BIT(volt_sel);
- opp_table = dev_pm_opp_set_supported_hw(dev, supported_hw, 2);
- if (IS_ERR(opp_table)) {
- LOG_DEV_ERROR(dev, "failed to set supported opp\n");
- return PTR_ERR(opp_table);
- }
-
- return 0;
-}
-#endif
-
-static int rk3588_npu_set_read_margin(struct device *dev,
- struct rockchip_opp_info *opp_info,
- u32 rm)
-{
- u32 offset = 0, val = 0;
- int i, ret = 0;
-
- if (!opp_info->grf || !opp_info->volt_rm_tbl)
- return 0;
-
- if (rm == opp_info->current_rm || rm == UINT_MAX)
- return 0;
-
- LOG_DEV_DEBUG(dev, "set rm to %d\n", rm);
-
- for (i = 0; i < 3; i++) {
- ret = regmap_read(opp_info->grf, offset, &val);
- if (ret < 0) {
- LOG_DEV_ERROR(dev, "failed to get rm from 0x%x\n",
- offset);
- return ret;
- }
- val &= ~0x1c;
- regmap_write(opp_info->grf, offset, val | (rm << 2));
- offset += 4;
- }
- opp_info->current_rm = rm;
-
- return 0;
-}
-
-#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
-static int npu_opp_config_regulators(struct device *dev,
- struct dev_pm_opp *old_opp,
- struct dev_pm_opp *new_opp,
- struct regulator **regulators,
- unsigned int count)
-{
- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
-
- return rockchip_opp_config_regulators(dev, old_opp, new_opp, regulators,
- count, &rknpu_dev->opp_info);
-}
-
-static int npu_opp_config_clks(struct device *dev, struct opp_table *opp_table,
- struct dev_pm_opp *opp, void *data,
- bool scaling_down)
-{
- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
-
- return rockchip_opp_config_clks(dev, opp_table, opp, data, scaling_down,
- &rknpu_dev->opp_info);
-}
-#endif
-
-static const struct rockchip_opp_data rk3576_npu_opp_data = {
- .set_read_margin = rk3576_npu_set_read_margin,
-#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
- .config_regulators = npu_opp_config_regulators,
- .config_clks = npu_opp_config_clks,
-#endif
-};
-
-static const struct rockchip_opp_data rk3588_npu_opp_data = {
- .get_soc_info = rk3588_npu_get_soc_info,
- .set_soc_info = rk3588_npu_set_soc_info,
- .set_read_margin = rk3588_npu_set_read_margin,
-#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
- .config_regulators = npu_opp_config_regulators,
- .config_clks = npu_opp_config_clks,
-#endif
-};
-
-static const struct of_device_id rockchip_npu_of_match[] = {
- {
- .compatible = "rockchip,rk3576",
- .data = (void *)&rk3576_npu_opp_data,
- },
- {
- .compatible = "rockchip,rk3588",
- .data = (void *)&rk3588_npu_opp_data,
- },
- {},
-};
-
-#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
-void rknpu_devfreq_lock(struct rknpu_device *rknpu_dev)
-{
- if (rknpu_dev->devfreq)
- rockchip_opp_dvfs_lock(&rknpu_dev->opp_info);
-}
-EXPORT_SYMBOL(rknpu_devfreq_lock);
-
-void rknpu_devfreq_unlock(struct rknpu_device *rknpu_dev)
-{
- if (rknpu_dev->devfreq)
- rockchip_opp_dvfs_unlock(&rknpu_dev->opp_info);
-}
-EXPORT_SYMBOL(rknpu_devfreq_unlock);
-
-static int npu_devfreq_target(struct device *dev, unsigned long *freq,
- u32 flags)
-{
- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
- struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
- struct dev_pm_opp *opp;
- unsigned long opp_volt;
- int ret = 0;
-
- if (!opp_info->is_rate_volt_checked)
- return -EINVAL;
-
- opp = devfreq_recommended_opp(dev, freq, flags);
- if (IS_ERR(opp))
- return PTR_ERR(opp);
- opp_volt = dev_pm_opp_get_voltage(opp);
- dev_pm_opp_put(opp);
-
- if (*freq == rknpu_dev->current_freq)
- return 0;
-
- rockchip_opp_dvfs_lock(opp_info);
- if (pm_runtime_active(dev))
- opp_info->is_runtime_active = true;
- else
- opp_info->is_runtime_active = false;
- ret = dev_pm_opp_set_rate(dev, *freq);
- if (!ret) {
- rknpu_dev->current_freq = *freq;
- if (rknpu_dev->devfreq)
- rknpu_dev->devfreq->last_status.current_frequency =
- *freq;
- rknpu_dev->current_volt = opp_volt;
- LOG_DEV_DEBUG(dev, "set rknpu freq: %lu, volt: %lu\n",
- rknpu_dev->current_freq, rknpu_dev->current_volt);
- }
- rockchip_opp_dvfs_unlock(opp_info);
-
- return ret;
-}
-
-static const struct rockchip_opp_data rockchip_npu_opp_data = {
- .config_clks = npu_opp_config_clks,
-};
-
-int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
-{
- struct rockchip_opp_info *info = &rknpu_dev->opp_info;
- struct device *dev = rknpu_dev->dev;
- struct devfreq_dev_profile *dp;
- struct dev_pm_opp *opp;
- unsigned int dyn_power_coeff = 0;
- int ret = 0;
-
- info->data = &rockchip_npu_opp_data;
- rockchip_get_opp_data(rockchip_npu_of_match, info);
- ret = rockchip_init_opp_table(dev, info, "clk_npu", "rknpu");
- if (ret) {
- LOG_DEV_ERROR(dev, "failed to init_opp_table\n");
- return -EINVAL;
- }
-
- rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
- opp = devfreq_recommended_opp(dev, &rknpu_dev->current_freq, 0);
- if (IS_ERR(opp)) {
- ret = PTR_ERR(opp);
- goto err_uinit_table;
- }
- dev_pm_opp_put(opp);
-
- dp = &npu_devfreq_profile;
- dp->initial_freq = rknpu_dev->current_freq;
- of_property_read_u32(dev->of_node, "dynamic-power-coefficient",
- &dyn_power_coeff);
- if (dyn_power_coeff)
- dp->is_cooling_device = true;
-
- ret = devfreq_add_governor(&devfreq_rknpu_ondemand);
- if (ret) {
- LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n");
- goto err_uinit_table;
- }
-
- rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand",
- (void *)rknpu_dev);
- if (IS_ERR(rknpu_dev->devfreq)) {
- LOG_DEV_ERROR(dev, "failed to add devfreq\n");
- ret = PTR_ERR(rknpu_dev->devfreq);
- rknpu_dev->devfreq = NULL;
- goto err_remove_governor;
- }
-
- npu_mdevp.data = rknpu_dev->devfreq;
- npu_mdevp.opp_info = &rknpu_dev->opp_info;
- rknpu_dev->mdev_info =
- rockchip_system_monitor_register(dev, &npu_mdevp);
- if (IS_ERR(rknpu_dev->mdev_info)) {
- dev_dbg(dev, "without system monitor\n");
- rknpu_dev->mdev_info = NULL;
- }
-
- rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
- rknpu_dev->ondemand_freq = rknpu_dev->current_freq;
- rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd);
-
- rknpu_dev->devfreq->previous_freq = rknpu_dev->current_freq;
- if (rknpu_dev->devfreq->suspend_freq)
- rknpu_dev->devfreq->resume_freq = rknpu_dev->current_freq;
- rknpu_dev->devfreq->last_status.current_frequency =
- rknpu_dev->current_freq;
- rknpu_dev->devfreq->last_status.total_time = 1;
- rknpu_dev->devfreq->last_status.busy_time = 1;
-
- return 0;
-
-err_remove_governor:
- devfreq_remove_governor(&devfreq_rknpu_ondemand);
-err_uinit_table:
- rockchip_uninit_opp_table(dev, info);
-
- return ret;
-}
-EXPORT_SYMBOL(rknpu_devfreq_init);
-
-int rknpu_devfreq_runtime_suspend(struct device *dev)
-{
- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
- struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
-
- if (opp_info->is_scmi_clk) {
- if (clk_set_rate(opp_info->clk, POWER_DOWN_FREQ))
- LOG_DEV_ERROR(dev, "failed to restore clk rate\n");
- }
- opp_info->current_rm = UINT_MAX;
-
- return 0;
-}
-EXPORT_SYMBOL(rknpu_devfreq_runtime_suspend);
-
-int rknpu_devfreq_runtime_resume(struct device *dev)
-{
- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
- struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
- int ret = 0;
-
- if (!rknpu_dev->current_freq || !rknpu_dev->current_volt)
- return 0;
-
- ret = clk_bulk_prepare_enable(opp_info->nclocks, opp_info->clocks);
- if (ret) {
- LOG_DEV_INFO(dev, "failed to enable opp clks\n");
- return ret;
- }
-
- if (opp_info->data && opp_info->data->set_read_margin)
- opp_info->data->set_read_margin(dev, opp_info,
- opp_info->target_rm);
- if (opp_info->is_scmi_clk) {
- if (clk_set_rate(opp_info->clk, rknpu_dev->current_freq))
- LOG_DEV_ERROR(dev, "failed to set power down rate\n");
- }
-
- clk_bulk_disable_unprepare(opp_info->nclocks, opp_info->clocks);
-
- return ret;
-}
-EXPORT_SYMBOL(rknpu_devfreq_runtime_resume);
-#else
-void rknpu_devfreq_lock(struct rknpu_device *rknpu_dev)
-{
- rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
-}
-EXPORT_SYMBOL(rknpu_devfreq_lock);
-
-void rknpu_devfreq_unlock(struct rknpu_device *rknpu_dev)
-{
- rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
-}
-EXPORT_SYMBOL(rknpu_devfreq_unlock);
-
-static int npu_opp_helper(struct dev_pm_set_opp_data *data)
-{
- struct device *dev = data->dev;
- struct dev_pm_opp_supply *old_supply_vdd = &data->old_opp.supplies[0];
- struct dev_pm_opp_supply *old_supply_mem = &data->old_opp.supplies[1];
- struct dev_pm_opp_supply *new_supply_vdd = &data->new_opp.supplies[0];
- struct dev_pm_opp_supply *new_supply_mem = &data->new_opp.supplies[1];
- struct regulator *vdd_reg = data->regulators[0];
- struct regulator *mem_reg = data->regulators[1];
- struct clk *clk = data->clk;
- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
- struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
- unsigned long old_freq = data->old_opp.rate;
- unsigned long new_freq = data->new_opp.rate;
- bool is_set_rm = true;
- bool is_set_clk = true;
- u32 target_rm = UINT_MAX;
- int ret = 0;
-
- if (!pm_runtime_active(dev)) {
- is_set_rm = false;
- if (opp_info->scmi_clk)
- is_set_clk = false;
- }
-
- ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks);
- if (ret < 0) {
- LOG_DEV_ERROR(dev, "failed to enable opp clks\n");
- return ret;
- }
- rockchip_get_read_margin(dev, opp_info, new_supply_vdd->u_volt,
- &target_rm);
-
- /* Change frequency */
- LOG_DEV_DEBUG(dev, "switching OPP: %lu Hz --> %lu Hz\n", old_freq,
- new_freq);
- /* Scaling up? Scale voltage before frequency */
- if (new_freq >= old_freq) {
- rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq,
- new_freq, true, is_set_clk);
- ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt,
- INT_MAX);
- if (ret) {
- LOG_DEV_ERROR(dev,
- "failed to set volt %lu uV for mem reg\n",
- new_supply_mem->u_volt);
- goto restore_voltage;
- }
- ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt,
- INT_MAX);
- if (ret) {
- LOG_DEV_ERROR(dev,
- "failed to set volt %lu uV for vdd reg\n",
- new_supply_vdd->u_volt);
- goto restore_voltage;
- }
- rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm);
- if (is_set_clk && clk_set_rate(clk, new_freq)) {
- ret = -EINVAL;
- LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret);
- goto restore_rm;
- }
- /* Scaling down? Scale voltage after frequency */
- } else {
- rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq,
- new_freq, false, is_set_clk);
- rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm);
- if (is_set_clk && clk_set_rate(clk, new_freq)) {
- ret = -EINVAL;
- LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret);
- goto restore_rm;
- }
- ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt,
- INT_MAX);
- if (ret) {
- LOG_DEV_ERROR(dev,
- "failed to set volt %lu uV for vdd reg\n",
- new_supply_vdd->u_volt);
- goto restore_freq;
- }
- ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt,
- INT_MAX);
- if (ret) {
- LOG_DEV_ERROR(dev,
- "failed to set volt %lu uV for mem reg\n",
- new_supply_mem->u_volt);
- goto restore_freq;
- }
- }
-
- clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
-
- return 0;
-
-restore_freq:
- if (is_set_clk && clk_set_rate(clk, old_freq))
- LOG_DEV_ERROR(dev, "failed to restore old-freq %lu Hz\n",
- old_freq);
-restore_rm:
- rockchip_get_read_margin(dev, opp_info, old_supply_vdd->u_volt,
- &target_rm);
- rockchip_set_read_margin(dev, opp_info, opp_info->current_rm,
- is_set_rm);
-restore_voltage:
- regulator_set_voltage(mem_reg, old_supply_mem->u_volt, INT_MAX);
- regulator_set_voltage(vdd_reg, old_supply_vdd->u_volt, INT_MAX);
- clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
-
- return ret;
-}
-
-static int npu_devfreq_target(struct device *dev, unsigned long *freq,
- u32 flags)
-{
- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
- struct dev_pm_opp *opp;
- unsigned long opp_volt;
- int ret = 0;
-
- if (!npu_mdevp.is_checked)
- return -EINVAL;
-
- opp = devfreq_recommended_opp(dev, freq, flags);
- if (IS_ERR(opp))
- return PTR_ERR(opp);
- opp_volt = dev_pm_opp_get_voltage(opp);
- dev_pm_opp_put(opp);
-
- rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
-
- ret = dev_pm_opp_set_rate(dev, *freq);
- if (!ret) {
- rknpu_dev->current_freq = *freq;
- if (rknpu_dev->devfreq)
- rknpu_dev->devfreq->last_status.current_frequency =
- *freq;
- rknpu_dev->current_volt = opp_volt;
- LOG_DEV_DEBUG(dev, "set rknpu freq: %lu, volt: %lu\n",
- rknpu_dev->current_freq, rknpu_dev->current_volt);
- }
-
- rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
-
- return ret;
-}
-
-static unsigned long npu_get_static_power(struct devfreq *devfreq,
- unsigned long voltage)
-{
- struct device *dev = devfreq->dev.parent;
- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
-
- if (!rknpu_dev->model_data)
- return 0;
-
- return rockchip_ipa_get_static_power(rknpu_dev->model_data, voltage);
-}
-
-static struct devfreq_cooling_power npu_cooling_power = {
- .get_static_power = &npu_get_static_power,
-};
-
-int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
-{
- struct device *dev = rknpu_dev->dev;
- struct devfreq_dev_profile *dp = &npu_devfreq_profile;
- struct dev_pm_opp *opp;
- struct opp_table *reg_table = NULL;
- struct opp_table *opp_table = NULL;
- const char *const reg_names[] = { "rknpu", "mem" };
- int ret = -EINVAL;
-
- if (strstr(__clk_get_name(rknpu_dev->clks[0].clk), "scmi"))
- rknpu_dev->opp_info.scmi_clk = rknpu_dev->clks[0].clk;
-
- if (of_find_property(dev->of_node, "rknpu-supply", NULL) &&
- of_find_property(dev->of_node, "mem-supply", NULL)) {
- reg_table = dev_pm_opp_set_regulators(dev, reg_names, 2);
- if (IS_ERR(reg_table))
- return PTR_ERR(reg_table);
- opp_table =
- dev_pm_opp_register_set_opp_helper(dev, npu_opp_helper);
- if (IS_ERR(opp_table)) {
- dev_pm_opp_put_regulators(reg_table);
- return PTR_ERR(opp_table);
- }
- } else {
- reg_table = dev_pm_opp_set_regulators(dev, reg_names, 1);
- if (IS_ERR(reg_table))
- return PTR_ERR(reg_table);
- }
-
- rockchip_get_opp_data(rockchip_npu_of_match, &rknpu_dev->opp_info);
- ret = rockchip_init_opp_table(dev, &rknpu_dev->opp_info, "npu_leakage",
- "rknpu");
- if (ret) {
- LOG_DEV_ERROR(dev, "failed to init_opp_table\n");
- return ret;
- }
-
- rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
-
- opp = devfreq_recommended_opp(dev, &rknpu_dev->current_freq, 0);
- if (IS_ERR(opp)) {
- ret = PTR_ERR(opp);
- goto err_remove_table;
- }
- dev_pm_opp_put(opp);
- dp->initial_freq = rknpu_dev->current_freq;
-
- ret = devfreq_add_governor(&devfreq_rknpu_ondemand);
- if (ret) {
- LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n");
- goto err_remove_table;
- }
-
- rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand",
- (void *)rknpu_dev);
- if (IS_ERR(rknpu_dev->devfreq)) {
- LOG_DEV_ERROR(dev, "failed to add devfreq\n");
- ret = PTR_ERR(rknpu_dev->devfreq);
- goto err_remove_governor;
- }
-
- npu_mdevp.data = rknpu_dev->devfreq;
- npu_mdevp.opp_info = &rknpu_dev->opp_info;
- rknpu_dev->mdev_info =
- rockchip_system_monitor_register(dev, &npu_mdevp);
- if (IS_ERR(rknpu_dev->mdev_info)) {
- LOG_DEV_DEBUG(dev, "without system monitor\n");
- rknpu_dev->mdev_info = NULL;
- npu_mdevp.is_checked = true;
- }
- rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
- rknpu_dev->ondemand_freq = rknpu_dev->current_freq;
- rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd);
-
- rknpu_dev->devfreq->previous_freq = rknpu_dev->current_freq;
- if (rknpu_dev->devfreq->suspend_freq)
- rknpu_dev->devfreq->resume_freq = rknpu_dev->current_freq;
- rknpu_dev->devfreq->last_status.current_frequency =
- rknpu_dev->current_freq;
- rknpu_dev->devfreq->last_status.total_time = 1;
- rknpu_dev->devfreq->last_status.busy_time = 1;
-
- of_property_read_u32(dev->of_node, "dynamic-power-coefficient",
- (u32 *)&npu_cooling_power.dyn_power_coeff);
- rknpu_dev->model_data =
- rockchip_ipa_power_model_init(dev, "npu_leakage");
- if (IS_ERR_OR_NULL(rknpu_dev->model_data)) {
- rknpu_dev->model_data = NULL;
- LOG_DEV_ERROR(dev, "failed to initialize power model\n");
- } else if (rknpu_dev->model_data->dynamic_coefficient) {
- npu_cooling_power.dyn_power_coeff =
- rknpu_dev->model_data->dynamic_coefficient;
- }
- if (!npu_cooling_power.dyn_power_coeff) {
- LOG_DEV_ERROR(dev, "failed to get dynamic-coefficient\n");
- goto out;
- }
-
- rknpu_dev->devfreq_cooling = of_devfreq_cooling_register_power(
- dev->of_node, rknpu_dev->devfreq, &npu_cooling_power);
- if (IS_ERR_OR_NULL(rknpu_dev->devfreq_cooling))
- LOG_DEV_ERROR(dev, "failed to register cooling device\n");
-
-out:
- return 0;
-
-err_remove_governor:
- devfreq_remove_governor(&devfreq_rknpu_ondemand);
-err_remove_table:
- rockchip_uninit_opp_table(dev, &rknpu_dev->opp_info);
-
- rknpu_dev->devfreq = NULL;
-
- return ret;
-}
-EXPORT_SYMBOL(rknpu_devfreq_init);
-
-int rknpu_devfreq_runtime_suspend(struct device *dev)
-{
- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
- struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
-
- if (opp_info->scmi_clk) {
- if (clk_set_rate(opp_info->scmi_clk, POWER_DOWN_FREQ))
- LOG_DEV_ERROR(dev, "failed to restore clk rate\n");
- }
- opp_info->current_rm = UINT_MAX;
-
- return 0;
-}
-EXPORT_SYMBOL(rknpu_devfreq_runtime_suspend);
-
-int rknpu_devfreq_runtime_resume(struct device *dev)
-{
- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
- struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
- int ret = 0;
-
- if (!rknpu_dev->current_freq || !rknpu_dev->current_volt)
- return 0;
-
- ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks);
- if (ret) {
- LOG_DEV_INFO(dev, "failed to enable opp clks\n");
- return ret;
- }
-
- if (opp_info->data && opp_info->data->set_read_margin)
- opp_info->data->set_read_margin(dev, opp_info,
- opp_info->target_rm);
- if (opp_info->scmi_clk) {
- if (clk_set_rate(opp_info->scmi_clk, rknpu_dev->current_freq))
- LOG_DEV_ERROR(dev, "failed to set power down rate\n");
- }
-
- clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
-
- return ret;
-}
-EXPORT_SYMBOL(rknpu_devfreq_runtime_resume);
-#endif
-
-void rknpu_devfreq_remove(struct rknpu_device *rknpu_dev)
-{
- if (rknpu_dev->mdev_info) {
- rockchip_system_monitor_unregister(rknpu_dev->mdev_info);
- rknpu_dev->mdev_info = NULL;
- }
- if (rknpu_dev->devfreq)
- devfreq_remove_governor(&devfreq_rknpu_ondemand);
- rockchip_uninit_opp_table(rknpu_dev->dev, &rknpu_dev->opp_info);
-}
-EXPORT_SYMBOL(rknpu_devfreq_remove);
diff --git a/rknpu-driver/driver-0.9.6/rknpu_drv.c b/rknpu-driver/driver-0.9.6/rknpu_drv.c
deleted file mode 100644
index f4f4838..0000000
--- a/rknpu-driver/driver-0.9.6/rknpu_drv.c
+++ /dev/null
@@ -1,1549 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) Rockchip Electronics Co.Ltd
- * Author: Felix Zeng
- */
-
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-
-#ifndef FPGA_PLATFORM
-#include
-#endif
-
-#include "rknpu_ioctl.h"
-#include "rknpu_reset.h"
-#include "rknpu_fence.h"
-#include "rknpu_drv.h"
-#include "rknpu_gem.h"
-#include "rknpu_devfreq.h"
-#include "rknpu_iommu.h"
-
-#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
-#include
-#include
-#include
-#include
-#include "rknpu_gem.h"
-#endif
-
-#ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
-#include
-#include "rknpu_mem.h"
-#endif
-
-#define POWER_DOWN_FREQ 200000000
-#define NPU_MMU_DISABLED_POLL_PERIOD_US 1000
-#define NPU_MMU_DISABLED_POLL_TIMEOUT_US 20000
-
-static int bypass_irq_handler;
-module_param(bypass_irq_handler, int, 0644);
-MODULE_PARM_DESC(bypass_irq_handler,
- "bypass RKNPU irq handler if set it to 1, disabled by default");
-
-static int bypass_soft_reset;
-module_param(bypass_soft_reset, int, 0644);
-MODULE_PARM_DESC(bypass_soft_reset,
- "bypass RKNPU soft reset if set it to 1, disabled by default");
-
-static const struct rknpu_irqs_data rknpu_irqs[] = {
- { "npu_irq", rknpu_core0_irq_handler }
-};
-
-static const struct rknpu_irqs_data rk3576_npu_irqs[] = {
- { "npu0_irq", rknpu_core0_irq_handler },
- { "npu1_irq", rknpu_core1_irq_handler }
-};
-
-static const struct rknpu_irqs_data rk3588_npu_irqs[] = {
- { "npu0_irq", rknpu_core0_irq_handler },
- { "npu1_irq", rknpu_core1_irq_handler },
- { "npu2_irq", rknpu_core2_irq_handler }
-};
-
-static const struct rknpu_amount_data rknpu_old_top_amount = {
- .offset_clr_all = 0x8010,
- .offset_dt_wr = 0x8034,
- .offset_dt_rd = 0x8038,
- .offset_wt_rd = 0x803c,
-};
-
-static const struct rknpu_amount_data rknpu_top_amount = {
- .offset_clr_all = 0x2210,
- .offset_dt_wr = 0x2234,
- .offset_dt_rd = 0x2238,
- .offset_wt_rd = 0x223c
-};
-
-static const struct rknpu_amount_data rknpu_core_amount = {
- .offset_clr_all = 0x2410,
- .offset_dt_wr = 0x2434,
- .offset_dt_rd = 0x2438,
- .offset_wt_rd = 0x243c,
-};
-
-static const struct rknpu_config rk356x_rknpu_config = {
- .bw_priority_addr = 0xfe180008,
- .bw_priority_length = 0x10,
- .dma_mask = DMA_BIT_MASK(32),
- .pc_data_amount_scale = 1,
- .pc_task_number_bits = 12,
- .pc_task_number_mask = 0xfff,
- .pc_task_status_offset = 0x3c,
- .pc_dma_ctrl = 0,
- .irqs = rknpu_irqs,
- .num_irqs = ARRAY_SIZE(rknpu_irqs),
- .nbuf_phyaddr = 0,
- .nbuf_size = 0,
- .max_submit_number = (1 << 12) - 1,
- .core_mask = 0x1,
- .amount_top = &rknpu_old_top_amount,
- .amount_core = NULL,
-};
-
-static const struct rknpu_config rk3588_rknpu_config = {
- .bw_priority_addr = 0x0,
- .bw_priority_length = 0x0,
- .dma_mask = DMA_BIT_MASK(40),
- .pc_data_amount_scale = 2,
- .pc_task_number_bits = 12,
- .pc_task_number_mask = 0xfff,
- .pc_task_status_offset = 0x3c,
- .pc_dma_ctrl = 0,
- .irqs = rk3588_npu_irqs,
- .num_irqs = ARRAY_SIZE(rk3588_npu_irqs),
- .nbuf_phyaddr = 0,
- .nbuf_size = 0,
- .max_submit_number = (1 << 12) - 1,
- .core_mask = 0x7,
- .amount_top = NULL,
- .amount_core = NULL,
-};
-
-static const struct rknpu_config rk3583_rknpu_config = {
- .bw_priority_addr = 0x0,
- .bw_priority_length = 0x0,
- .dma_mask = DMA_BIT_MASK(40),
- .pc_data_amount_scale = 2,
- .pc_task_number_bits = 12,
- .pc_task_number_mask = 0xfff,
- .pc_task_status_offset = 0x3c,
- .pc_dma_ctrl = 0,
- .irqs = rk3588_npu_irqs,
- .num_irqs = 2,
- .nbuf_phyaddr = 0,
- .nbuf_size = 0,
- .max_submit_number = (1 << 12) - 1,
- .core_mask = 0x3,
- .amount_top = NULL,
- .amount_core = NULL,
-};
-
-static const struct rknpu_config rv1106_rknpu_config = {
- .bw_priority_addr = 0x0,
- .bw_priority_length = 0x0,
- .dma_mask = DMA_BIT_MASK(32),
- .pc_data_amount_scale = 2,
- .pc_task_number_bits = 16,
- .pc_task_number_mask = 0xffff,
- .pc_task_status_offset = 0x3c,
- .pc_dma_ctrl = 0,
- .irqs = rknpu_irqs,
- .num_irqs = ARRAY_SIZE(rknpu_irqs),
- .nbuf_phyaddr = 0,
- .nbuf_size = 0,
- .max_submit_number = (1 << 16) - 1,
- .core_mask = 0x1,
- .amount_top = &rknpu_old_top_amount,
- .amount_core = NULL,
-};
-
-static const struct rknpu_config rk3562_rknpu_config = {
- .bw_priority_addr = 0x0,
- .bw_priority_length = 0x0,
- .dma_mask = DMA_BIT_MASK(40),
- .pc_data_amount_scale = 2,
- .pc_task_number_bits = 16,
- .pc_task_number_mask = 0xffff,
- .pc_task_status_offset = 0x48,
- .pc_dma_ctrl = 1,
- .irqs = rknpu_irqs,
- .num_irqs = ARRAY_SIZE(rknpu_irqs),
- .nbuf_phyaddr = 0xfe400000,
- .nbuf_size = 256 * 1024,
- .max_submit_number = (1 << 16) - 1,
- .core_mask = 0x1,
- .amount_top = &rknpu_old_top_amount,
- .amount_core = NULL,
-};
-
-static const struct rknpu_config rk3576_rknpu_config = {
- .bw_priority_addr = 0x0,
- .bw_priority_length = 0x0,
- .dma_mask = DMA_BIT_MASK(40),
- .pc_data_amount_scale = 2,
- .pc_task_number_bits = 16,
- .pc_task_number_mask = 0xffff,
- .pc_task_status_offset = 0x48,
- .pc_dma_ctrl = 1,
- .irqs = rk3576_npu_irqs,
- .num_irqs = ARRAY_SIZE(rk3576_npu_irqs),
- .nbuf_phyaddr = 0x3fe80000,
- .nbuf_size = 1024 * 1024,
- .max_submit_number = (1 << 16) - 1,
- .core_mask = 0x3,
- .amount_top = &rknpu_top_amount,
- .amount_core = &rknpu_core_amount,
-};
-
-/* driver probe and init */
-static const struct of_device_id rknpu_of_match[] = {
- {
- .compatible = "rockchip,rknpu",
- .data = &rk356x_rknpu_config,
- },
- {
- .compatible = "rockchip,rk3568-rknpu",
- .data = &rk356x_rknpu_config,
- },
- {
- .compatible = "rockchip,rk3588-rknpu",
- .data = &rk3588_rknpu_config,
- },
- {
- .compatible = "rockchip,rv1106-rknpu",
- .data = &rv1106_rknpu_config,
- },
- {
- .compatible = "rockchip,rk3562-rknpu",
- .data = &rk3562_rknpu_config,
- },
- {
- .compatible = "rockchip,rk3576-rknpu",
- .data = &rk3576_rknpu_config,
- },
- {},
-};
-
-static int rknpu_get_drv_version(uint32_t *version)
-{
- *version = RKNPU_GET_DRV_VERSION_CODE(DRIVER_MAJOR, DRIVER_MINOR,
- DRIVER_PATCHLEVEL);
- return 0;
-}
-
-static int rknpu_power_on(struct rknpu_device *rknpu_dev);
-static int rknpu_power_off(struct rknpu_device *rknpu_dev);
-
-static void rknpu_power_off_delay_work(struct work_struct *power_off_work)
-{
- struct rknpu_device *rknpu_dev =
- container_of(to_delayed_work(power_off_work),
- struct rknpu_device, power_off_work);
- mutex_lock(&rknpu_dev->power_lock);
- if (atomic_dec_if_positive(&rknpu_dev->power_refcount) == 0)
- rknpu_power_off(rknpu_dev);
- mutex_unlock(&rknpu_dev->power_lock);
-}
-
-int rknpu_power_get(struct rknpu_device *rknpu_dev)
-{
- int ret = 0;
-
- mutex_lock(&rknpu_dev->power_lock);
- if (atomic_inc_return(&rknpu_dev->power_refcount) == 1)
- ret = rknpu_power_on(rknpu_dev);
- mutex_unlock(&rknpu_dev->power_lock);
-
- return ret;
-}
-
-int rknpu_power_put(struct rknpu_device *rknpu_dev)
-{
- int ret = 0;
-
- mutex_lock(&rknpu_dev->power_lock);
- if (atomic_dec_if_positive(&rknpu_dev->power_refcount) == 0)
- ret = rknpu_power_off(rknpu_dev);
- mutex_unlock(&rknpu_dev->power_lock);
-
- return ret;
-}
-
-static int rknpu_power_put_delay(struct rknpu_device *rknpu_dev)
-{
- if (rknpu_dev->power_put_delay == 0)
- return rknpu_power_put(rknpu_dev);
-
- mutex_lock(&rknpu_dev->power_lock);
- if (atomic_read(&rknpu_dev->power_refcount) == 1)
- queue_delayed_work(
- rknpu_dev->power_off_wq, &rknpu_dev->power_off_work,
- msecs_to_jiffies(rknpu_dev->power_put_delay));
- else
- atomic_dec_if_positive(&rknpu_dev->power_refcount);
- mutex_unlock(&rknpu_dev->power_lock);
-
- return 0;
-}
-
-static int rknpu_action(struct rknpu_device *rknpu_dev,
- struct rknpu_action *args)
-{
- int ret = -EINVAL;
-
- switch (args->flags) {
- case RKNPU_GET_HW_VERSION:
- ret = rknpu_get_hw_version(rknpu_dev, &args->value);
- break;
- case RKNPU_GET_DRV_VERSION:
- ret = rknpu_get_drv_version(&args->value);
- break;
- case RKNPU_GET_FREQ:
-#ifndef FPGA_PLATFORM
- args->value = clk_get_rate(rknpu_dev->clks[0].clk);
-#endif
- ret = 0;
- break;
- case RKNPU_SET_FREQ:
- break;
- case RKNPU_GET_VOLT:
-#ifndef FPGA_PLATFORM
- args->value = regulator_get_voltage(rknpu_dev->vdd);
-#endif
- ret = 0;
- break;
- case RKNPU_SET_VOLT:
- break;
- case RKNPU_ACT_RESET:
- ret = rknpu_soft_reset(rknpu_dev);
- break;
- case RKNPU_GET_BW_PRIORITY:
- ret = rknpu_get_bw_priority(rknpu_dev, &args->value, NULL,
- NULL);
- break;
- case RKNPU_SET_BW_PRIORITY:
- ret = rknpu_set_bw_priority(rknpu_dev, args->value, 0, 0);
- break;
- case RKNPU_GET_BW_EXPECT:
- ret = rknpu_get_bw_priority(rknpu_dev, NULL, &args->value,
- NULL);
- break;
- case RKNPU_SET_BW_EXPECT:
- ret = rknpu_set_bw_priority(rknpu_dev, 0, args->value, 0);
- break;
- case RKNPU_GET_BW_TW:
- ret = rknpu_get_bw_priority(rknpu_dev, NULL, NULL,
- &args->value);
- break;
- case RKNPU_SET_BW_TW:
- ret = rknpu_set_bw_priority(rknpu_dev, 0, 0, args->value);
- break;
- case RKNPU_ACT_CLR_TOTAL_RW_AMOUNT:
- ret = rknpu_clear_rw_amount(rknpu_dev);
- break;
- case RKNPU_GET_DT_WR_AMOUNT:
- ret = rknpu_get_rw_amount(rknpu_dev, &args->value, NULL, NULL);
- break;
- case RKNPU_GET_DT_RD_AMOUNT:
- ret = rknpu_get_rw_amount(rknpu_dev, NULL, &args->value, NULL);
- break;
- case RKNPU_GET_WT_RD_AMOUNT:
- ret = rknpu_get_rw_amount(rknpu_dev, NULL, NULL, &args->value);
- break;
- case RKNPU_GET_TOTAL_RW_AMOUNT:
- ret = rknpu_get_total_rw_amount(rknpu_dev, &args->value);
- break;
- case RKNPU_GET_IOMMU_EN:
- args->value = rknpu_dev->iommu_en;
- ret = 0;
- break;
- case RKNPU_SET_PROC_NICE:
- set_user_nice(current, *(int32_t *)&args->value);
- ret = 0;
- break;
- case RKNPU_GET_TOTAL_SRAM_SIZE:
- if (rknpu_dev->sram_mm)
- args->value = rknpu_dev->sram_mm->total_chunks *
- rknpu_dev->sram_mm->chunk_size;
- else
- args->value = 0;
- ret = 0;
- break;
- case RKNPU_GET_FREE_SRAM_SIZE:
- if (rknpu_dev->sram_mm)
- args->value = rknpu_dev->sram_mm->free_chunks *
- rknpu_dev->sram_mm->chunk_size;
- else
- args->value = 0;
- ret = 0;
- break;
- case RKNPU_GET_IOMMU_DOMAIN_ID:
- args->value = rknpu_dev->iommu_domain_id;
- ret = 0;
- break;
- case RKNPU_SET_IOMMU_DOMAIN_ID: {
- ret = rknpu_iommu_switch_domain(rknpu_dev,
- *(int32_t *)&args->value);
- break;
- }
- default:
- ret = -EINVAL;
- break;
- }
-
- return ret;
-}
-
-#ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
-static int rknpu_open(struct inode *inode, struct file *file)
-{
- struct rknpu_device *rknpu_dev =
- container_of(file->private_data, struct rknpu_device, miscdev);
- struct rknpu_session *session = NULL;
-
- session = kzalloc(sizeof(*session), GFP_KERNEL);
- if (!session) {
- LOG_ERROR("rknpu session alloc failed\n");
- return -ENOMEM;
- }
-
- session->rknpu_dev = rknpu_dev;
- INIT_LIST_HEAD(&session->list);
-
- file->private_data = (void *)session;
-
- return nonseekable_open(inode, file);
-}
-
-static int rknpu_release(struct inode *inode, struct file *file)
-{
- struct rknpu_mem_object *entry;
- struct rknpu_session *session = file->private_data;
- struct rknpu_device *rknpu_dev = session->rknpu_dev;
- LIST_HEAD(local_list);
-
- spin_lock(&rknpu_dev->lock);
- list_replace_init(&session->list, &local_list);
- file->private_data = NULL;
- spin_unlock(&rknpu_dev->lock);
-
- while (!list_empty(&local_list)) {
- entry = list_first_entry(&local_list, struct rknpu_mem_object,
- head);
-
- LOG_DEBUG(
- "Fd close free rknpu_obj: %#llx, rknpu_obj->dma_addr: %#llx\n",
- (__u64)(uintptr_t)entry, (__u64)entry->dma_addr);
-
- vunmap(entry->kv_addr);
- entry->kv_addr = NULL;
-
- if (!entry->owner)
- dma_buf_put(entry->dmabuf);
-
- list_del(&entry->head);
- kfree(entry);
- }
-
- kfree(session);
-
- return 0;
-}
-
-static int rknpu_action_ioctl(struct rknpu_device *rknpu_dev,
- unsigned long data)
-{
- struct rknpu_action args;
- int ret = -EINVAL;
-
- if (unlikely(copy_from_user(&args, (struct rknpu_action *)data,
- sizeof(struct rknpu_action)))) {
- LOG_ERROR("%s: copy_from_user failed\n", __func__);
- ret = -EFAULT;
- return ret;
- }
-
- ret = rknpu_action(rknpu_dev, &args);
-
- if (unlikely(copy_to_user((struct rknpu_action *)data, &args,
- sizeof(struct rknpu_action)))) {
- LOG_ERROR("%s: copy_to_user failed\n", __func__);
- ret = -EFAULT;
- return ret;
- }
-
- return ret;
-}
-
-static long rknpu_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
-{
- long ret = -EINVAL;
- struct rknpu_device *rknpu_dev = NULL;
-
- if (!file->private_data)
- return -EINVAL;
-
- rknpu_dev = ((struct rknpu_session *)file->private_data)->rknpu_dev;
-
- rknpu_power_get(rknpu_dev);
-
- switch (_IOC_NR(cmd)) {
- case RKNPU_ACTION:
- ret = rknpu_action_ioctl(rknpu_dev, arg);
- break;
- case RKNPU_SUBMIT:
- ret = rknpu_submit_ioctl(rknpu_dev, arg);
- break;
- case RKNPU_MEM_CREATE:
- ret = rknpu_mem_create_ioctl(rknpu_dev, file, cmd, arg);
- break;
- case RKNPU_MEM_MAP:
- break;
- case RKNPU_MEM_DESTROY:
- ret = rknpu_mem_destroy_ioctl(rknpu_dev, file, arg);
- break;
- case RKNPU_MEM_SYNC:
- ret = rknpu_mem_sync_ioctl(rknpu_dev, arg);
- break;
- default:
- break;
- }
-
- rknpu_power_put_delay(rknpu_dev);
-
- return ret;
-}
-const struct file_operations rknpu_fops = {
- .owner = THIS_MODULE,
- .open = rknpu_open,
- .release = rknpu_release,
- .unlocked_ioctl = rknpu_ioctl,
-#ifdef CONFIG_COMPAT
- .compat_ioctl = rknpu_ioctl,
-#endif
-};
-#endif
-
-#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
-#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
-static const struct vm_operations_struct rknpu_gem_vm_ops = {
- .fault = rknpu_gem_fault,
- .open = drm_gem_vm_open,
- .close = drm_gem_vm_close,
-};
-#endif
-
-static int rknpu_action_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file_priv)
-{
- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev->dev);
-
- return rknpu_action(rknpu_dev, (struct rknpu_action *)data);
-}
-
-#define RKNPU_IOCTL(func) \
- static int __##func(struct drm_device *dev, void *data, \
- struct drm_file *file_priv) \
- { \
- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev->dev); \
- int ret = -EINVAL; \
- rknpu_power_get(rknpu_dev); \
- ret = func(dev, data, file_priv); \
- rknpu_power_put_delay(rknpu_dev); \
- return ret; \
- }
-
-RKNPU_IOCTL(rknpu_action_ioctl);
-RKNPU_IOCTL(rknpu_submit_ioctl);
-RKNPU_IOCTL(rknpu_gem_create_ioctl);
-RKNPU_IOCTL(rknpu_gem_map_ioctl);
-RKNPU_IOCTL(rknpu_gem_destroy_ioctl);
-RKNPU_IOCTL(rknpu_gem_sync_ioctl);
-
-static const struct drm_ioctl_desc rknpu_ioctls[] = {
- DRM_IOCTL_DEF_DRV(RKNPU_ACTION, __rknpu_action_ioctl, DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(RKNPU_SUBMIT, __rknpu_submit_ioctl, DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(RKNPU_MEM_CREATE, __rknpu_gem_create_ioctl,
- DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(RKNPU_MEM_MAP, __rknpu_gem_map_ioctl,
- DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(RKNPU_MEM_DESTROY, __rknpu_gem_destroy_ioctl,
- DRM_RENDER_ALLOW),
- DRM_IOCTL_DEF_DRV(RKNPU_MEM_SYNC, __rknpu_gem_sync_ioctl,
- DRM_RENDER_ALLOW),
-};
-
-#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
-DEFINE_DRM_GEM_FOPS(rknpu_drm_driver_fops);
-#else
-static const struct file_operations rknpu_drm_driver_fops = {
- .owner = THIS_MODULE,
- .open = drm_open,
- .mmap = rknpu_gem_mmap,
- .poll = drm_poll,
- .read = drm_read,
- .unlocked_ioctl = drm_ioctl,
-#ifdef CONFIG_COMPAT
- .compat_ioctl = drm_compat_ioctl,
-#endif
- .release = drm_release,
- .llseek = noop_llseek,
-};
-#endif
-
-static struct drm_driver rknpu_drm_driver = {
-#if KERNEL_VERSION(5, 4, 0) <= LINUX_VERSION_CODE
- .driver_features = DRIVER_GEM | DRIVER_RENDER,
-#else
- .driver_features = DRIVER_GEM | DRIVER_PRIME | DRIVER_RENDER,
-#endif
-#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
- .gem_free_object_unlocked = rknpu_gem_free_object,
- .gem_vm_ops = &rknpu_gem_vm_ops,
- .dumb_destroy = drm_gem_dumb_destroy,
- .gem_prime_export = drm_gem_prime_export,
- .gem_prime_get_sg_table = rknpu_gem_prime_get_sg_table,
- .gem_prime_vmap = rknpu_gem_prime_vmap,
- .gem_prime_vunmap = rknpu_gem_prime_vunmap,
-#endif
- .dumb_create = rknpu_gem_dumb_create,
-#if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE
- .dumb_map_offset = rknpu_gem_dumb_map_offset,
-#else
- .dumb_map_offset = drm_gem_dumb_map_offset,
-#endif
- .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
- .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
-#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
- .gem_prime_import = rknpu_gem_prime_import,
-#else
- .gem_prime_import = drm_gem_prime_import,
-#endif
- .gem_prime_import_sg_table = rknpu_gem_prime_import_sg_table,
-#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
- .gem_prime_mmap = drm_gem_prime_mmap,
-#else
- .gem_prime_mmap = rknpu_gem_prime_mmap,
-#endif
- .ioctls = rknpu_ioctls,
- .num_ioctls = ARRAY_SIZE(rknpu_ioctls),
- .fops = &rknpu_drm_driver_fops,
- .name = DRIVER_NAME,
- .desc = DRIVER_DESC,
- .date = DRIVER_DATE,
- .major = DRIVER_MAJOR,
- .minor = DRIVER_MINOR,
- .patchlevel = DRIVER_PATCHLEVEL,
-};
-
-#endif
-
-static enum hrtimer_restart hrtimer_handler(struct hrtimer *timer)
-{
- struct rknpu_device *rknpu_dev =
- container_of(timer, struct rknpu_device, timer);
- struct rknpu_subcore_data *subcore_data = NULL;
- struct rknpu_job *job = NULL;
- ktime_t now;
- unsigned long flags;
- int i;
-
- for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
- subcore_data = &rknpu_dev->subcore_datas[i];
-
- spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
-
- job = subcore_data->job;
- if (job) {
- now = ktime_get();
- subcore_data->timer.busy_time +=
- ktime_sub(now, job->hw_recoder_time);
- job->hw_recoder_time = now;
- }
-
- subcore_data->timer.total_busy_time =
- subcore_data->timer.busy_time;
- subcore_data->timer.busy_time = 0;
-
- spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
- }
-
- hrtimer_forward_now(timer, rknpu_dev->kt);
- return HRTIMER_RESTART;
-}
-
-static void rknpu_init_timer(struct rknpu_device *rknpu_dev)
-{
- rknpu_dev->kt = ktime_set(0, RKNPU_LOAD_INTERVAL);
- hrtimer_init(&rknpu_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
- rknpu_dev->timer.function = hrtimer_handler;
- hrtimer_start(&rknpu_dev->timer, rknpu_dev->kt, HRTIMER_MODE_REL);
-}
-
-static void rknpu_cancel_timer(struct rknpu_device *rknpu_dev)
-{
- hrtimer_cancel(&rknpu_dev->timer);
-}
-
-static bool rknpu_is_iommu_enable(struct device *dev)
-{
- struct device_node *iommu = NULL;
-
- iommu = of_parse_phandle(dev->of_node, "iommus", 0);
- if (!iommu) {
- LOG_DEV_INFO(
- dev,
- "rknpu iommu device-tree entry not found!, using non-iommu mode\n");
- return false;
- }
-
- if (!of_device_is_available(iommu)) {
- LOG_DEV_INFO(dev,
- "rknpu iommu is disabled, using non-iommu mode\n");
- of_node_put(iommu);
- return false;
- }
- of_node_put(iommu);
-
- LOG_DEV_INFO(dev, "rknpu iommu is enabled, using iommu mode\n");
-
- return true;
-}
-
-#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
-static int drm_fake_dev_register(struct rknpu_device *rknpu_dev)
-{
- const struct platform_device_info rknpu_dev_info = {
- .name = "rknpu_dev",
- .id = PLATFORM_DEVID_AUTO,
- .dma_mask = rknpu_dev->config->dma_mask,
- };
- struct platform_device *pdev = NULL;
- int ret = -EINVAL;
-
- pdev = platform_device_register_full(&rknpu_dev_info);
- if (pdev) {
- ret = of_dma_configure(&pdev->dev, NULL, true);
- if (ret) {
- platform_device_unregister(pdev);
- pdev = NULL;
- }
- }
-
- rknpu_dev->fake_dev = pdev ? &pdev->dev : NULL;
-
- return ret;
-}
-
-static void drm_fake_dev_unregister(struct rknpu_device *rknpu_dev)
-{
- struct platform_device *pdev = NULL;
-
- if (!rknpu_dev->fake_dev)
- return;
-
- pdev = to_platform_device(rknpu_dev->fake_dev);
-
- platform_device_unregister(pdev);
-}
-
-static int rknpu_drm_probe(struct rknpu_device *rknpu_dev)
-{
- struct device *dev = rknpu_dev->dev;
- struct drm_device *drm_dev = NULL;
- int ret = -EINVAL;
-
- drm_dev = drm_dev_alloc(&rknpu_drm_driver, dev);
- if (IS_ERR(drm_dev))
- return PTR_ERR(drm_dev);
-
- /* register the DRM device */
- ret = drm_dev_register(drm_dev, 0);
- if (ret < 0)
- goto err_free_drm;
-
- drm_dev->dev_private = rknpu_dev;
- rknpu_dev->drm_dev = drm_dev;
-
- drm_fake_dev_register(rknpu_dev);
-
- return 0;
-
-err_free_drm:
-#if KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE
- drm_dev_put(drm_dev);
-#else
- drm_dev_unref(drm_dev);
-#endif
-
- return ret;
-}
-
-static void rknpu_drm_remove(struct rknpu_device *rknpu_dev)
-{
- struct drm_device *drm_dev = rknpu_dev->drm_dev;
-
- drm_fake_dev_unregister(rknpu_dev);
-
- drm_dev_unregister(drm_dev);
-
-#if KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE
- drm_dev_put(drm_dev);
-#else
- drm_dev_unref(drm_dev);
-#endif
-}
-#endif
-
-static int rknpu_power_on(struct rknpu_device *rknpu_dev)
-{
- struct device *dev = rknpu_dev->dev;
- int ret = -EINVAL;
-
-#ifndef FPGA_PLATFORM
- if (rknpu_dev->vdd) {
- ret = regulator_enable(rknpu_dev->vdd);
- if (ret) {
- LOG_DEV_ERROR(
- dev,
- "failed to enable vdd reg for rknpu, ret: %d\n",
- ret);
- return ret;
- }
- }
-
- if (rknpu_dev->mem) {
- ret = regulator_enable(rknpu_dev->mem);
- if (ret) {
- LOG_DEV_ERROR(
- dev,
- "failed to enable mem reg for rknpu, ret: %d\n",
- ret);
- return ret;
- }
- }
-#endif
-
- ret = clk_bulk_prepare_enable(rknpu_dev->num_clks, rknpu_dev->clks);
- if (ret) {
- LOG_DEV_ERROR(dev, "failed to enable clk for rknpu, ret: %d\n",
- ret);
- return ret;
- }
-
-#ifndef FPGA_PLATFORM
- rknpu_devfreq_lock(rknpu_dev);
-#endif
-
- if (rknpu_dev->multiple_domains) {
- if (rknpu_dev->genpd_dev_npu0) {
-#if KERNEL_VERSION(5, 5, 0) < LINUX_VERSION_CODE
- ret = pm_runtime_resume_and_get(
- rknpu_dev->genpd_dev_npu0);
-#else
- ret = pm_runtime_get_sync(rknpu_dev->genpd_dev_npu0);
-#endif
- if (ret < 0) {
- LOG_DEV_ERROR(
- dev,
- "failed to get pm runtime for npu0, ret: %d\n",
- ret);
- goto out;
- }
- }
- if (rknpu_dev->genpd_dev_npu1) {
-#if KERNEL_VERSION(5, 5, 0) < LINUX_VERSION_CODE
- ret = pm_runtime_resume_and_get(
- rknpu_dev->genpd_dev_npu1);
-#else
- ret = pm_runtime_get_sync(rknpu_dev->genpd_dev_npu1);
-#endif
- if (ret < 0) {
- LOG_DEV_ERROR(
- dev,
- "failed to get pm runtime for npu1, ret: %d\n",
- ret);
- goto out;
- }
- }
- if (rknpu_dev->genpd_dev_npu2) {
-#if KERNEL_VERSION(5, 5, 0) < LINUX_VERSION_CODE
- ret = pm_runtime_resume_and_get(
- rknpu_dev->genpd_dev_npu2);
-#else
- ret = pm_runtime_get_sync(rknpu_dev->genpd_dev_npu2);
-#endif
- if (ret < 0) {
- LOG_DEV_ERROR(
- dev,
- "failed to get pm runtime for npu2, ret: %d\n",
- ret);
- goto out;
- }
- }
- }
- ret = pm_runtime_get_sync(dev);
- if (ret < 0) {
- LOG_DEV_ERROR(dev,
- "failed to get pm runtime for rknpu, ret: %d\n",
- ret);
- }
-
-out:
-#ifndef FPGA_PLATFORM
- rknpu_devfreq_unlock(rknpu_dev);
-#endif
-
- return ret;
-}
-
-static int rknpu_power_off(struct rknpu_device *rknpu_dev)
-{
- struct device *dev = rknpu_dev->dev;
-
-#ifndef FPGA_PLATFORM
- int ret;
- bool val;
-
- rknpu_devfreq_lock(rknpu_dev);
-#endif
-
- pm_runtime_put_sync(dev);
-
- if (rknpu_dev->multiple_domains) {
-#ifndef FPGA_PLATFORM
- /*
- * Because IOMMU's runtime suspend callback is asynchronous,
- * So it may be executed after the NPU is turned off after PD/CLK/VD,
- * and the runtime suspend callback has a register access.
- * If the PD/VD/CLK is closed, the register access will crash.
- * As a workaround, it's safe to close pd stuff until iommu disabled.
- * If pm runtime framework can handle this issue in the future, remove
- * this.
- */
- ret = readx_poll_timeout(rockchip_iommu_is_enabled, dev, val,
- !val, NPU_MMU_DISABLED_POLL_PERIOD_US,
- NPU_MMU_DISABLED_POLL_TIMEOUT_US);
- if (ret) {
- LOG_DEV_ERROR(dev, "iommu still enabled\n");
- pm_runtime_get_sync(dev);
- rknpu_devfreq_unlock(rknpu_dev);
- return ret;
- }
-#else
- if (rknpu_dev->iommu_en)
- msleep(20);
-#endif
- if (rknpu_dev->genpd_dev_npu2)
- pm_runtime_put_sync(rknpu_dev->genpd_dev_npu2);
- if (rknpu_dev->genpd_dev_npu1)
- pm_runtime_put_sync(rknpu_dev->genpd_dev_npu1);
- if (rknpu_dev->genpd_dev_npu0)
- pm_runtime_put_sync(rknpu_dev->genpd_dev_npu0);
- }
-
-#ifndef FPGA_PLATFORM
- rknpu_devfreq_unlock(rknpu_dev);
-#endif
-
- clk_bulk_disable_unprepare(rknpu_dev->num_clks, rknpu_dev->clks);
-
-#ifndef FPGA_PLATFORM
- if (rknpu_dev->vdd)
- regulator_disable(rknpu_dev->vdd);
-
- if (rknpu_dev->mem)
- regulator_disable(rknpu_dev->mem);
-#endif
-
- return 0;
-}
-
-static int rknpu_register_irq(struct platform_device *pdev,
- struct rknpu_device *rknpu_dev)
-{
- const struct rknpu_config *config = rknpu_dev->config;
- struct device *dev = &pdev->dev;
-#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
- struct resource *res;
-#endif
- int i, ret, irq;
-
-#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
- res = platform_get_resource_byname(pdev, IORESOURCE_IRQ,
- config->irqs[0].name);
- if (res) {
- /* there are irq names in dts */
- for (i = 0; i < config->num_irqs; i++) {
- irq = platform_get_irq_byname(pdev,
- config->irqs[i].name);
- if (irq < 0) {
- LOG_DEV_ERROR(dev, "no npu %s in dts\n",
- config->irqs[i].name);
- return irq;
- }
-
- ret = devm_request_irq(dev, irq,
- config->irqs[i].irq_hdl,
- IRQF_SHARED, dev_name(dev),
- rknpu_dev);
- if (ret < 0) {
- LOG_DEV_ERROR(dev, "request %s failed: %d\n",
- config->irqs[i].name, ret);
- return ret;
- }
- }
- } else {
- /* no irq names in dts */
- irq = platform_get_irq(pdev, 0);
- if (irq < 0) {
- LOG_DEV_ERROR(dev, "no npu irq in dts\n");
- return irq;
- }
-
- ret = devm_request_irq(dev, irq, rknpu_core0_irq_handler,
- IRQF_SHARED, dev_name(dev), rknpu_dev);
- if (ret < 0) {
- LOG_DEV_ERROR(dev, "request irq failed: %d\n", ret);
- return ret;
- }
- }
-#else
- /* there are irq names in dts */
- for (i = 0; i < config->num_irqs; i++) {
- irq = platform_get_irq_byname(pdev, config->irqs[i].name);
- if (irq < 0) {
- irq = platform_get_irq(pdev, i);
- if (irq < 0) {
- LOG_DEV_ERROR(dev, "no npu %s in dts\n",
- config->irqs[i].name);
- return irq;
- }
- }
-
- ret = devm_request_irq(dev, irq, config->irqs[i].irq_hdl,
- IRQF_SHARED, dev_name(dev), rknpu_dev);
- if (ret < 0) {
- LOG_DEV_ERROR(dev, "request %s failed: %d\n",
- config->irqs[i].name, ret);
- return ret;
- }
- }
-#endif
-
- return 0;
-}
-
-static int rknpu_find_sram_resource(struct rknpu_device *rknpu_dev)
-{
- struct device *dev = rknpu_dev->dev;
- struct device_node *sram_node = NULL;
- struct resource sram_res;
- uint32_t sram_size = 0;
- int ret = -EINVAL;
-
- /* get sram device node */
- sram_node = of_parse_phandle(dev->of_node, "rockchip,sram", 0);
- rknpu_dev->sram_size = 0;
- if (!sram_node)
- return -EINVAL;
-
- /* get sram start and size */
- ret = of_address_to_resource(sram_node, 0, &sram_res);
- of_node_put(sram_node);
- if (ret)
- return ret;
-
- /* check sram start and size is PAGE_SIZE align */
- rknpu_dev->sram_start = round_up(sram_res.start, PAGE_SIZE);
- rknpu_dev->sram_end = round_down(
- sram_res.start + resource_size(&sram_res), PAGE_SIZE);
- if (rknpu_dev->sram_end <= rknpu_dev->sram_start) {
- LOG_DEV_WARN(
- dev,
- "invalid sram resource, sram start %pa, sram end %pa\n",
- &rknpu_dev->sram_start, &rknpu_dev->sram_end);
- return -EINVAL;
- }
-
- sram_size = rknpu_dev->sram_end - rknpu_dev->sram_start;
-
- rknpu_dev->sram_base_io =
- devm_ioremap(dev, rknpu_dev->sram_start, sram_size);
- if (IS_ERR(rknpu_dev->sram_base_io)) {
- LOG_DEV_ERROR(dev, "failed to remap sram base io!\n");
- rknpu_dev->sram_base_io = NULL;
- }
-
- rknpu_dev->sram_size = sram_size;
-
- LOG_DEV_INFO(dev, "sram region: [%pa, %pa), sram size: %#x\n",
- &rknpu_dev->sram_start, &rknpu_dev->sram_end,
- rknpu_dev->sram_size);
-
- return 0;
-}
-
-static int rknpu_find_nbuf_resource(struct rknpu_device *rknpu_dev)
-{
- struct device *dev = rknpu_dev->dev;
-
- if (rknpu_dev->config->nbuf_size == 0)
- return -EINVAL;
-
- rknpu_dev->nbuf_start = rknpu_dev->config->nbuf_phyaddr;
- rknpu_dev->nbuf_size = rknpu_dev->config->nbuf_size;
- rknpu_dev->nbuf_base_io =
- devm_ioremap(dev, rknpu_dev->nbuf_start, rknpu_dev->nbuf_size);
- if (IS_ERR(rknpu_dev->nbuf_base_io)) {
- LOG_DEV_ERROR(dev, "failed to remap nbuf base io!\n");
- rknpu_dev->nbuf_base_io = NULL;
- }
-
- rknpu_dev->nbuf_end = rknpu_dev->nbuf_start + rknpu_dev->nbuf_size;
-
- LOG_DEV_INFO(dev, "nbuf region: [%pa, %pa), nbuf size: %#x\n",
- &rknpu_dev->nbuf_start, &rknpu_dev->nbuf_end,
- rknpu_dev->nbuf_size);
-
- return 0;
-}
-
-static int rknpu_get_invalid_core_mask(struct device *dev)
-{
- int ret = 0;
- u8 invalid_core_mask = 0;
-
- if (of_property_match_string(dev->of_node, "nvmem-cell-names",
- "cores") >= 0) {
- ret = rockchip_nvmem_cell_read_u8(dev->of_node, "cores",
- &invalid_core_mask);
- /* The default valid npu cores for RK3583 are core0 and core1 */
- invalid_core_mask |= RKNPU_CORE2_MASK;
- if (ret) {
- LOG_DEV_ERROR(
- dev,
- "failed to get specification_serial_number\n");
- return invalid_core_mask;
- }
- }
-
- return (int)invalid_core_mask;
-}
-
-static int rknpu_probe(struct platform_device *pdev)
-{
- struct resource *res = NULL;
- struct rknpu_device *rknpu_dev = NULL;
- struct device *dev = &pdev->dev;
- struct device *virt_dev = NULL;
- const struct of_device_id *match = NULL;
- const struct rknpu_config *config = NULL;
- int ret = -EINVAL, i = 0;
-
- if (!pdev->dev.of_node) {
- LOG_DEV_ERROR(dev, "rknpu device-tree data is missing!\n");
- return -ENODEV;
- }
-
- match = of_match_device(rknpu_of_match, dev);
- if (!match) {
- LOG_DEV_ERROR(dev, "rknpu device-tree entry is missing!\n");
- return -ENODEV;
- }
-
- rknpu_dev = devm_kzalloc(dev, sizeof(*rknpu_dev), GFP_KERNEL);
- if (!rknpu_dev) {
- LOG_DEV_ERROR(dev, "failed to allocate rknpu device!\n");
- return -ENOMEM;
- }
-
- config = of_device_get_match_data(dev);
- if (!config)
- return -EINVAL;
-
- if (match->data == (void *)&rk3588_rknpu_config) {
- int invalid_core_mask = rknpu_get_invalid_core_mask(dev);
- /* The default valid npu cores for RK3583 are core0 and core1 */
- if (invalid_core_mask & RKNPU_CORE2_MASK) {
- if ((invalid_core_mask & RKNPU_CORE0_MASK) ||
- (invalid_core_mask & RKNPU_CORE1_MASK)) {
- LOG_DEV_ERROR(
- dev,
- "rknpu core invalid, invalid core mask: %#x\n",
- invalid_core_mask);
- return -ENODEV;
- }
- config = &rk3583_rknpu_config;
- }
- }
-
- rknpu_dev->config = config;
- rknpu_dev->dev = dev;
-
- rknpu_dev->iommu_en = rknpu_is_iommu_enable(dev);
- if (rknpu_dev->iommu_en) {
- rknpu_dev->iommu_group = iommu_group_get(dev);
- if (!rknpu_dev->iommu_group)
- return -EINVAL;
- } else {
- /* Initialize reserved memory resources */
- ret = of_reserved_mem_device_init(dev);
- if (!ret) {
- LOG_DEV_INFO(
- dev,
- "initialize reserved memory for rknpu device!\n");
- }
- }
-
- rknpu_dev->bypass_irq_handler = bypass_irq_handler;
- rknpu_dev->bypass_soft_reset = bypass_soft_reset;
-
- rknpu_reset_get(rknpu_dev);
-
- rknpu_dev->num_clks = devm_clk_bulk_get_all(dev, &rknpu_dev->clks);
- if (rknpu_dev->num_clks < 1) {
- LOG_DEV_ERROR(dev, "failed to get clk source for rknpu\n");
-#ifndef FPGA_PLATFORM
- return -ENODEV;
-#endif
- }
-
-#ifndef FPGA_PLATFORM
- rknpu_dev->vdd = devm_regulator_get_optional(dev, "rknpu");
- if (IS_ERR(rknpu_dev->vdd)) {
- if (PTR_ERR(rknpu_dev->vdd) != -ENODEV) {
- ret = PTR_ERR(rknpu_dev->vdd);
- LOG_DEV_ERROR(
- dev,
- "failed to get vdd regulator for rknpu: %d\n",
- ret);
- return ret;
- }
- rknpu_dev->vdd = NULL;
- }
-
- rknpu_dev->mem = devm_regulator_get_optional(dev, "mem");
- if (IS_ERR(rknpu_dev->mem)) {
- if (PTR_ERR(rknpu_dev->mem) != -ENODEV) {
- ret = PTR_ERR(rknpu_dev->mem);
- LOG_DEV_ERROR(
- dev,
- "failed to get mem regulator for rknpu: %d\n",
- ret);
- return ret;
- }
- rknpu_dev->mem = NULL;
- }
-#endif
-
- spin_lock_init(&rknpu_dev->lock);
- spin_lock_init(&rknpu_dev->irq_lock);
- mutex_init(&rknpu_dev->power_lock);
- mutex_init(&rknpu_dev->reset_lock);
- mutex_init(&rknpu_dev->domain_lock);
- for (i = 0; i < config->num_irqs; i++) {
- INIT_LIST_HEAD(&rknpu_dev->subcore_datas[i].todo_list);
- init_waitqueue_head(&rknpu_dev->subcore_datas[i].job_done_wq);
- rknpu_dev->subcore_datas[i].task_num = 0;
- res = platform_get_resource(pdev, IORESOURCE_MEM, i);
- if (!res) {
- LOG_DEV_ERROR(
- dev,
- "failed to get memory resource for rknpu\n");
- return -ENXIO;
- }
-
- rknpu_dev->base[i] = devm_ioremap_resource(dev, res);
- if (PTR_ERR(rknpu_dev->base[i]) == -EBUSY) {
- rknpu_dev->base[i] = devm_ioremap(dev, res->start,
- resource_size(res));
- }
-
- if (IS_ERR(rknpu_dev->base[i])) {
- LOG_DEV_ERROR(dev,
- "failed to remap register for rknpu\n");
- return PTR_ERR(rknpu_dev->base[i]);
- }
- }
-
- if (config->bw_priority_length > 0) {
- rknpu_dev->bw_priority_base =
- devm_ioremap(dev, config->bw_priority_addr,
- config->bw_priority_length);
- if (IS_ERR(rknpu_dev->bw_priority_base)) {
- LOG_DEV_ERROR(
- rknpu_dev->dev,
- "failed to remap bw priority register for rknpu\n");
- rknpu_dev->bw_priority_base = NULL;
- }
- }
-
- if (!rknpu_dev->bypass_irq_handler) {
- ret = rknpu_register_irq(pdev, rknpu_dev);
- if (ret)
- return ret;
- } else {
- LOG_DEV_WARN(dev, "bypass irq handler!\n");
- }
-
-#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
- ret = rknpu_drm_probe(rknpu_dev);
- if (ret) {
- LOG_DEV_ERROR(dev, "failed to probe device for rknpu\n");
- return ret;
- }
-#endif
-#ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
- rknpu_dev->miscdev.minor = MISC_DYNAMIC_MINOR;
- rknpu_dev->miscdev.name = "rknpu";
- rknpu_dev->miscdev.fops = &rknpu_fops;
-
- ret = misc_register(&rknpu_dev->miscdev);
- if (ret) {
- LOG_DEV_ERROR(dev, "cannot register miscdev (%d)\n", ret);
- return ret;
- }
-
- rknpu_dev->heap = rk_dma_heap_find("rk-dma-heap-cma");
- if (!rknpu_dev->heap) {
- LOG_DEV_ERROR(dev, "failed to find cma heap\n");
- return -ENOMEM;
- }
- rk_dma_heap_set_dev(dev);
- LOG_DEV_INFO(dev, "Initialized %s: v%d.%d.%d for %s\n", DRIVER_DESC,
- DRIVER_MAJOR, DRIVER_MINOR, DRIVER_PATCHLEVEL,
- DRIVER_DATE);
-#endif
-
-#ifdef CONFIG_ROCKCHIP_RKNPU_FENCE
- ret = rknpu_fence_context_alloc(rknpu_dev);
- if (ret) {
- LOG_DEV_ERROR(dev,
- "failed to allocate fence context for rknpu\n");
- goto err_remove_drv;
- }
-#endif
-
- platform_set_drvdata(pdev, rknpu_dev);
-
- pm_runtime_enable(dev);
-
- if (of_count_phandle_with_args(dev->of_node, "power-domains",
- "#power-domain-cells") > 1) {
- virt_dev = dev_pm_domain_attach_by_name(dev, "npu0");
- if (!IS_ERR(virt_dev))
- rknpu_dev->genpd_dev_npu0 = virt_dev;
- virt_dev = dev_pm_domain_attach_by_name(dev, "npu1");
- if (!IS_ERR(virt_dev))
- rknpu_dev->genpd_dev_npu1 = virt_dev;
- if (config->num_irqs > 2) {
- virt_dev = dev_pm_domain_attach_by_name(dev, "npu2");
- if (!IS_ERR(virt_dev))
- rknpu_dev->genpd_dev_npu2 = virt_dev;
- }
- rknpu_dev->multiple_domains = true;
- }
-
- ret = rknpu_power_on(rknpu_dev);
- if (ret)
- goto err_remove_drv;
-
-#ifndef FPGA_PLATFORM
- rknpu_devfreq_init(rknpu_dev);
-#endif
-
- // set default power put delay to 3s
- rknpu_dev->power_put_delay = 3000;
- rknpu_dev->power_off_wq =
- create_freezable_workqueue("rknpu_power_off_wq");
- if (!rknpu_dev->power_off_wq) {
- LOG_DEV_ERROR(dev, "rknpu couldn't create power_off workqueue");
- ret = -ENOMEM;
- goto err_devfreq_remove;
- }
- INIT_DEFERRABLE_WORK(&rknpu_dev->power_off_work,
- rknpu_power_off_delay_work);
-
- if (IS_ENABLED(CONFIG_NO_GKI) &&
- IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->iommu_en) {
- if (!rknpu_find_sram_resource(rknpu_dev)) {
- ret = rknpu_mm_create(rknpu_dev->sram_size, PAGE_SIZE,
- &rknpu_dev->sram_mm);
- if (ret != 0)
- goto err_remove_wq;
- } else {
- LOG_DEV_WARN(dev, "could not find sram resource!\n");
- }
- }
-
- if (IS_ENABLED(CONFIG_NO_GKI) && rknpu_dev->iommu_en &&
- rknpu_dev->config->nbuf_size > 0)
- rknpu_find_nbuf_resource(rknpu_dev);
-
- if (rknpu_dev->iommu_en)
- rknpu_iommu_init_domain(rknpu_dev);
-
- rknpu_power_off(rknpu_dev);
- atomic_set(&rknpu_dev->power_refcount, 0);
- atomic_set(&rknpu_dev->cmdline_power_refcount, 0);
-
- rknpu_debugger_init(rknpu_dev);
- rknpu_init_timer(rknpu_dev);
-
- return 0;
-
-err_remove_wq:
- destroy_workqueue(rknpu_dev->power_off_wq);
-
-err_devfreq_remove:
-#ifndef FPGA_PLATFORM
- rknpu_devfreq_remove(rknpu_dev);
-#endif
-
-err_remove_drv:
-#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
- rknpu_drm_remove(rknpu_dev);
-#endif
-#ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
- misc_deregister(&(rknpu_dev->miscdev));
-#endif
-
- return ret;
-}
-
-static int rknpu_remove(struct platform_device *pdev)
-{
- struct rknpu_device *rknpu_dev = platform_get_drvdata(pdev);
- int i = 0;
-
- cancel_delayed_work_sync(&rknpu_dev->power_off_work);
- destroy_workqueue(rknpu_dev->power_off_wq);
-
- rknpu_debugger_remove(rknpu_dev);
- rknpu_cancel_timer(rknpu_dev);
-
- for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
- WARN_ON(rknpu_dev->subcore_datas[i].job);
- WARN_ON(!list_empty(&rknpu_dev->subcore_datas[i].todo_list));
- }
-
- if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->sram_mm)
- rknpu_mm_destroy(rknpu_dev->sram_mm);
-
- if (rknpu_dev->iommu_en) {
- rknpu_iommu_free_domains(rknpu_dev);
- iommu_group_put(rknpu_dev->iommu_group);
- }
-
-#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
- rknpu_drm_remove(rknpu_dev);
-#endif
-#ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
- misc_deregister(&(rknpu_dev->miscdev));
-#endif
-
-#ifndef FPGA_PLATFORM
- rknpu_devfreq_remove(rknpu_dev);
-#endif
-
- mutex_lock(&rknpu_dev->power_lock);
- if (atomic_read(&rknpu_dev->power_refcount) > 0)
- rknpu_power_off(rknpu_dev);
- mutex_unlock(&rknpu_dev->power_lock);
-
- if (rknpu_dev->multiple_domains) {
- if (rknpu_dev->genpd_dev_npu0)
- dev_pm_domain_detach(rknpu_dev->genpd_dev_npu0, true);
- if (rknpu_dev->genpd_dev_npu1)
- dev_pm_domain_detach(rknpu_dev->genpd_dev_npu1, true);
- if (rknpu_dev->genpd_dev_npu2)
- dev_pm_domain_detach(rknpu_dev->genpd_dev_npu2, true);
- }
-
- pm_runtime_disable(&pdev->dev);
-
- return 0;
-}
-
-#ifndef FPGA_PLATFORM
-static int rknpu_runtime_suspend(struct device *dev)
-{
- return rknpu_devfreq_runtime_suspend(dev);
-}
-
-static int rknpu_runtime_resume(struct device *dev)
-{
- return rknpu_devfreq_runtime_resume(dev);
-}
-
-static const struct dev_pm_ops rknpu_pm_ops = {
- SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
- pm_runtime_force_resume)
- SET_RUNTIME_PM_OPS(rknpu_runtime_suspend, rknpu_runtime_resume,
- NULL)
-};
-#endif
-
-static struct platform_driver rknpu_driver = {
- .probe = rknpu_probe,
- .remove = rknpu_remove,
- .driver = {
- .owner = THIS_MODULE,
- .name = "RKNPU",
-#ifndef FPGA_PLATFORM
- .pm = &rknpu_pm_ops,
-#endif
- .of_match_table = of_match_ptr(rknpu_of_match),
- },
-};
-
-static int rknpu_init(void)
-{
- return platform_driver_register(&rknpu_driver);
-}
-
-static void rknpu_exit(void)
-{
- platform_driver_unregister(&rknpu_driver);
-}
-
-late_initcall(rknpu_init);
-module_exit(rknpu_exit);
-
-MODULE_DESCRIPTION("RKNPU driver");
-MODULE_AUTHOR("Felix Zeng ");
-MODULE_ALIAS("rockchip-rknpu");
-MODULE_LICENSE("GPL v2");
-MODULE_VERSION(RKNPU_GET_DRV_VERSION_STRING(DRIVER_MAJOR, DRIVER_MINOR,
- DRIVER_PATCHLEVEL));
-#if KERNEL_VERSION(5, 16, 0) < LINUX_VERSION_CODE
-MODULE_IMPORT_NS(DMA_BUF);
-#endif
diff --git a/rknpu-driver/driver-0.9.6/rknpu_fence.c b/rknpu-driver/driver-0.9.6/rknpu_fence.c
deleted file mode 100644
index dc22ea1..0000000
--- a/rknpu-driver/driver-0.9.6/rknpu_fence.c
+++ /dev/null
@@ -1,80 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) Rockchip Electronics Co.Ltd
- * Author: Felix Zeng
- */
-
-#include
-#include
-#include
-#include
-
-#include "rknpu_drv.h"
-#include "rknpu_job.h"
-
-#include "rknpu_fence.h"
-
-static const char *rknpu_fence_get_name(struct dma_fence *fence)
-{
- return DRIVER_NAME;
-}
-
-static const struct dma_fence_ops rknpu_fence_ops = {
- .get_driver_name = rknpu_fence_get_name,
- .get_timeline_name = rknpu_fence_get_name,
-};
-
-int rknpu_fence_context_alloc(struct rknpu_device *rknpu_dev)
-{
- struct rknpu_fence_context *fence_ctx = NULL;
-
- fence_ctx =
- devm_kzalloc(rknpu_dev->dev, sizeof(*fence_ctx), GFP_KERNEL);
- if (!fence_ctx)
- return -ENOMEM;
-
- fence_ctx->context = dma_fence_context_alloc(1);
- spin_lock_init(&fence_ctx->spinlock);
-
- rknpu_dev->fence_ctx = fence_ctx;
-
- return 0;
-}
-
-int rknpu_fence_alloc(struct rknpu_job *job)
-{
- struct rknpu_fence_context *fence_ctx = job->rknpu_dev->fence_ctx;
- struct dma_fence *fence = NULL;
-
- fence = kzalloc(sizeof(*fence), GFP_KERNEL);
- if (!fence)
- return -ENOMEM;
-
- dma_fence_init(fence, &rknpu_fence_ops, &fence_ctx->spinlock,
- fence_ctx->context, ++fence_ctx->seqno);
-
- job->fence = fence;
-
- return 0;
-}
-
-int rknpu_fence_get_fd(struct rknpu_job *job)
-{
- struct sync_file *sync_file = NULL;
- int fence_fd = -1;
-
- if (!job->fence)
- return -EINVAL;
-
- fence_fd = get_unused_fd_flags(O_CLOEXEC);
- if (fence_fd < 0)
- return fence_fd;
-
- sync_file = sync_file_create(job->fence);
- if (!sync_file)
- return -ENOMEM;
-
- fd_install(fence_fd, sync_file->file);
-
- return fence_fd;
-}
diff --git a/rknpu-driver/driver-0.9.6/rknpu_gem.c b/rknpu-driver/driver-0.9.6/rknpu_gem.c
deleted file mode 100644
index 4cabe13..0000000
--- a/rknpu-driver/driver-0.9.6/rknpu_gem.c
+++ /dev/null
@@ -1,1504 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) Rockchip Electronics Co.Ltd
- * Author: Felix Zeng
- */
-
-#include
-#include
-#include
-#include
-#include
-
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
-#include
-#endif
-
-#include "rknpu_drv.h"
-#include "rknpu_ioctl.h"
-#include "rknpu_gem.h"
-#include "rknpu_iommu.h"
-
-#define RKNPU_GEM_ALLOC_FROM_PAGES 1
-
-#if RKNPU_GEM_ALLOC_FROM_PAGES
-static int rknpu_gem_get_pages(struct rknpu_gem_object *rknpu_obj)
-{
- struct drm_device *drm = rknpu_obj->base.dev;
- struct scatterlist *s = NULL;
- dma_addr_t dma_addr = 0;
- dma_addr_t phys = 0;
- int ret = -EINVAL, i = 0;
-
- rknpu_obj->pages = drm_gem_get_pages(&rknpu_obj->base);
- if (IS_ERR(rknpu_obj->pages)) {
- ret = PTR_ERR(rknpu_obj->pages);
- LOG_ERROR("failed to get pages: %d\n", ret);
- return ret;
- }
-
- rknpu_obj->num_pages = rknpu_obj->size >> PAGE_SHIFT;
-
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
- rknpu_obj->sgt = drm_prime_pages_to_sg(drm, rknpu_obj->pages,
- rknpu_obj->num_pages);
-#else
- rknpu_obj->sgt =
- drm_prime_pages_to_sg(rknpu_obj->pages, rknpu_obj->num_pages);
-#endif
- if (IS_ERR(rknpu_obj->sgt)) {
- ret = PTR_ERR(rknpu_obj->sgt);
- LOG_ERROR("failed to allocate sgt: %d\n", ret);
- goto put_pages;
- }
-
- ret = dma_map_sg(drm->dev, rknpu_obj->sgt->sgl, rknpu_obj->sgt->nents,
- DMA_BIDIRECTIONAL);
- if (ret == 0) {
- ret = -EFAULT;
- LOG_DEV_ERROR(drm->dev, "%s: dma map %zu fail\n", __func__,
- rknpu_obj->size);
- goto free_sgt;
- }
- iommu_flush_iotlb_all(iommu_get_domain_for_dev(drm->dev));
-
- if (rknpu_obj->flags & RKNPU_MEM_KERNEL_MAPPING) {
- rknpu_obj->cookie = vmap(rknpu_obj->pages, rknpu_obj->num_pages,
- VM_MAP, PAGE_KERNEL);
- if (!rknpu_obj->cookie) {
- ret = -ENOMEM;
- LOG_ERROR("failed to vmap: %d\n", ret);
- goto unmap_sg;
- }
- rknpu_obj->kv_addr = rknpu_obj->cookie;
- }
-
- dma_addr = sg_dma_address(rknpu_obj->sgt->sgl);
- rknpu_obj->dma_addr = dma_addr;
-
- for_each_sg(rknpu_obj->sgt->sgl, s, rknpu_obj->sgt->nents, i) {
- dma_addr += s->length;
- phys = sg_phys(s);
- LOG_DEBUG(
- "gem pages alloc sgt[%d], dma_address: %pad, length: %#x, phys: %pad, virt: %p\n",
- i, &dma_addr, s->length, &phys, sg_virt(s));
- }
-
- return 0;
-
-unmap_sg:
- dma_unmap_sg(drm->dev, rknpu_obj->sgt->sgl, rknpu_obj->sgt->nents,
- DMA_BIDIRECTIONAL);
-
-free_sgt:
- sg_free_table(rknpu_obj->sgt);
- kfree(rknpu_obj->sgt);
-
-put_pages:
- drm_gem_put_pages(&rknpu_obj->base, rknpu_obj->pages, false, false);
-
- return ret;
-}
-
-static void rknpu_gem_put_pages(struct rknpu_gem_object *rknpu_obj)
-{
- struct drm_device *drm = rknpu_obj->base.dev;
-
- if (rknpu_obj->flags & RKNPU_MEM_KERNEL_MAPPING) {
- vunmap(rknpu_obj->kv_addr);
- rknpu_obj->kv_addr = NULL;
- }
-
- if (rknpu_obj->sgt != NULL) {
- dma_unmap_sg(drm->dev, rknpu_obj->sgt->sgl,
- rknpu_obj->sgt->nents, DMA_BIDIRECTIONAL);
- sg_free_table(rknpu_obj->sgt);
- kfree(rknpu_obj->sgt);
- }
-
- drm_gem_put_pages(&rknpu_obj->base, rknpu_obj->pages, true, true);
-}
-#endif
-
-static int rknpu_gem_alloc_buf(struct rknpu_gem_object *rknpu_obj)
-{
- struct drm_device *drm = rknpu_obj->base.dev;
- struct rknpu_device *rknpu_dev = drm->dev_private;
- unsigned int nr_pages = 0;
- struct sg_table *sgt = NULL;
- struct scatterlist *s = NULL;
- gfp_t gfp_mask = GFP_KERNEL;
- int ret = -EINVAL, i = 0;
-
- if (rknpu_obj->dma_addr) {
- LOG_DEBUG("buffer already allocated.\n");
- return 0;
- }
-
- rknpu_obj->dma_attrs = 0;
-
- /*
- * if RKNPU_MEM_CONTIGUOUS, fully physically contiguous memory
- * region will be allocated else physically contiguous
- * as possible.
- */
- if (!(rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS))
- rknpu_obj->dma_attrs |= DMA_ATTR_FORCE_CONTIGUOUS;
-
- // cacheable mapping or writecombine mapping
- if (rknpu_obj->flags & RKNPU_MEM_CACHEABLE) {
-#ifdef DMA_ATTR_NON_CONSISTENT
- rknpu_obj->dma_attrs |= DMA_ATTR_NON_CONSISTENT;
-#endif
-#ifdef DMA_ATTR_SYS_CACHE_ONLY
- rknpu_obj->dma_attrs |= DMA_ATTR_SYS_CACHE_ONLY;
-#endif
- } else if (rknpu_obj->flags & RKNPU_MEM_WRITE_COMBINE) {
- rknpu_obj->dma_attrs |= DMA_ATTR_WRITE_COMBINE;
- }
-
- if (!(rknpu_obj->flags & RKNPU_MEM_KERNEL_MAPPING))
- rknpu_obj->dma_attrs |= DMA_ATTR_NO_KERNEL_MAPPING;
-
-#ifdef DMA_ATTR_SKIP_ZEROING
- if (!(rknpu_obj->flags & RKNPU_MEM_ZEROING))
- rknpu_obj->dma_attrs |= DMA_ATTR_SKIP_ZEROING;
-#endif
-
-#if RKNPU_GEM_ALLOC_FROM_PAGES
- if ((rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS) &&
- rknpu_dev->iommu_en) {
- return rknpu_gem_get_pages(rknpu_obj);
- }
-#endif
-
- if (rknpu_obj->flags & RKNPU_MEM_ZEROING)
- gfp_mask |= __GFP_ZERO;
-
- if (!rknpu_dev->iommu_en ||
- rknpu_dev->config->dma_mask <= DMA_BIT_MASK(32) ||
- (rknpu_obj->flags & RKNPU_MEM_DMA32)) {
- gfp_mask &= ~__GFP_HIGHMEM;
- gfp_mask |= __GFP_DMA32;
- }
-
- nr_pages = rknpu_obj->size >> PAGE_SHIFT;
-
- rknpu_obj->pages = rknpu_gem_alloc_page(nr_pages);
- if (!rknpu_obj->pages) {
- LOG_ERROR("failed to allocate pages.\n");
- return -ENOMEM;
- }
-
- rknpu_obj->cookie =
- dma_alloc_attrs(drm->dev, rknpu_obj->size, &rknpu_obj->dma_addr,
- gfp_mask, rknpu_obj->dma_attrs);
- if (!rknpu_obj->cookie) {
- /*
- * when RKNPU_MEM_CONTIGUOUS and IOMMU is available
- * try to fallback to allocate non-contiguous buffer
- */
- if (!(rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS) &&
- rknpu_dev->iommu_en) {
- LOG_DEV_WARN(
- drm->dev,
- "try to fallback to allocate non-contiguous %lu buffer.\n",
- rknpu_obj->size);
- rknpu_obj->dma_attrs &= ~DMA_ATTR_FORCE_CONTIGUOUS;
- rknpu_obj->flags |= RKNPU_MEM_NON_CONTIGUOUS;
- rknpu_obj->cookie =
- dma_alloc_attrs(drm->dev, rknpu_obj->size,
- &rknpu_obj->dma_addr, gfp_mask,
- rknpu_obj->dma_attrs);
- if (!rknpu_obj->cookie) {
- LOG_DEV_ERROR(
- drm->dev,
- "failed to allocate non-contiguous %lu buffer.\n",
- rknpu_obj->size);
- goto err_free;
- }
- } else {
- LOG_DEV_ERROR(drm->dev,
- "failed to allocate %lu buffer.\n",
- rknpu_obj->size);
- goto err_free;
- }
- }
-
- if (rknpu_obj->flags & RKNPU_MEM_KERNEL_MAPPING)
- rknpu_obj->kv_addr = rknpu_obj->cookie;
-
- sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
- if (!sgt) {
- ret = -ENOMEM;
- goto err_free_dma;
- }
-
- ret = dma_get_sgtable_attrs(drm->dev, sgt, rknpu_obj->cookie,
- rknpu_obj->dma_addr, rknpu_obj->size,
- rknpu_obj->dma_attrs);
- if (ret < 0) {
- LOG_DEV_ERROR(drm->dev, "failed to get sgtable.\n");
- goto err_free_sgt;
- }
-
- for_each_sg(sgt->sgl, s, sgt->nents, i) {
- sg_dma_address(s) = sg_phys(s);
- LOG_DEBUG("dma alloc sgt[%d], phys_address: %pad, length: %u\n",
- i, &s->dma_address, s->length);
- }
-
-#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
- ret = drm_prime_sg_to_page_addr_arrays(sgt, rknpu_obj->pages, NULL,
- nr_pages);
-#else
- ret = drm_prime_sg_to_page_array(sgt, rknpu_obj->pages, nr_pages);
-#endif
-
- if (ret < 0) {
- LOG_DEV_ERROR(drm->dev, "invalid sgtable, ret: %d\n", ret);
- goto err_free_sg_table;
- }
-
- rknpu_obj->sgt = sgt;
-
- return ret;
-
-err_free_sg_table:
- sg_free_table(sgt);
-err_free_sgt:
- kfree(sgt);
-err_free_dma:
- dma_free_attrs(drm->dev, rknpu_obj->size, rknpu_obj->cookie,
- rknpu_obj->dma_addr, rknpu_obj->dma_attrs);
-err_free:
- rknpu_gem_free_page(rknpu_obj->pages);
-
- return ret;
-}
-
-static void rknpu_gem_free_buf(struct rknpu_gem_object *rknpu_obj)
-{
- struct drm_device *drm = rknpu_obj->base.dev;
-#if RKNPU_GEM_ALLOC_FROM_PAGES
- struct rknpu_device *rknpu_dev = drm->dev_private;
-#endif
-
- if (!rknpu_obj->dma_addr) {
- LOG_DEBUG("dma handle is invalid.\n");
- return;
- }
-
-#if RKNPU_GEM_ALLOC_FROM_PAGES
- if ((rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS) &&
- rknpu_dev->iommu_en) {
- rknpu_gem_put_pages(rknpu_obj);
- return;
- }
-#endif
-
- sg_free_table(rknpu_obj->sgt);
- kfree(rknpu_obj->sgt);
-
- dma_free_attrs(drm->dev, rknpu_obj->size, rknpu_obj->cookie,
- rknpu_obj->dma_addr, rknpu_obj->dma_attrs);
-
- rknpu_gem_free_page(rknpu_obj->pages);
-
- rknpu_obj->dma_addr = 0;
-}
-
-static int rknpu_gem_handle_create(struct drm_gem_object *obj,
- struct drm_file *file_priv,
- unsigned int *handle)
-{
- int ret = -EINVAL;
- /*
- * allocate a id of idr table where the obj is registered
- * and handle has the id what user can see.
- */
- ret = drm_gem_handle_create(file_priv, obj, handle);
- if (ret)
- return ret;
-
- LOG_DEBUG("gem handle: %#x\n", *handle);
-
- /* drop reference from allocate - handle holds it now. */
- rknpu_gem_object_put(obj);
-
- return 0;
-}
-
-static int rknpu_gem_handle_destroy(struct drm_file *file_priv,
- unsigned int handle)
-{
- return drm_gem_handle_delete(file_priv, handle);
-}
-
-#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
-static const struct vm_operations_struct vm_ops = {
- .fault = rknpu_gem_fault,
- .open = drm_gem_vm_open,
- .close = drm_gem_vm_close,
-};
-
-static const struct drm_gem_object_funcs rknpu_gem_object_funcs = {
- .free = rknpu_gem_free_object,
- .export = drm_gem_prime_export,
- .get_sg_table = rknpu_gem_prime_get_sg_table,
- .vmap = rknpu_gem_prime_vmap,
- .vunmap = rknpu_gem_prime_vunmap,
- .mmap = rknpu_gem_mmap_obj,
- .vm_ops = &vm_ops,
-};
-#endif
-
-static struct rknpu_gem_object *rknpu_gem_init(struct drm_device *drm,
- unsigned long size)
-{
- struct rknpu_device *rknpu_dev = drm->dev_private;
- struct rknpu_gem_object *rknpu_obj = NULL;
- struct drm_gem_object *obj = NULL;
- gfp_t gfp_mask;
- int ret = -EINVAL;
-
- rknpu_obj = kzalloc(sizeof(*rknpu_obj), GFP_KERNEL);
- if (!rknpu_obj)
- return ERR_PTR(-ENOMEM);
-
- obj = &rknpu_obj->base;
-#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
- obj->funcs = &rknpu_gem_object_funcs;
-#endif
-
- ret = drm_gem_object_init(drm, obj, size);
- if (ret < 0) {
- LOG_DEV_ERROR(drm->dev, "failed to initialize gem object\n");
- kfree(rknpu_obj);
- return ERR_PTR(ret);
- }
-
- rknpu_obj->size = rknpu_obj->base.size;
-
- gfp_mask = mapping_gfp_mask(obj->filp->f_mapping);
-
- if (rknpu_obj->flags & RKNPU_MEM_ZEROING)
- gfp_mask |= __GFP_ZERO;
-
- if (!rknpu_dev->iommu_en ||
- rknpu_dev->config->dma_mask <= DMA_BIT_MASK(32) ||
- (rknpu_obj->flags & RKNPU_MEM_DMA32)) {
- gfp_mask &= ~__GFP_HIGHMEM;
- gfp_mask |= __GFP_DMA32;
- }
-
- mapping_set_gfp_mask(obj->filp->f_mapping, gfp_mask);
-
- return rknpu_obj;
-}
-
-static void rknpu_gem_release(struct rknpu_gem_object *rknpu_obj)
-{
- /* release file pointer to gem object. */
- drm_gem_object_release(&rknpu_obj->base);
- kfree(rknpu_obj);
-}
-
-static int rknpu_gem_alloc_buf_with_cache(struct rknpu_gem_object *rknpu_obj,
- enum rknpu_cache_type cache_type)
-{
- struct drm_device *drm = rknpu_obj->base.dev;
- struct rknpu_device *rknpu_dev = drm->dev_private;
- struct iommu_domain *domain = NULL;
- struct rknpu_iommu_dma_cookie *cookie = NULL;
- struct iova_domain *iovad = NULL;
- struct scatterlist *s = NULL;
- unsigned long length = 0;
- unsigned long size = 0;
- unsigned long offset = 0;
- int i = 0;
- int ret = -EINVAL;
- phys_addr_t cache_start = 0;
- unsigned long cache_offset = 0;
- unsigned long cache_size = 0;
-
- switch (cache_type) {
- case RKNPU_CACHE_SRAM:
- cache_start = rknpu_dev->sram_start;
- cache_offset = rknpu_obj->sram_obj->range_start *
- rknpu_dev->sram_mm->chunk_size;
- cache_size = rknpu_obj->sram_size;
- break;
- case RKNPU_CACHE_NBUF:
- cache_start = rknpu_dev->nbuf_start;
- cache_offset = 0;
- cache_size = rknpu_obj->nbuf_size;
- break;
- default:
- LOG_ERROR("Unknown rknpu_cache_type: %d", cache_type);
- return -EINVAL;
- }
-
- /* iova map to cache */
- domain = iommu_get_domain_for_dev(rknpu_dev->dev);
- if (!domain) {
- LOG_ERROR("failed to get iommu domain!");
- return -EINVAL;
- }
-
- cookie = (void *)domain->iova_cookie;
- iovad = &cookie->iovad;
- rknpu_obj->iova_size = iova_align(iovad, cache_size + rknpu_obj->size);
- rknpu_obj->iova_start = rknpu_iommu_dma_alloc_iova(
- domain, rknpu_obj->iova_size, dma_get_mask(drm->dev), drm->dev);
- if (!rknpu_obj->iova_start) {
- LOG_ERROR("iommu_dma_alloc_iova failed\n");
- return -ENOMEM;
- }
-
- LOG_INFO("allocate iova start: %pad, size: %lu\n",
- &rknpu_obj->iova_start, rknpu_obj->iova_size);
-
- /*
- * Overview cache + DDR map to IOVA
- * --------
- * cache_size:
- * - allocate from CACHE, this size value has been page-aligned
- * size: rknpu_obj->size
- * - allocate from DDR pages, this size value has been page-aligned
- * iova_size: rknpu_obj->iova_size
- * - from iova_align(cache_size + size)
- * - it may be larger than the (cache_size + size), and the larger part is not mapped
- * --------
- *
- * |<- cache_size ->| |<- - - - size - - - ->|
- * +---------------+ +----------------------+
- * | CACHE | | DDR |
- * +---------------+ +----------------------+
- * | |
- * | V | V |
- * +---------------------------------------+
- * | IOVA range |
- * +---------------------------------------+
- * |<- - - - - - - iova_size - - - - - - ->|
- *
- */
- ret = iommu_map(domain, rknpu_obj->iova_start,
- cache_start + cache_offset, cache_size,
- IOMMU_READ | IOMMU_WRITE);
- if (ret) {
- LOG_ERROR("cache iommu_map error: %d\n", ret);
- goto free_iova;
- }
-
- rknpu_obj->dma_addr = rknpu_obj->iova_start;
-
- if (rknpu_obj->size == 0) {
- LOG_INFO("allocate cache size: %lu\n", cache_size);
- return 0;
- }
-
- rknpu_obj->pages = drm_gem_get_pages(&rknpu_obj->base);
- if (IS_ERR(rknpu_obj->pages)) {
- ret = PTR_ERR(rknpu_obj->pages);
- LOG_ERROR("failed to get pages: %d\n", ret);
- goto cache_unmap;
- }
-
- rknpu_obj->num_pages = rknpu_obj->size >> PAGE_SHIFT;
-
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
- rknpu_obj->sgt = drm_prime_pages_to_sg(drm, rknpu_obj->pages,
- rknpu_obj->num_pages);
-#else
- rknpu_obj->sgt =
- drm_prime_pages_to_sg(rknpu_obj->pages, rknpu_obj->num_pages);
-#endif
- if (IS_ERR(rknpu_obj->sgt)) {
- ret = PTR_ERR(rknpu_obj->sgt);
- LOG_ERROR("failed to allocate sgt: %d\n", ret);
- goto put_pages;
- }
-
- length = rknpu_obj->size;
- offset = rknpu_obj->iova_start + cache_size;
-
- for_each_sg(rknpu_obj->sgt->sgl, s, rknpu_obj->sgt->nents, i) {
- size = (length < s->length) ? length : s->length;
-
- ret = iommu_map(domain, offset, sg_phys(s), size,
- IOMMU_READ | IOMMU_WRITE);
- if (ret) {
- LOG_ERROR("ddr iommu_map error: %d\n", ret);
- goto sgl_unmap;
- }
-
- length -= size;
- offset += size;
-
- if (length == 0)
- break;
- }
-
- LOG_INFO("allocate size: %lu with cache size: %lu\n", rknpu_obj->size,
- cache_size);
-
- return 0;
-
-sgl_unmap:
- iommu_unmap(domain, rknpu_obj->iova_start + cache_size,
- rknpu_obj->size - length);
- sg_free_table(rknpu_obj->sgt);
- kfree(rknpu_obj->sgt);
-
-put_pages:
- drm_gem_put_pages(&rknpu_obj->base, rknpu_obj->pages, false, false);
-
-cache_unmap:
- iommu_unmap(domain, rknpu_obj->iova_start, cache_size);
-
-free_iova:
- rknpu_iommu_dma_free_iova((void *)domain->iova_cookie,
- rknpu_obj->iova_start, rknpu_obj->iova_size);
-
- return ret;
-}
-
-static void rknpu_gem_free_buf_with_cache(struct rknpu_gem_object *rknpu_obj,
- enum rknpu_cache_type cache_type)
-{
- struct drm_device *drm = rknpu_obj->base.dev;
- struct rknpu_device *rknpu_dev = drm->dev_private;
- struct iommu_domain *domain = NULL;
- unsigned long cache_size = 0;
-
- switch (cache_type) {
- case RKNPU_CACHE_SRAM:
- cache_size = rknpu_obj->sram_size;
- break;
- case RKNPU_CACHE_NBUF:
- cache_size = rknpu_obj->nbuf_size;
- break;
- default:
- LOG_ERROR("Unknown rknpu_cache_type: %d", cache_type);
- return;
- }
-
- domain = iommu_get_domain_for_dev(rknpu_dev->dev);
- if (domain) {
- iommu_unmap(domain, rknpu_obj->iova_start, cache_size);
- if (rknpu_obj->size > 0)
- iommu_unmap(domain, rknpu_obj->iova_start + cache_size,
- rknpu_obj->size);
- rknpu_iommu_dma_free_iova((void *)domain->iova_cookie,
- rknpu_obj->iova_start,
- rknpu_obj->iova_size);
- }
-
- if (rknpu_obj->pages)
- drm_gem_put_pages(&rknpu_obj->base, rknpu_obj->pages, true,
- true);
-
- if (rknpu_obj->sgt != NULL) {
- sg_free_table(rknpu_obj->sgt);
- kfree(rknpu_obj->sgt);
- }
-}
-
-struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm,
- unsigned int flags,
- unsigned long size,
- unsigned long sram_size,
- int iommu_domain_id)
-{
- struct rknpu_device *rknpu_dev = drm->dev_private;
- struct rknpu_gem_object *rknpu_obj = NULL;
- size_t remain_ddr_size = 0;
- int ret = -EINVAL;
-
- if (!size) {
- LOG_DEV_ERROR(drm->dev, "invalid buffer size: %lu\n", size);
- return ERR_PTR(-EINVAL);
- }
-
- remain_ddr_size = round_up(size, PAGE_SIZE);
-
- rknpu_obj = rknpu_gem_init(drm, remain_ddr_size);
- if (IS_ERR(rknpu_obj))
- return rknpu_obj;
-
- if (!rknpu_iommu_switch_domain(rknpu_dev, iommu_domain_id))
- rknpu_obj->iommu_domain_id = iommu_domain_id;
-
- if (!rknpu_dev->iommu_en && (flags & RKNPU_MEM_NON_CONTIGUOUS)) {
- /*
- * when no IOMMU is available, all allocated buffers are
- * contiguous anyway, so drop RKNPU_MEM_NON_CONTIGUOUS flag
- */
- flags &= ~RKNPU_MEM_NON_CONTIGUOUS;
- LOG_WARN(
- "non-contiguous allocation is not supported without IOMMU, falling back to contiguous buffer\n");
- }
-
- if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) &&
- (flags & RKNPU_MEM_TRY_ALLOC_SRAM) && rknpu_dev->sram_size > 0) {
- size_t sram_free_size = 0;
- size_t real_sram_size = 0;
-
- if (sram_size != 0)
- sram_size = round_up(sram_size, PAGE_SIZE);
-
- /* set memory type and cache attribute from user side. */
- rknpu_obj->flags = flags;
-
- sram_free_size = rknpu_dev->sram_mm->free_chunks *
- rknpu_dev->sram_mm->chunk_size;
- if (sram_free_size > 0) {
- real_sram_size = remain_ddr_size;
- if (sram_size != 0 && remain_ddr_size > sram_size)
- real_sram_size = sram_size;
- if (real_sram_size > sram_free_size)
- real_sram_size = sram_free_size;
- ret = rknpu_mm_alloc(rknpu_dev->sram_mm, real_sram_size,
- &rknpu_obj->sram_obj);
- if (ret != 0) {
- sram_free_size =
- rknpu_dev->sram_mm->free_chunks *
- rknpu_dev->sram_mm->chunk_size;
- LOG_WARN(
- "mm allocate %zu failed, ret: %d, free size: %zu\n",
- real_sram_size, ret, sram_free_size);
- real_sram_size = 0;
- }
- }
-
- if (real_sram_size > 0) {
- rknpu_obj->sram_size = real_sram_size;
-
- ret = rknpu_gem_alloc_buf_with_cache(rknpu_obj,
- RKNPU_CACHE_SRAM);
- if (ret < 0)
- goto mm_free;
- remain_ddr_size = 0;
- }
- } else if (IS_ENABLED(CONFIG_NO_GKI) &&
- (flags & RKNPU_MEM_TRY_ALLOC_NBUF) &&
- rknpu_dev->nbuf_size > 0) {
- size_t nbuf_size = remain_ddr_size <= rknpu_dev->nbuf_size ?
- remain_ddr_size :
- rknpu_dev->nbuf_size;
-
- /* set memory type and cache attribute from user side. */
- rknpu_obj->flags = flags;
-
- if (nbuf_size > 0) {
- rknpu_obj->nbuf_size = nbuf_size;
-
- ret = rknpu_gem_alloc_buf_with_cache(rknpu_obj,
- RKNPU_CACHE_NBUF);
- if (ret < 0)
- goto gem_release;
- remain_ddr_size = 0;
- }
- }
-
- if (remain_ddr_size > 0) {
- /* set memory type and cache attribute from user side. */
- rknpu_obj->flags = flags;
-
- ret = rknpu_gem_alloc_buf(rknpu_obj);
- if (ret < 0)
- goto gem_release;
- }
-
- LOG_DEBUG(
- "created dma addr: %pad, cookie: %p, ddr size: %lu, sram size: %lu, nbuf size: %lu, attrs: %#lx, flags: %#x, iommu domain id: %d\n",
- &rknpu_obj->dma_addr, rknpu_obj->cookie, rknpu_obj->size,
- rknpu_obj->sram_size, rknpu_obj->nbuf_size,
- rknpu_obj->dma_attrs, rknpu_obj->flags,
- rknpu_obj->iommu_domain_id);
-
- return rknpu_obj;
-
-mm_free:
- if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) &&
- rknpu_obj->sram_obj != NULL)
- rknpu_mm_free(rknpu_dev->sram_mm, rknpu_obj->sram_obj);
-
-gem_release:
- rknpu_gem_release(rknpu_obj);
-
- return ERR_PTR(ret);
-}
-
-void rknpu_gem_object_destroy(struct rknpu_gem_object *rknpu_obj)
-{
- struct drm_gem_object *obj = &rknpu_obj->base;
- struct rknpu_device *rknpu_dev = obj->dev->dev_private;
-
- LOG_DEBUG(
- "destroy dma addr: %pad, cookie: %p, size: %lu, attrs: %#lx, flags: %#x, handle count: %d\n",
- &rknpu_obj->dma_addr, rknpu_obj->cookie, rknpu_obj->size,
- rknpu_obj->dma_attrs, rknpu_obj->flags, obj->handle_count);
-
- rknpu_iommu_switch_domain(rknpu_dev, rknpu_obj->iommu_domain_id);
-
- /*
- * do not release memory region from exporter.
- *
- * the region will be released by exporter
- * once dmabuf's refcount becomes 0.
- */
- if (obj->import_attach) {
- drm_prime_gem_destroy(obj, rknpu_obj->sgt);
- rknpu_gem_free_page(rknpu_obj->pages);
- } else {
- if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) &&
- rknpu_obj->sram_size > 0) {
- if (rknpu_obj->sram_obj != NULL)
- rknpu_mm_free(rknpu_dev->sram_mm,
- rknpu_obj->sram_obj);
- rknpu_gem_free_buf_with_cache(rknpu_obj,
- RKNPU_CACHE_SRAM);
- } else if (IS_ENABLED(CONFIG_NO_GKI) &&
- rknpu_obj->nbuf_size > 0) {
- rknpu_gem_free_buf_with_cache(rknpu_obj,
- RKNPU_CACHE_NBUF);
- } else {
- rknpu_gem_free_buf(rknpu_obj);
- }
- }
-
- rknpu_gem_release(rknpu_obj);
-}
-
-int rknpu_gem_create_ioctl(struct drm_device *drm, void *data,
- struct drm_file *file_priv)
-{
- struct rknpu_mem_create *args = data;
- struct rknpu_gem_object *rknpu_obj = NULL;
- int ret = -EINVAL;
-
- rknpu_obj = rknpu_gem_object_find(file_priv, args->handle);
- if (!rknpu_obj) {
- rknpu_obj = rknpu_gem_object_create(drm, args->flags,
- args->size, args->sram_size,
- args->iommu_domain_id);
- if (IS_ERR(rknpu_obj))
- return PTR_ERR(rknpu_obj);
-
- ret = rknpu_gem_handle_create(&rknpu_obj->base, file_priv,
- &args->handle);
- if (ret) {
- rknpu_gem_object_destroy(rknpu_obj);
- return ret;
- }
- }
-
- // rknpu_gem_object_get(&rknpu_obj->base);
-
- args->size = rknpu_obj->size;
- args->sram_size = rknpu_obj->sram_size;
- args->obj_addr = (__u64)(uintptr_t)rknpu_obj;
- args->dma_addr = rknpu_obj->dma_addr;
-
- return 0;
-}
-
-int rknpu_gem_map_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file_priv)
-{
- struct rknpu_mem_map *args = data;
-
-#if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE
- return rknpu_gem_dumb_map_offset(file_priv, dev, args->handle,
- &args->offset);
-#else
- return drm_gem_dumb_map_offset(file_priv, dev, args->handle,
- &args->offset);
-#endif
-}
-
-int rknpu_gem_destroy_ioctl(struct drm_device *drm, void *data,
- struct drm_file *file_priv)
-{
- struct rknpu_device *rknpu_dev = drm->dev_private;
- struct rknpu_gem_object *rknpu_obj = NULL;
- struct rknpu_mem_destroy *args = data;
-
- rknpu_obj = rknpu_gem_object_find(file_priv, args->handle);
- if (!rknpu_obj)
- return -EINVAL;
-
- rknpu_iommu_switch_domain(rknpu_dev, rknpu_obj->iommu_domain_id);
-
- // rknpu_gem_object_put(&rknpu_obj->base);
-
- return rknpu_gem_handle_destroy(file_priv, args->handle);
-}
-
-#if RKNPU_GEM_ALLOC_FROM_PAGES
-/*
- * __vm_map_pages - maps range of kernel pages into user vma
- * @vma: user vma to map to
- * @pages: pointer to array of source kernel pages
- * @num: number of pages in page array
- * @offset: user's requested vm_pgoff
- *
- * This allows drivers to map range of kernel pages into a user vma.
- *
- * Return: 0 on success and error code otherwise.
- */
-static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages,
- unsigned long num, unsigned long offset)
-{
- unsigned long count = vma_pages(vma);
- unsigned long uaddr = vma->vm_start;
- int ret = -EINVAL, i = 0;
-
- /* Fail if the user requested offset is beyond the end of the object */
- if (offset >= num)
- return -ENXIO;
-
- /* Fail if the user requested size exceeds available object size */
- if (count > num - offset)
- return -ENXIO;
-
- for (i = 0; i < count; i++) {
- ret = vm_insert_page(vma, uaddr, pages[offset + i]);
- if (ret < 0)
- return ret;
- uaddr += PAGE_SIZE;
- }
-
- return 0;
-}
-
-static int rknpu_gem_mmap_pages(struct rknpu_gem_object *rknpu_obj,
- struct vm_area_struct *vma)
-{
- struct drm_device *drm = rknpu_obj->base.dev;
- int ret = -EINVAL;
-
- vm_flags_set(vma, VM_MIXEDMAP);
-
- ret = __vm_map_pages(vma, rknpu_obj->pages, rknpu_obj->num_pages,
- vma->vm_pgoff);
- if (ret < 0)
- LOG_DEV_ERROR(drm->dev, "failed to map pages into vma: %d\n",
- ret);
-
- return ret;
-}
-#endif
-
-static int rknpu_gem_mmap_cache(struct rknpu_gem_object *rknpu_obj,
- struct vm_area_struct *vma,
- enum rknpu_cache_type cache_type)
-{
- struct drm_device *drm = rknpu_obj->base.dev;
-#if RKNPU_GEM_ALLOC_FROM_PAGES
- struct rknpu_device *rknpu_dev = drm->dev_private;
-#endif
- unsigned long vm_size = 0;
- int ret = -EINVAL;
- unsigned long offset = 0;
- unsigned long num_pages = 0;
- int i = 0;
- phys_addr_t cache_start = 0;
- unsigned long cache_offset = 0;
- unsigned long cache_size = 0;
-
- switch (cache_type) {
- case RKNPU_CACHE_SRAM:
- cache_start = rknpu_dev->sram_start;
- cache_offset = rknpu_obj->sram_obj->range_start *
- rknpu_dev->sram_mm->chunk_size;
- cache_size = rknpu_obj->sram_size;
- break;
- case RKNPU_CACHE_NBUF:
- cache_start = rknpu_dev->nbuf_start;
- cache_offset = 0;
- cache_size = rknpu_obj->nbuf_size;
- break;
- default:
- LOG_ERROR("Unknown rknpu_cache_type: %d", cache_type);
- return -EINVAL;
- }
-
- vm_flags_set(vma, VM_MIXEDMAP);
-
- vm_size = vma->vm_end - vma->vm_start;
-
- /*
- * Convert a physical address in a cache area to a page frame number (PFN),
- * and store the resulting PFN in the vm_pgoff field of the given VMA.
- *
- * NOTE: This conversion carries a risk because the resulting PFN is not a true
- * page frame number and may not be valid or usable in all contexts.
- */
- vma->vm_pgoff = __phys_to_pfn(cache_start + cache_offset);
-
- ret = remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, cache_size,
- vma->vm_page_prot);
- if (ret)
- return -EAGAIN;
-
- if (rknpu_obj->size == 0)
- return 0;
-
- offset = cache_size;
-
- num_pages = (vm_size - cache_size) / PAGE_SIZE;
- for (i = 0; i < num_pages; ++i) {
- ret = vm_insert_page(vma, vma->vm_start + offset,
- rknpu_obj->pages[i]);
- if (ret < 0)
- return ret;
- offset += PAGE_SIZE;
- }
-
- return 0;
-}
-
-static int rknpu_gem_mmap_buffer(struct rknpu_gem_object *rknpu_obj,
- struct vm_area_struct *vma)
-{
- struct drm_device *drm = rknpu_obj->base.dev;
-#if RKNPU_GEM_ALLOC_FROM_PAGES
- struct rknpu_device *rknpu_dev = drm->dev_private;
-#endif
- unsigned long vm_size = 0;
- int ret = -EINVAL;
-
- /*
- * clear the VM_PFNMAP flag that was set by drm_gem_mmap(), and set the
- * vm_pgoff (used as a fake buffer offset by DRM) to 0 as we want to map
- * the whole buffer.
- */
- vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO);
- vm_flags_clear(vma, VM_PFNMAP);
- vma->vm_pgoff = 0;
-
- vm_size = vma->vm_end - vma->vm_start;
-
- /* check if user-requested size is valid. */
- if (vm_size > rknpu_obj->size)
- return -EINVAL;
-
- if (rknpu_obj->sram_size > 0)
- return rknpu_gem_mmap_cache(rknpu_obj, vma, RKNPU_CACHE_SRAM);
- else if (rknpu_obj->nbuf_size > 0)
- return rknpu_gem_mmap_cache(rknpu_obj, vma, RKNPU_CACHE_NBUF);
-
-#if RKNPU_GEM_ALLOC_FROM_PAGES
- if ((rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS) &&
- rknpu_dev->iommu_en) {
- return rknpu_gem_mmap_pages(rknpu_obj, vma);
- }
-#endif
-
- ret = dma_mmap_attrs(drm->dev, vma, rknpu_obj->cookie,
- rknpu_obj->dma_addr, rknpu_obj->size,
- rknpu_obj->dma_attrs);
- if (ret < 0) {
- LOG_DEV_ERROR(drm->dev, "failed to mmap, ret: %d\n", ret);
- return ret;
- }
-
- return 0;
-}
-
-void rknpu_gem_free_object(struct drm_gem_object *obj)
-{
- rknpu_gem_object_destroy(to_rknpu_obj(obj));
-}
-
-int rknpu_gem_dumb_create(struct drm_file *file_priv, struct drm_device *drm,
- struct drm_mode_create_dumb *args)
-{
- struct rknpu_device *rknpu_dev = drm->dev_private;
- struct rknpu_gem_object *rknpu_obj = NULL;
- unsigned int flags = 0;
- int ret = -EINVAL;
-
- /*
- * allocate memory to be used for framebuffer.
- * - this callback would be called by user application
- * with DRM_IOCTL_MODE_CREATE_DUMB command.
- */
- args->pitch = args->width * ((args->bpp + 7) / 8);
- args->size = args->pitch * args->height;
-
- if (rknpu_dev->iommu_en)
- flags = RKNPU_MEM_NON_CONTIGUOUS | RKNPU_MEM_WRITE_COMBINE;
- else
- flags = RKNPU_MEM_CONTIGUOUS | RKNPU_MEM_WRITE_COMBINE;
-
- rknpu_obj = rknpu_gem_object_create(drm, flags, args->size, 0, 0);
- if (IS_ERR(rknpu_obj)) {
- LOG_DEV_ERROR(drm->dev, "gem object allocate failed.\n");
- return PTR_ERR(rknpu_obj);
- }
-
- ret = rknpu_gem_handle_create(&rknpu_obj->base, file_priv,
- &args->handle);
- if (ret) {
- rknpu_gem_object_destroy(rknpu_obj);
- return ret;
- }
-
- return 0;
-}
-
-#if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE
-int rknpu_gem_dumb_map_offset(struct drm_file *file_priv,
- struct drm_device *drm, uint32_t handle,
- uint64_t *offset)
-{
- struct rknpu_gem_object *rknpu_obj = NULL;
- struct drm_gem_object *obj = NULL;
- int ret = -EINVAL;
-
- rknpu_obj = rknpu_gem_object_find(file_priv, handle);
- if (!rknpu_obj)
- return 0;
-
- /* Don't allow imported objects to be mapped */
- obj = &rknpu_obj->base;
- if (obj->import_attach)
- return -EINVAL;
-
- ret = drm_gem_create_mmap_offset(obj);
- if (ret)
- return ret;
-
- *offset = drm_vma_node_offset_addr(&obj->vma_node);
-
- return 0;
-}
-#endif
-
-#if KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE
-vm_fault_t rknpu_gem_fault(struct vm_fault *vmf)
-{
- struct vm_area_struct *vma = vmf->vma;
- struct drm_gem_object *obj = vma->vm_private_data;
- struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
- struct drm_device *drm = rknpu_obj->base.dev;
- unsigned long pfn = 0;
- pgoff_t page_offset = 0;
-
- page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
-
- if (page_offset >= (rknpu_obj->size >> PAGE_SHIFT)) {
- LOG_DEV_ERROR(drm->dev, "invalid page offset\n");
- return VM_FAULT_SIGBUS;
- }
-
- pfn = page_to_pfn(rknpu_obj->pages[page_offset]);
- return vmf_insert_mixed(vma, vmf->address,
- __pfn_to_pfn_t(pfn, PFN_DEV));
-}
-#elif KERNEL_VERSION(4, 14, 0) <= LINUX_VERSION_CODE
-int rknpu_gem_fault(struct vm_fault *vmf)
-{
- struct vm_area_struct *vma = vmf->vma;
- struct drm_gem_object *obj = vma->vm_private_data;
- struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
- struct drm_device *drm = rknpu_obj->base.dev;
- unsigned long pfn = 0;
- pgoff_t page_offset = 0;
- int ret = -EINVAL;
-
- page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
-
- if (page_offset >= (rknpu_obj->size >> PAGE_SHIFT)) {
- LOG_DEV_ERROR(drm->dev, "invalid page offset\n");
- ret = -EINVAL;
- goto out;
- }
-
- pfn = page_to_pfn(rknpu_obj->pages[page_offset]);
- ret = vm_insert_mixed(vma, vmf->address, __pfn_to_pfn_t(pfn, PFN_DEV));
-
-out:
- switch (ret) {
- case 0:
- case -ERESTARTSYS:
- case -EINTR:
- return VM_FAULT_NOPAGE;
- case -ENOMEM:
- return VM_FAULT_OOM;
- default:
- return VM_FAULT_SIGBUS;
- }
-}
-#else
-int rknpu_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
-{
- struct drm_gem_object *obj = vma->vm_private_data;
- struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
- struct drm_device *drm = rknpu_obj->base.dev;
- unsigned long pfn = 0;
- pgoff_t page_offset = 0;
- int ret = -EINVAL;
-
- page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
- PAGE_SHIFT;
-
- if (page_offset >= (rknpu_obj->size >> PAGE_SHIFT)) {
- LOG_DEV_ERROR(drm->dev, "invalid page offset\n");
- ret = -EINVAL;
- goto out;
- }
-
- pfn = page_to_pfn(rknpu_obj->pages[page_offset]);
- ret = vm_insert_mixed(vma, (unsigned long)vmf->virtual_address,
- __pfn_to_pfn_t(pfn, PFN_DEV));
-
-out:
- switch (ret) {
- case 0:
- case -ERESTARTSYS:
- case -EINTR:
- return VM_FAULT_NOPAGE;
- case -ENOMEM:
- return VM_FAULT_OOM;
- default:
- return VM_FAULT_SIGBUS;
- }
-}
-#endif
-
-int rknpu_gem_mmap_obj(struct drm_gem_object *obj, struct vm_area_struct *vma)
-{
- struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
- int ret = -EINVAL;
-
- LOG_DEBUG("flags: %#x\n", rknpu_obj->flags);
-
- /* non-cacheable as default. */
- if (rknpu_obj->flags & RKNPU_MEM_CACHEABLE) {
- vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
- } else if (rknpu_obj->flags & RKNPU_MEM_WRITE_COMBINE) {
- vma->vm_page_prot =
- pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
- } else {
- vma->vm_page_prot =
- pgprot_noncached(vm_get_page_prot(vma->vm_flags));
- }
-
- ret = rknpu_gem_mmap_buffer(rknpu_obj, vma);
- if (ret)
- goto err_close_vm;
-
- return 0;
-
-err_close_vm:
- drm_gem_vm_close(vma);
-
- return ret;
-}
-
-int rknpu_gem_mmap(struct file *filp, struct vm_area_struct *vma)
-{
- struct drm_gem_object *obj = NULL;
- int ret = -EINVAL;
-
- /* set vm_area_struct. */
- ret = drm_gem_mmap(filp, vma);
- if (ret < 0) {
- LOG_ERROR("failed to mmap, ret: %d\n", ret);
- return ret;
- }
-
- obj = vma->vm_private_data;
-
- if (obj->import_attach)
- return dma_buf_mmap(obj->dma_buf, vma, 0);
-
- return rknpu_gem_mmap_obj(obj, vma);
-}
-
-/* low-level interface prime helpers */
-#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
-struct drm_gem_object *rknpu_gem_prime_import(struct drm_device *dev,
- struct dma_buf *dma_buf)
-{
- return drm_gem_prime_import_dev(dev, dma_buf, dev->dev);
-}
-#endif
-
-struct sg_table *rknpu_gem_prime_get_sg_table(struct drm_gem_object *obj)
-{
- struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
- int npages = 0;
-
- npages = rknpu_obj->size >> PAGE_SHIFT;
-
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
- return drm_prime_pages_to_sg(obj->dev, rknpu_obj->pages, npages);
-#else
- return drm_prime_pages_to_sg(rknpu_obj->pages, npages);
-#endif
-}
-
-struct drm_gem_object *
-rknpu_gem_prime_import_sg_table(struct drm_device *dev,
- struct dma_buf_attachment *attach,
- struct sg_table *sgt)
-{
- struct rknpu_gem_object *rknpu_obj = NULL;
- int npages = 0;
- int ret = -EINVAL;
-
- rknpu_obj = rknpu_gem_init(dev, attach->dmabuf->size);
- if (IS_ERR(rknpu_obj)) {
- ret = PTR_ERR(rknpu_obj);
- return ERR_PTR(ret);
- }
-
- rknpu_obj->dma_addr = sg_dma_address(sgt->sgl);
-
- npages = rknpu_obj->size >> PAGE_SHIFT;
- rknpu_obj->pages = rknpu_gem_alloc_page(npages);
- if (!rknpu_obj->pages) {
- ret = -ENOMEM;
- goto err;
- }
-
-#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
- ret = drm_prime_sg_to_page_addr_arrays(sgt, rknpu_obj->pages, NULL,
- npages);
-#else
- ret = drm_prime_sg_to_page_array(sgt, rknpu_obj->pages, npages);
-#endif
- if (ret < 0)
- goto err_free_large;
-
- rknpu_obj->sgt = sgt;
-
- if (sgt->nents == 1) {
- /* always physically continuous memory if sgt->nents is 1. */
- rknpu_obj->flags |= RKNPU_MEM_CONTIGUOUS;
- } else {
- /*
- * this case could be CONTIG or NONCONTIG type but for now
- * sets NONCONTIG.
- * TODO. we have to find a way that exporter can notify
- * the type of its own buffer to importer.
- */
- rknpu_obj->flags |= RKNPU_MEM_NON_CONTIGUOUS;
- }
-
- return &rknpu_obj->base;
-
-err_free_large:
- rknpu_gem_free_page(rknpu_obj->pages);
-err:
- rknpu_gem_release(rknpu_obj);
- return ERR_PTR(ret);
-}
-
-#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
-void *rknpu_gem_prime_vmap(struct drm_gem_object *obj)
-{
- struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
-
- if (!rknpu_obj->pages)
- return NULL;
-
- return vmap(rknpu_obj->pages, rknpu_obj->num_pages, VM_MAP,
- PAGE_KERNEL);
-}
-
-void rknpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr)
-{
- vunmap(vaddr);
-}
-#else
-int rknpu_gem_prime_vmap(struct drm_gem_object *obj, struct iosys_map *map)
-{
- struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
- void *vaddr = NULL;
-
- if (!rknpu_obj->pages)
- return -EINVAL;
-
- vaddr = vmap(rknpu_obj->pages, rknpu_obj->num_pages, VM_MAP,
- PAGE_KERNEL);
- if (!vaddr)
- return -ENOMEM;
-
- iosys_map_set_vaddr(map, vaddr);
-
- return 0;
-}
-
-void rknpu_gem_prime_vunmap(struct drm_gem_object *obj, struct iosys_map *map)
-{
- struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
-
- if (rknpu_obj->pages) {
- vunmap(map->vaddr);
- map->vaddr = NULL;
- }
-}
-#endif
-
-int rknpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
-{
- int ret = -EINVAL;
-
- ret = drm_gem_mmap_obj(obj, obj->size, vma);
- if (ret < 0)
- return ret;
-
- return rknpu_gem_mmap_obj(obj, vma);
-}
-
-static int rknpu_cache_sync(struct rknpu_gem_object *rknpu_obj,
- unsigned long *length, unsigned long *offset,
- enum rknpu_cache_type cache_type)
-{
-#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
- struct drm_gem_object *obj = &rknpu_obj->base;
- struct rknpu_device *rknpu_dev = obj->dev->dev_private;
- void __iomem *cache_base_io = NULL;
- unsigned long cache_offset = 0;
- unsigned long cache_size = 0;
-
- switch (cache_type) {
- case RKNPU_CACHE_SRAM:
- cache_base_io = rknpu_dev->sram_base_io;
- cache_offset = rknpu_obj->sram_obj->range_start *
- rknpu_dev->sram_mm->chunk_size;
- cache_size = rknpu_obj->sram_size;
- break;
- case RKNPU_CACHE_NBUF:
- cache_base_io = rknpu_dev->nbuf_base_io;
- cache_offset = 0;
- cache_size = rknpu_obj->nbuf_size;
- break;
- default:
- LOG_ERROR("Unknown rknpu_cache_type: %d", cache_type);
- return -EINVAL;
- }
-
- if ((*offset + *length) <= cache_size) {
- __dma_map_area(cache_base_io + *offset + cache_offset, *length,
- DMA_TO_DEVICE);
- __dma_unmap_area(cache_base_io + *offset + cache_offset,
- *length, DMA_FROM_DEVICE);
- *length = 0;
- *offset = 0;
- } else if (*offset >= cache_size) {
- *offset -= cache_size;
- } else {
- unsigned long cache_length = cache_size - *offset;
-
- __dma_map_area(cache_base_io + *offset + cache_offset,
- cache_length, DMA_TO_DEVICE);
- __dma_unmap_area(cache_base_io + *offset + cache_offset,
- cache_length, DMA_FROM_DEVICE);
- *length -= cache_length;
- *offset = 0;
- }
-#endif
-
- return 0;
-}
-
-int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file_priv)
-{
- struct rknpu_gem_object *rknpu_obj = NULL;
- struct rknpu_device *rknpu_dev = dev->dev_private;
- struct rknpu_mem_sync *args = data;
- struct scatterlist *sg;
- dma_addr_t sg_phys_addr;
- unsigned long length, offset = 0;
- unsigned long sg_offset, sg_left, size = 0;
- unsigned long len = 0;
- int i;
-
- rknpu_obj = (struct rknpu_gem_object *)(uintptr_t)args->obj_addr;
- if (!rknpu_obj)
- return -EINVAL;
-
- if (!(rknpu_obj->flags & RKNPU_MEM_CACHEABLE))
- return -EINVAL;
-
- if (!(rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS)) {
- if (args->flags & RKNPU_MEM_SYNC_TO_DEVICE) {
- dma_sync_single_range_for_device(
- dev->dev, rknpu_obj->dma_addr, args->offset,
- args->size, DMA_TO_DEVICE);
- }
- if (args->flags & RKNPU_MEM_SYNC_FROM_DEVICE) {
- dma_sync_single_range_for_cpu(dev->dev,
- rknpu_obj->dma_addr,
- args->offset, args->size,
- DMA_FROM_DEVICE);
- }
- } else {
- WARN_ON(!rknpu_dev->fake_dev);
-
- length = args->size;
- offset = args->offset;
-
- if (IS_ENABLED(CONFIG_NO_GKI) &&
- IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) &&
- rknpu_obj->sram_size > 0) {
- rknpu_cache_sync(rknpu_obj, &length, &offset,
- RKNPU_CACHE_SRAM);
- } else if (IS_ENABLED(CONFIG_NO_GKI) &&
- rknpu_obj->nbuf_size > 0) {
- rknpu_cache_sync(rknpu_obj, &length, &offset,
- RKNPU_CACHE_NBUF);
- }
-
- for_each_sg(rknpu_obj->sgt->sgl, sg, rknpu_obj->sgt->nents,
- i) {
- if (length == 0)
- break;
-
- len += sg->length;
- if (len <= offset)
- continue;
-
- sg_phys_addr = sg_phys(sg);
-
- sg_left = len - offset;
- sg_offset = sg->length - sg_left;
-
- size = (length < sg_left) ? length : sg_left;
-
- if (args->flags & RKNPU_MEM_SYNC_TO_DEVICE) {
- dma_sync_single_range_for_device(
- rknpu_dev->fake_dev, sg_phys_addr,
- sg_offset, size, DMA_TO_DEVICE);
- }
-
- if (args->flags & RKNPU_MEM_SYNC_FROM_DEVICE) {
- dma_sync_single_range_for_cpu(
- rknpu_dev->fake_dev, sg_phys_addr,
- sg_offset, size, DMA_FROM_DEVICE);
- }
-
- offset += size;
- length -= size;
- }
- }
-
- return 0;
-}
diff --git a/rknpu-driver/driver-0.9.6/rknpu_iommu.c b/rknpu-driver/driver-0.9.6/rknpu_iommu.c
deleted file mode 100644
index 53bca78..0000000
--- a/rknpu-driver/driver-0.9.6/rknpu_iommu.c
+++ /dev/null
@@ -1,254 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) Rockchip Electronics Co.Ltd
- * Author: Felix Zeng
- */
-
-#include "rknpu_iommu.h"
-
-dma_addr_t rknpu_iommu_dma_alloc_iova(struct iommu_domain *domain, size_t size,
- u64 dma_limit, struct device *dev)
-{
- struct rknpu_iommu_dma_cookie *cookie = (void *)domain->iova_cookie;
- struct iova_domain *iovad = &cookie->iovad;
- unsigned long shift, iova_len, iova = 0;
-#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE)
- dma_addr_t limit;
-#endif
-
- shift = iova_shift(iovad);
- iova_len = size >> shift;
-
-#if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
- /*
- * Freeing non-power-of-two-sized allocations back into the IOVA caches
- * will come back to bite us badly, so we have to waste a bit of space
- * rounding up anything cacheable to make sure that can't happen. The
- * order of the unadjusted size will still match upon freeing.
- */
- if (iova_len < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1)))
- iova_len = roundup_pow_of_two(iova_len);
-#endif
-
-#if (KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE)
- dma_limit = min_not_zero(dma_limit, dev->bus_dma_limit);
-#else
- if (dev->bus_dma_mask)
- dma_limit &= dev->bus_dma_mask;
-#endif
-
- if (domain->geometry.force_aperture)
- dma_limit =
- min_t(u64, dma_limit, domain->geometry.aperture_end);
-
-#if (KERNEL_VERSION(5, 4, 0) <= LINUX_VERSION_CODE)
- iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift, true);
-#else
- limit = min_t(dma_addr_t, dma_limit >> shift, iovad->end_pfn);
-
- iova = alloc_iova_fast(iovad, iova_len, limit, true);
-#endif
-
- return (dma_addr_t)iova << shift;
-}
-
-void rknpu_iommu_dma_free_iova(struct rknpu_iommu_dma_cookie *cookie,
- dma_addr_t iova, size_t size)
-{
- struct iova_domain *iovad = &cookie->iovad;
-
- free_iova_fast(iovad, iova_pfn(iovad, iova), size >> iova_shift(iovad));
-}
-
-#if defined(CONFIG_IOMMU_API) && defined(CONFIG_NO_GKI)
-
-#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
-struct iommu_group {
- struct kobject kobj;
- struct kobject *devices_kobj;
- struct list_head devices;
-#ifdef __ANDROID_COMMON_KERNEL__
- struct xarray pasid_array;
-#endif
- struct mutex mutex;
- void *iommu_data;
- void (*iommu_data_release)(void *iommu_data);
- char *name;
- int id;
- struct iommu_domain *default_domain;
- struct iommu_domain *blocking_domain;
- struct iommu_domain *domain;
- struct list_head entry;
- unsigned int owner_cnt;
- void *owner;
-};
-#else
-struct iommu_group {
- struct kobject kobj;
- struct kobject *devices_kobj;
- struct list_head devices;
- struct mutex mutex;
- struct blocking_notifier_head notifier;
- void *iommu_data;
- void (*iommu_data_release)(void *iommu_data);
- char *name;
- int id;
- struct iommu_domain *default_domain;
- struct iommu_domain *domain;
- struct list_head entry;
-};
-#endif
-
-int rknpu_iommu_init_domain(struct rknpu_device *rknpu_dev)
-{
- // init domain 0
- if (!rknpu_dev->iommu_domains[0]) {
- rknpu_dev->iommu_domain_id = 0;
- rknpu_dev->iommu_domains[rknpu_dev->iommu_domain_id] =
- iommu_get_domain_for_dev(rknpu_dev->dev);
- rknpu_dev->iommu_domain_num = 1;
- }
- return 0;
-}
-
-int rknpu_iommu_switch_domain(struct rknpu_device *rknpu_dev, int domain_id)
-{
- struct iommu_domain *src_domain = NULL;
- struct iommu_domain *dst_domain = NULL;
- struct bus_type *bus = NULL;
- int src_domain_id = 0;
- int ret = -EINVAL;
-
- if (!rknpu_dev->iommu_en)
- return -EINVAL;
-
- if (domain_id < 0 || domain_id > (RKNPU_MAX_IOMMU_DOMAIN_NUM - 1)) {
- LOG_DEV_ERROR(
- rknpu_dev->dev,
- "invalid iommu domain id: %d, reuse domain id: %d\n",
- domain_id, rknpu_dev->iommu_domain_id);
- return -EINVAL;
- }
-
- bus = rknpu_dev->dev->bus;
- if (!bus)
- return -EFAULT;
-
- mutex_lock(&rknpu_dev->domain_lock);
-
- src_domain_id = rknpu_dev->iommu_domain_id;
- if (domain_id == src_domain_id) {
- mutex_unlock(&rknpu_dev->domain_lock);
- return 0;
- }
-
- src_domain = iommu_get_domain_for_dev(rknpu_dev->dev);
- if (src_domain != rknpu_dev->iommu_domains[src_domain_id]) {
- LOG_DEV_ERROR(
- rknpu_dev->dev,
- "mismatch domain get from iommu_get_domain_for_dev\n");
- mutex_unlock(&rknpu_dev->domain_lock);
- return -EINVAL;
- }
-
- dst_domain = rknpu_dev->iommu_domains[domain_id];
- if (dst_domain != NULL) {
- iommu_detach_device(src_domain, rknpu_dev->dev);
- ret = iommu_attach_device(dst_domain, rknpu_dev->dev);
- if (ret) {
- LOG_DEV_ERROR(
- rknpu_dev->dev,
- "failed to attach dst iommu domain, id: %d, ret: %d\n",
- domain_id, ret);
- if (iommu_attach_device(src_domain, rknpu_dev->dev)) {
- LOG_DEV_ERROR(
- rknpu_dev->dev,
- "failed to reattach src iommu domain, id: %d\n",
- src_domain_id);
- }
- mutex_unlock(&rknpu_dev->domain_lock);
- return ret;
- }
- rknpu_dev->iommu_domain_id = domain_id;
- } else {
- uint64_t dma_limit = 1ULL << 32;
-
- dst_domain = iommu_domain_alloc(bus);
- if (!dst_domain) {
- LOG_DEV_ERROR(rknpu_dev->dev,
- "failed to allocate iommu domain\n");
- mutex_unlock(&rknpu_dev->domain_lock);
- return -EIO;
- }
- // init domain iova_cookie
- iommu_get_dma_cookie(dst_domain);
-
- iommu_detach_device(src_domain, rknpu_dev->dev);
- ret = iommu_attach_device(dst_domain, rknpu_dev->dev);
- if (ret) {
- LOG_DEV_ERROR(
- rknpu_dev->dev,
- "failed to attach iommu domain, id: %d, ret: %d\n",
- domain_id, ret);
- iommu_domain_free(dst_domain);
- mutex_unlock(&rknpu_dev->domain_lock);
- return ret;
- }
-
- // set domain type to dma domain
- dst_domain->type |= __IOMMU_DOMAIN_DMA_API;
- // iommu dma init domain
- iommu_setup_dma_ops(rknpu_dev->dev, 0, dma_limit);
-
- rknpu_dev->iommu_domain_id = domain_id;
- rknpu_dev->iommu_domains[domain_id] = dst_domain;
- rknpu_dev->iommu_domain_num++;
- }
-
- // reset default iommu domain
- rknpu_dev->iommu_group->default_domain = dst_domain;
-
- mutex_unlock(&rknpu_dev->domain_lock);
-
- LOG_INFO("switch iommu domain from %d to %d\n", src_domain_id,
- domain_id);
-
- return ret;
-}
-
-void rknpu_iommu_free_domains(struct rknpu_device *rknpu_dev)
-{
- int i = 0;
-
- rknpu_iommu_switch_domain(rknpu_dev, 0);
-
- for (i = 1; i < RKNPU_MAX_IOMMU_DOMAIN_NUM; i++) {
- struct iommu_domain *domain = rknpu_dev->iommu_domains[i];
-
- if (domain == NULL)
- continue;
-
- iommu_detach_device(domain, rknpu_dev->dev);
- iommu_domain_free(domain);
-
- rknpu_dev->iommu_domains[i] = NULL;
- }
-}
-
-#else
-
-int rknpu_iommu_init_domain(struct rknpu_device *rknpu_dev)
-{
- return 0;
-}
-
-int rknpu_iommu_switch_domain(struct rknpu_device *rknpu_dev, int domain_id)
-{
- return 0;
-}
-
-void rknpu_iommu_free_domains(struct rknpu_device *rknpu_dev)
-{
-}
-
-#endif
diff --git a/rknpu-driver/driver-0.9.6/rknpu_job.c b/rknpu-driver/driver-0.9.6/rknpu_job.c
deleted file mode 100644
index 312d9c7..0000000
--- a/rknpu-driver/driver-0.9.6/rknpu_job.c
+++ /dev/null
@@ -1,1049 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) Rockchip Electronics Co.Ltd
- * Author: Felix Zeng
- */
-
-#include
-#include
-#include
-#include
-
-#include "rknpu_ioctl.h"
-#include "rknpu_drv.h"
-#include "rknpu_reset.h"
-#include "rknpu_gem.h"
-#include "rknpu_fence.h"
-#include "rknpu_mem.h"
-#include "rknpu_iommu.h"
-#include "rknpu_job.h"
-
-#define _REG_READ(base, offset) readl(base + (offset))
-#define _REG_WRITE(base, value, offset) writel(value, base + (offset))
-
-#define REG_READ(offset) _REG_READ(rknpu_core_base, offset)
-#define REG_WRITE(value, offset) _REG_WRITE(rknpu_core_base, value, offset)
-
-static int rknpu_wait_core_index(int core_mask)
-{
- int index = 0;
-
- switch (core_mask) {
- case RKNPU_CORE0_MASK:
- case RKNPU_CORE0_MASK | RKNPU_CORE1_MASK:
- case RKNPU_CORE0_MASK | RKNPU_CORE1_MASK | RKNPU_CORE2_MASK:
- index = 0;
- break;
- case RKNPU_CORE1_MASK:
- index = 1;
- break;
- case RKNPU_CORE2_MASK:
- index = 2;
- break;
- default:
- break;
- }
-
- return index;
-}
-
-static int rknpu_core_mask(int core_index)
-{
- int core_mask = RKNPU_CORE_AUTO_MASK;
-
- switch (core_index) {
- case 0:
- core_mask = RKNPU_CORE0_MASK;
- break;
- case 1:
- core_mask = RKNPU_CORE1_MASK;
- break;
- case 2:
- core_mask = RKNPU_CORE2_MASK;
- break;
- default:
- break;
- }
-
- return core_mask;
-}
-
-static int rknpu_get_task_number(struct rknpu_job *job, int core_index)
-{
- struct rknpu_device *rknpu_dev = job->rknpu_dev;
- int task_num = job->args->task_number;
-
- if (core_index >= RKNPU_MAX_CORES || core_index < 0) {
- LOG_ERROR("invalid rknpu core index: %d", core_index);
- return 0;
- }
-
- if (rknpu_dev->config->num_irqs > 1) {
- if (job->use_core_num == 1 || job->use_core_num == 2)
- task_num =
- job->args->subcore_task[core_index].task_number;
- else if (job->use_core_num == 3)
- task_num = job->args->subcore_task[core_index + 2]
- .task_number;
- }
-
- return task_num;
-}
-
-static void rknpu_job_free(struct rknpu_job *job)
-{
-#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
- struct rknpu_gem_object *task_obj = NULL;
-
- task_obj =
- (struct rknpu_gem_object *)(uintptr_t)job->args->task_obj_addr;
- if (task_obj)
- rknpu_gem_object_put(&task_obj->base);
-#endif
-
- if (job->fence)
- dma_fence_put(job->fence);
-
- if (job->args_owner)
- kfree(job->args);
-
- kfree(job);
-}
-
-static int rknpu_job_cleanup(struct rknpu_job *job)
-{
- rknpu_job_free(job);
-
- return 0;
-}
-
-static void rknpu_job_cleanup_work(struct work_struct *work)
-{
- struct rknpu_job *job =
- container_of(work, struct rknpu_job, cleanup_work);
-
- rknpu_job_cleanup(job);
-}
-
-static inline struct rknpu_job *rknpu_job_alloc(struct rknpu_device *rknpu_dev,
- struct rknpu_submit *args)
-{
- struct rknpu_job *job = NULL;
- int i = 0;
-#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
- struct rknpu_gem_object *task_obj = NULL;
-#endif
-
- job = kzalloc(sizeof(*job), GFP_KERNEL);
- if (!job)
- return NULL;
-
- job->timestamp = ktime_get();
- job->rknpu_dev = rknpu_dev;
- job->use_core_num = (args->core_mask & RKNPU_CORE0_MASK) +
- ((args->core_mask & RKNPU_CORE1_MASK) >> 1) +
- ((args->core_mask & RKNPU_CORE2_MASK) >> 2);
- atomic_set(&job->run_count, job->use_core_num);
- atomic_set(&job->interrupt_count, job->use_core_num);
- job->iommu_domain_id = args->iommu_domain_id;
- for (i = 0; i < rknpu_dev->config->num_irqs; i++)
- atomic_set(&job->submit_count[i], 0);
-#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
- task_obj = (struct rknpu_gem_object *)(uintptr_t)args->task_obj_addr;
- if (task_obj)
- rknpu_gem_object_get(&task_obj->base);
-#endif
-
- if (!(args->flags & RKNPU_JOB_NONBLOCK)) {
- job->args = args;
- job->args_owner = false;
- return job;
- }
-
- job->args = kzalloc(sizeof(*args), GFP_KERNEL);
- if (!job->args) {
- kfree(job);
- return NULL;
- }
- *job->args = *args;
- job->args_owner = true;
-
- INIT_WORK(&job->cleanup_work, rknpu_job_cleanup_work);
-
- return job;
-}
-
-static inline int rknpu_job_wait(struct rknpu_job *job)
-{
- struct rknpu_device *rknpu_dev = job->rknpu_dev;
- struct rknpu_submit *args = job->args;
- struct rknpu_task *last_task = NULL;
- struct rknpu_subcore_data *subcore_data = NULL;
- struct rknpu_job *entry, *q;
- void __iomem *rknpu_core_base = NULL;
- int core_index = rknpu_wait_core_index(job->args->core_mask);
- unsigned long flags;
- int wait_count = 0;
- bool continue_wait = false;
- int ret = -EINVAL;
- int i = 0;
-
- subcore_data = &rknpu_dev->subcore_datas[core_index];
-
- do {
- ret = wait_event_timeout(subcore_data->job_done_wq,
- job->flags & RKNPU_JOB_DONE ||
- rknpu_dev->soft_reseting,
- msecs_to_jiffies(args->timeout));
-
- if (++wait_count >= 3)
- break;
-
- if (ret == 0) {
- int64_t elapse_time_us = 0;
- spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
- elapse_time_us = ktime_us_delta(ktime_get(),
- job->hw_commit_time);
- continue_wait =
- job->hw_commit_time == 0 ?
- true :
- (elapse_time_us < args->timeout * 1000);
- spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
- LOG_ERROR(
- "job: %p, iommu domain id: %d, wait_count: %d, continue wait: %d, commit elapse time: %lldus, wait time: %lldus, timeout: %uus\n",
- job, job->iommu_domain_id, wait_count,
- continue_wait,
- (job->hw_commit_time == 0 ? 0 : elapse_time_us),
- ktime_us_delta(ktime_get(), job->timestamp),
- args->timeout * 1000);
- }
- } while (ret == 0 && continue_wait);
-
- last_task = job->last_task;
- if (!last_task) {
- spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
- for (i = 0; i < job->use_core_num; i++) {
- subcore_data = &rknpu_dev->subcore_datas[i];
- list_for_each_entry_safe(
- entry, q, &subcore_data->todo_list, head[i]) {
- if (entry == job) {
- list_del(&job->head[i]);
- break;
- }
- }
- }
- spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
-
- LOG_ERROR("job commit failed\n");
- return ret < 0 ? ret : -EINVAL;
- }
-
- last_task->int_status = job->int_status[core_index];
-
- if (ret <= 0) {
- args->task_counter = 0;
- rknpu_core_base = rknpu_dev->base[core_index];
- if (args->flags & RKNPU_JOB_PC) {
- uint32_t task_status = REG_READ(
- rknpu_dev->config->pc_task_status_offset);
- args->task_counter =
- (task_status &
- rknpu_dev->config->pc_task_number_mask);
- }
-
- LOG_ERROR(
- "failed to wait job, task counter: %d, flags: %#x, ret = %d, elapsed time: %lldus\n",
- args->task_counter, args->flags, ret,
- ktime_us_delta(ktime_get(), job->timestamp));
-
- return ret < 0 ? ret : -ETIMEDOUT;
- }
-
- if (!(job->flags & RKNPU_JOB_DONE))
- return -EINVAL;
-
- args->task_counter = args->task_number;
- args->hw_elapse_time = job->hw_elapse_time;
-
- return 0;
-}
-
-static inline int rknpu_job_subcore_commit_pc(struct rknpu_job *job,
- int core_index)
-{
- struct rknpu_device *rknpu_dev = job->rknpu_dev;
- struct rknpu_submit *args = job->args;
-#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
- struct rknpu_gem_object *task_obj =
- (struct rknpu_gem_object *)(uintptr_t)args->task_obj_addr;
-#endif
-#ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
- struct rknpu_mem_object *task_obj =
- (struct rknpu_mem_object *)(uintptr_t)args->task_obj_addr;
-#endif
- struct rknpu_task *task_base = NULL;
- struct rknpu_task *first_task = NULL;
- struct rknpu_task *last_task = NULL;
- void __iomem *rknpu_core_base = rknpu_dev->base[core_index];
- int task_start = args->task_start;
- int task_end;
- int task_number = args->task_number;
- int task_pp_en = args->flags & RKNPU_JOB_PINGPONG ? 1 : 0;
- int pc_data_amount_scale = rknpu_dev->config->pc_data_amount_scale;
- int pc_task_number_bits = rknpu_dev->config->pc_task_number_bits;
- int i = 0;
- int submit_index = atomic_read(&job->submit_count[core_index]);
- int max_submit_number = rknpu_dev->config->max_submit_number;
- unsigned long flags;
-
- if (!task_obj) {
- job->ret = -EINVAL;
- return job->ret;
- }
-
- if (rknpu_dev->config->num_irqs > 1) {
- for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
- if (i == core_index) {
- REG_WRITE((0xe + 0x10000000 * i), 0x1004);
- REG_WRITE((0xe + 0x10000000 * i), 0x3004);
- }
- }
-
- switch (job->use_core_num) {
- case 1:
- case 2:
- task_start = args->subcore_task[core_index].task_start;
- task_number =
- args->subcore_task[core_index].task_number;
- break;
- case 3:
- task_start =
- args->subcore_task[core_index + 2].task_start;
- task_number =
- args->subcore_task[core_index + 2].task_number;
- break;
- default:
- LOG_ERROR("Unknown use core num %d\n",
- job->use_core_num);
- break;
- }
- }
-
- task_start = task_start + submit_index * max_submit_number;
- task_number = task_number - submit_index * max_submit_number;
- task_number = task_number > max_submit_number ? max_submit_number :
- task_number;
- task_end = task_start + task_number - 1;
-
- task_base = task_obj->kv_addr;
-
- first_task = &task_base[task_start];
- last_task = &task_base[task_end];
-
- if (rknpu_dev->config->pc_dma_ctrl) {
- spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
- REG_WRITE(first_task->regcmd_addr, RKNPU_OFFSET_PC_DATA_ADDR);
- spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
- } else {
- REG_WRITE(first_task->regcmd_addr, RKNPU_OFFSET_PC_DATA_ADDR);
- }
-
- REG_WRITE((first_task->regcfg_amount + RKNPU_PC_DATA_EXTRA_AMOUNT +
- pc_data_amount_scale - 1) /
- pc_data_amount_scale -
- 1,
- RKNPU_OFFSET_PC_DATA_AMOUNT);
-
- REG_WRITE(last_task->int_mask, RKNPU_OFFSET_INT_MASK);
-
- REG_WRITE(first_task->int_mask, RKNPU_OFFSET_INT_CLEAR);
-
- REG_WRITE(((0x6 | task_pp_en) << pc_task_number_bits) | task_number,
- RKNPU_OFFSET_PC_TASK_CONTROL);
-
- REG_WRITE(args->task_base_addr, RKNPU_OFFSET_PC_DMA_BASE_ADDR);
-
- job->first_task = first_task;
- job->last_task = last_task;
- job->int_mask[core_index] = last_task->int_mask;
-
- REG_WRITE(0x1, RKNPU_OFFSET_PC_OP_EN);
- REG_WRITE(0x0, RKNPU_OFFSET_PC_OP_EN);
-
- return 0;
-}
-
-static inline int rknpu_job_subcore_commit(struct rknpu_job *job,
- int core_index)
-{
- struct rknpu_device *rknpu_dev = job->rknpu_dev;
- struct rknpu_submit *args = job->args;
- void __iomem *rknpu_core_base = rknpu_dev->base[core_index];
- unsigned long flags;
-
- // switch to slave mode
- if (rknpu_dev->config->pc_dma_ctrl) {
- spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
- REG_WRITE(0x1, RKNPU_OFFSET_PC_DATA_ADDR);
- spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
- } else {
- REG_WRITE(0x1, RKNPU_OFFSET_PC_DATA_ADDR);
- }
-
- if (!(args->flags & RKNPU_JOB_PC)) {
- job->ret = -EINVAL;
- return job->ret;
- }
-
- return rknpu_job_subcore_commit_pc(job, core_index);
-}
-
-static void rknpu_job_commit(struct rknpu_job *job)
-{
- switch (job->args->core_mask) {
- case RKNPU_CORE0_MASK:
- rknpu_job_subcore_commit(job, 0);
- break;
- case RKNPU_CORE1_MASK:
- rknpu_job_subcore_commit(job, 1);
- break;
- case RKNPU_CORE2_MASK:
- rknpu_job_subcore_commit(job, 2);
- break;
- case RKNPU_CORE0_MASK | RKNPU_CORE1_MASK:
- rknpu_job_subcore_commit(job, 0);
- rknpu_job_subcore_commit(job, 1);
- break;
- case RKNPU_CORE0_MASK | RKNPU_CORE1_MASK | RKNPU_CORE2_MASK:
- rknpu_job_subcore_commit(job, 0);
- rknpu_job_subcore_commit(job, 1);
- rknpu_job_subcore_commit(job, 2);
- break;
- default:
- LOG_ERROR("Unknown core mask: %d\n", job->args->core_mask);
- break;
- }
-}
-
-static void rknpu_job_next(struct rknpu_device *rknpu_dev, int core_index)
-{
- struct rknpu_job *job = NULL;
- struct rknpu_subcore_data *subcore_data = NULL;
- unsigned long flags;
-
- if (rknpu_dev->soft_reseting)
- return;
-
- subcore_data = &rknpu_dev->subcore_datas[core_index];
-
- spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
-
- if (subcore_data->job || list_empty(&subcore_data->todo_list)) {
- spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
- return;
- }
-
- job = list_first_entry(&subcore_data->todo_list, struct rknpu_job,
- head[core_index]);
-
- list_del_init(&job->head[core_index]);
- subcore_data->job = job;
- job->hw_commit_time = ktime_get();
- job->hw_recoder_time = job->hw_commit_time;
- spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
-
- if (atomic_dec_and_test(&job->run_count)) {
- rknpu_job_commit(job);
- }
-}
-
-static void rknpu_job_done(struct rknpu_job *job, int ret, int core_index)
-{
- struct rknpu_device *rknpu_dev = job->rknpu_dev;
- struct rknpu_subcore_data *subcore_data = NULL;
- ktime_t now;
- unsigned long flags;
- int max_submit_number = rknpu_dev->config->max_submit_number;
-
- if (atomic_inc_return(&job->submit_count[core_index]) <
- (rknpu_get_task_number(job, core_index) + max_submit_number - 1) /
- max_submit_number) {
- rknpu_job_subcore_commit(job, core_index);
- return;
- }
-
- subcore_data = &rknpu_dev->subcore_datas[core_index];
-
- spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
- subcore_data->job = NULL;
- subcore_data->task_num -= rknpu_get_task_number(job, core_index);
- now = ktime_get();
- job->hw_elapse_time = ktime_sub(now, job->hw_commit_time);
- subcore_data->timer.busy_time += ktime_sub(now, job->hw_recoder_time);
- spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
-
- if (atomic_dec_and_test(&job->interrupt_count)) {
- int use_core_num = job->use_core_num;
-
- job->flags |= RKNPU_JOB_DONE;
- job->ret = ret;
-
- if (job->fence)
- dma_fence_signal(job->fence);
-
- if (job->flags & RKNPU_JOB_ASYNC)
- schedule_work(&job->cleanup_work);
-
- if (use_core_num > 1)
- wake_up(&(&rknpu_dev->subcore_datas[0])->job_done_wq);
- else
- wake_up(&subcore_data->job_done_wq);
- }
-
- rknpu_job_next(rknpu_dev, core_index);
-}
-
-static int rknpu_schedule_core_index(struct rknpu_device *rknpu_dev)
-{
- int core_num = rknpu_dev->config->num_irqs;
- int task_num = rknpu_dev->subcore_datas[0].task_num;
- int core_index = 0;
- int i = 0;
-
- for (i = 1; i < core_num; i++) {
- if (task_num > rknpu_dev->subcore_datas[i].task_num) {
- core_index = i;
- task_num = rknpu_dev->subcore_datas[i].task_num;
- }
- }
-
- return core_index;
-}
-
-static void rknpu_job_schedule(struct rknpu_job *job)
-{
- struct rknpu_device *rknpu_dev = job->rknpu_dev;
- struct rknpu_subcore_data *subcore_data = NULL;
- int i = 0, core_index = 0;
- unsigned long flags;
-
- if (job->args->core_mask == RKNPU_CORE_AUTO_MASK) {
- core_index = rknpu_schedule_core_index(rknpu_dev);
- job->args->core_mask = rknpu_core_mask(core_index);
- job->use_core_num = 1;
- atomic_set(&job->run_count, job->use_core_num);
- atomic_set(&job->interrupt_count, job->use_core_num);
- }
-
- spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
- for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
- if (job->args->core_mask & rknpu_core_mask(i)) {
- subcore_data = &rknpu_dev->subcore_datas[i];
- list_add_tail(&job->head[i], &subcore_data->todo_list);
- subcore_data->task_num += rknpu_get_task_number(job, i);
- }
- }
- spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
-
- for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
- if (job->args->core_mask & rknpu_core_mask(i))
- rknpu_job_next(rknpu_dev, i);
- }
-}
-
-static void rknpu_job_abort(struct rknpu_job *job)
-{
- struct rknpu_device *rknpu_dev = job->rknpu_dev;
- struct rknpu_subcore_data *subcore_data = NULL;
- unsigned long flags;
- int i = 0;
-
- msleep(100);
-
- spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
- for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
- if (job->args->core_mask & rknpu_core_mask(i)) {
- subcore_data = &rknpu_dev->subcore_datas[i];
- if (job == subcore_data->job && !job->irq_entry[i]) {
- subcore_data->job = NULL;
- subcore_data->task_num -=
- rknpu_get_task_number(job, i);
- }
- }
- }
- spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
-
- if (job->ret == -ETIMEDOUT) {
- LOG_ERROR("job timeout, flags: %#x:\n", job->flags);
- for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
- if (job->args->core_mask & rknpu_core_mask(i)) {
- void __iomem *rknpu_core_base =
- rknpu_dev->base[i];
- LOG_ERROR(
- "\tcore %d irq status: %#x, raw status: %#x, require mask: %#x, task counter: %#x, elapsed time: %lldus\n",
- i, REG_READ(RKNPU_OFFSET_INT_STATUS),
- REG_READ(RKNPU_OFFSET_INT_RAW_STATUS),
- job->int_mask[i],
- (REG_READ(
- rknpu_dev->config
- ->pc_task_status_offset) &
- rknpu_dev->config->pc_task_number_mask),
- ktime_us_delta(ktime_get(),
- job->timestamp));
- }
- }
- rknpu_soft_reset(rknpu_dev);
- } else {
- LOG_ERROR(
- "job abort, flags: %#x, ret: %d, elapsed time: %lldus\n",
- job->flags, job->ret,
- ktime_us_delta(ktime_get(), job->timestamp));
- }
-
- rknpu_job_cleanup(job);
-}
-
-static inline uint32_t rknpu_fuzz_status(uint32_t status)
-{
- uint32_t fuzz_status = 0;
-
- if ((status & 0x3) != 0)
- fuzz_status |= 0x3;
-
- if ((status & 0xc) != 0)
- fuzz_status |= 0xc;
-
- if ((status & 0x30) != 0)
- fuzz_status |= 0x30;
-
- if ((status & 0xc0) != 0)
- fuzz_status |= 0xc0;
-
- if ((status & 0x300) != 0)
- fuzz_status |= 0x300;
-
- if ((status & 0xc00) != 0)
- fuzz_status |= 0xc00;
-
- return fuzz_status;
-}
-
-static inline irqreturn_t rknpu_irq_handler(int irq, void *data, int core_index)
-{
- struct rknpu_device *rknpu_dev = data;
- void __iomem *rknpu_core_base = rknpu_dev->base[core_index];
- struct rknpu_subcore_data *subcore_data = NULL;
- struct rknpu_job *job = NULL;
- uint32_t status = 0;
- unsigned long flags;
-
- subcore_data = &rknpu_dev->subcore_datas[core_index];
-
- spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
- job = subcore_data->job;
- if (!job) {
- spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
- REG_WRITE(RKNPU_INT_CLEAR, RKNPU_OFFSET_INT_CLEAR);
- rknpu_job_next(rknpu_dev, core_index);
- return IRQ_HANDLED;
- }
- job->irq_entry[core_index] = true;
- spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
-
- status = REG_READ(RKNPU_OFFSET_INT_STATUS);
-
- job->int_status[core_index] = status;
-
- if (rknpu_fuzz_status(status) != job->int_mask[core_index]) {
- LOG_ERROR(
- "invalid irq status: %#x, raw status: %#x, require mask: %#x, task counter: %#x\n",
- status, REG_READ(RKNPU_OFFSET_INT_RAW_STATUS),
- job->int_mask[core_index],
- (REG_READ(rknpu_dev->config->pc_task_status_offset) &
- rknpu_dev->config->pc_task_number_mask));
- REG_WRITE(RKNPU_INT_CLEAR, RKNPU_OFFSET_INT_CLEAR);
- return IRQ_HANDLED;
- }
-
- REG_WRITE(RKNPU_INT_CLEAR, RKNPU_OFFSET_INT_CLEAR);
-
- rknpu_job_done(job, 0, core_index);
-
- return IRQ_HANDLED;
-}
-
-irqreturn_t rknpu_core0_irq_handler(int irq, void *data)
-{
- return rknpu_irq_handler(irq, data, 0);
-}
-
-irqreturn_t rknpu_core1_irq_handler(int irq, void *data)
-{
- return rknpu_irq_handler(irq, data, 1);
-}
-
-irqreturn_t rknpu_core2_irq_handler(int irq, void *data)
-{
- return rknpu_irq_handler(irq, data, 2);
-}
-
-static void rknpu_job_timeout_clean(struct rknpu_device *rknpu_dev,
- int core_mask)
-{
- struct rknpu_job *job = NULL;
- unsigned long flags;
- struct rknpu_subcore_data *subcore_data = NULL;
- int i = 0;
-
- for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
- if (core_mask & rknpu_core_mask(i)) {
- subcore_data = &rknpu_dev->subcore_datas[i];
- job = subcore_data->job;
- if (job &&
- ktime_us_delta(ktime_get(), job->timestamp) >=
- job->args->timeout) {
- rknpu_soft_reset(rknpu_dev);
-
- spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
- subcore_data->job = NULL;
- spin_unlock_irqrestore(&rknpu_dev->irq_lock,
- flags);
-
- do {
- schedule_work(&job->cleanup_work);
-
- spin_lock_irqsave(&rknpu_dev->irq_lock,
- flags);
-
- if (!list_empty(
- &subcore_data->todo_list)) {
- job = list_first_entry(
- &subcore_data->todo_list,
- struct rknpu_job,
- head[i]);
- list_del_init(&job->head[i]);
- } else {
- job = NULL;
- }
-
- spin_unlock_irqrestore(
- &rknpu_dev->irq_lock, flags);
- } while (job);
- }
- }
- }
-}
-
-static int rknpu_submit(struct rknpu_device *rknpu_dev,
- struct rknpu_submit *args)
-{
- struct rknpu_job *job = NULL;
- int ret = -EINVAL;
-
- if (args->task_number == 0) {
- LOG_ERROR("invalid rknpu task number!\n");
- return -EINVAL;
- }
-
- if (args->core_mask > rknpu_dev->config->core_mask) {
- LOG_ERROR("invalid rknpu core mask: %#x", args->core_mask);
- return -EINVAL;
- }
-
- job = rknpu_job_alloc(rknpu_dev, args);
- if (!job) {
- LOG_ERROR("failed to allocate rknpu job!\n");
- return -ENOMEM;
- }
-
- if (args->flags & RKNPU_JOB_FENCE_IN) {
-#ifdef CONFIG_ROCKCHIP_RKNPU_FENCE
- struct dma_fence *in_fence;
-
- in_fence = sync_file_get_fence(args->fence_fd);
-
- if (!in_fence) {
- LOG_ERROR("invalid fence in fd, fd: %d\n",
- args->fence_fd);
- return -EINVAL;
- }
- args->fence_fd = -1;
-
- /*
- * Wait if the fence is from a foreign context, or if the fence
- * array contains any fence from a foreign context.
- */
- ret = 0;
- if (!dma_fence_match_context(in_fence,
- rknpu_dev->fence_ctx->context))
- ret = dma_fence_wait_timeout(in_fence, true,
- args->timeout);
- dma_fence_put(in_fence);
- if (ret < 0) {
- if (ret != -ERESTARTSYS)
- LOG_ERROR("Error (%d) waiting for fence!\n",
- ret);
-
- return ret;
- }
-#else
- LOG_ERROR(
- "failed to use rknpu fence, please enable rknpu fence config!\n");
- rknpu_job_free(job);
- return -EINVAL;
-#endif
- }
-
- if (args->flags & RKNPU_JOB_FENCE_OUT) {
-#ifdef CONFIG_ROCKCHIP_RKNPU_FENCE
- ret = rknpu_fence_alloc(job);
- if (ret) {
- rknpu_job_free(job);
- return ret;
- }
- job->args->fence_fd = rknpu_fence_get_fd(job);
- args->fence_fd = job->args->fence_fd;
-#else
- LOG_ERROR(
- "failed to use rknpu fence, please enable rknpu fence config!\n");
- rknpu_job_free(job);
- return -EINVAL;
-#endif
- }
-
- if (args->flags & RKNPU_JOB_NONBLOCK) {
- job->flags |= RKNPU_JOB_ASYNC;
- rknpu_job_timeout_clean(rknpu_dev, job->args->core_mask);
- rknpu_job_schedule(job);
- ret = job->ret;
- if (ret) {
- rknpu_job_abort(job);
- return ret;
- }
- } else {
- rknpu_job_schedule(job);
- if (args->flags & RKNPU_JOB_PC)
- job->ret = rknpu_job_wait(job);
-
- args->task_counter = job->args->task_counter;
- ret = job->ret;
- if (!ret)
- rknpu_job_cleanup(job);
- else
- rknpu_job_abort(job);
- }
-
- return ret;
-}
-
-#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
-int rknpu_submit_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file_priv)
-{
- struct rknpu_device *rknpu_dev = dev_get_drvdata(dev->dev);
- struct rknpu_submit *args = data;
-
- rknpu_iommu_switch_domain(rknpu_dev, args->iommu_domain_id);
-
- return rknpu_submit(rknpu_dev, args);
-}
-#endif
-
-#ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
-int rknpu_submit_ioctl(struct rknpu_device *rknpu_dev, unsigned long data)
-{
- struct rknpu_submit args;
- int ret = -EINVAL;
-
- if (unlikely(copy_from_user(&args, (struct rknpu_submit *)data,
- sizeof(struct rknpu_submit)))) {
- LOG_ERROR("%s: copy_from_user failed\n", __func__);
- ret = -EFAULT;
- return ret;
- }
-
- ret = rknpu_submit(rknpu_dev, &args);
-
- if (unlikely(copy_to_user((struct rknpu_submit *)data, &args,
- sizeof(struct rknpu_submit)))) {
- LOG_ERROR("%s: copy_to_user failed\n", __func__);
- ret = -EFAULT;
- return ret;
- }
-
- return ret;
-}
-#endif
-
-int rknpu_get_hw_version(struct rknpu_device *rknpu_dev, uint32_t *version)
-{
- void __iomem *rknpu_core_base = rknpu_dev->base[0];
-
- if (version == NULL)
- return -EINVAL;
-
- *version = REG_READ(RKNPU_OFFSET_VERSION) +
- (REG_READ(RKNPU_OFFSET_VERSION_NUM) & 0xffff);
-
- return 0;
-}
-
-int rknpu_get_bw_priority(struct rknpu_device *rknpu_dev, uint32_t *priority,
- uint32_t *expect, uint32_t *tw)
-{
- void __iomem *base = rknpu_dev->bw_priority_base;
-
- if (!base)
- return -EINVAL;
-
- spin_lock(&rknpu_dev->lock);
-
- if (priority != NULL)
- *priority = _REG_READ(base, 0x0);
-
- if (expect != NULL)
- *expect = _REG_READ(base, 0x8);
-
- if (tw != NULL)
- *tw = _REG_READ(base, 0xc);
-
- spin_unlock(&rknpu_dev->lock);
-
- return 0;
-}
-
-int rknpu_set_bw_priority(struct rknpu_device *rknpu_dev, uint32_t priority,
- uint32_t expect, uint32_t tw)
-{
- void __iomem *base = rknpu_dev->bw_priority_base;
-
- if (!base)
- return -EINVAL;
-
- spin_lock(&rknpu_dev->lock);
-
- if (priority != 0)
- _REG_WRITE(base, priority, 0x0);
-
- if (expect != 0)
- _REG_WRITE(base, expect, 0x8);
-
- if (tw != 0)
- _REG_WRITE(base, tw, 0xc);
-
- spin_unlock(&rknpu_dev->lock);
-
- return 0;
-}
-
-int rknpu_clear_rw_amount(struct rknpu_device *rknpu_dev)
-{
- void __iomem *rknpu_core_base = rknpu_dev->base[0];
- const struct rknpu_config *config = rknpu_dev->config;
- unsigned long flags;
-
- if (config->amount_top == NULL) {
- LOG_WARN("Clear rw_amount is not supported on this device!\n");
- return 0;
- }
-
- if (config->pc_dma_ctrl) {
- uint32_t pc_data_addr = 0;
-
- spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
- pc_data_addr = REG_READ(RKNPU_OFFSET_PC_DATA_ADDR);
-
- REG_WRITE(0x1, RKNPU_OFFSET_PC_DATA_ADDR);
- REG_WRITE(0x80000101, config->amount_top->offset_clr_all);
- REG_WRITE(0x00000101, config->amount_top->offset_clr_all);
- if (config->amount_core) {
- REG_WRITE(0x80000101,
- config->amount_core->offset_clr_all);
- REG_WRITE(0x00000101,
- config->amount_core->offset_clr_all);
- }
- REG_WRITE(pc_data_addr, RKNPU_OFFSET_PC_DATA_ADDR);
- spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
- } else {
- spin_lock(&rknpu_dev->lock);
- REG_WRITE(0x80000101, config->amount_top->offset_clr_all);
- REG_WRITE(0x00000101, config->amount_top->offset_clr_all);
- if (config->amount_core) {
- REG_WRITE(0x80000101,
- config->amount_core->offset_clr_all);
- REG_WRITE(0x00000101,
- config->amount_core->offset_clr_all);
- }
- spin_unlock(&rknpu_dev->lock);
- }
-
- return 0;
-}
-
-int rknpu_get_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *dt_wr,
- uint32_t *dt_rd, uint32_t *wd_rd)
-{
- void __iomem *rknpu_core_base = rknpu_dev->base[0];
- const struct rknpu_config *config = rknpu_dev->config;
- int amount_scale = config->pc_data_amount_scale;
-
- if (config->amount_top == NULL) {
- LOG_WARN("Get rw_amount is not supported on this device!\n");
- return 0;
- }
-
- spin_lock(&rknpu_dev->lock);
-
- if (dt_wr != NULL) {
- *dt_wr = REG_READ(config->amount_top->offset_dt_wr) *
- amount_scale;
- if (config->amount_core) {
- *dt_wr += REG_READ(config->amount_core->offset_dt_wr) *
- amount_scale;
- }
- }
-
- if (dt_rd != NULL) {
- *dt_rd = REG_READ(config->amount_top->offset_dt_rd) *
- amount_scale;
- if (config->amount_core) {
- *dt_rd += REG_READ(config->amount_core->offset_dt_rd) *
- amount_scale;
- }
- }
-
- if (wd_rd != NULL) {
- *wd_rd = REG_READ(config->amount_top->offset_wt_rd) *
- amount_scale;
- if (config->amount_core) {
- *wd_rd += REG_READ(config->amount_core->offset_wt_rd) *
- amount_scale;
- }
- }
-
- spin_unlock(&rknpu_dev->lock);
-
- return 0;
-}
-
-int rknpu_get_total_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *amount)
-{
- const struct rknpu_config *config = rknpu_dev->config;
- uint32_t dt_wr = 0;
- uint32_t dt_rd = 0;
- uint32_t wd_rd = 0;
- int ret = -EINVAL;
-
- if (config->amount_top == NULL) {
- LOG_WARN(
- "Get total_rw_amount is not supported on this device!\n");
- return 0;
- }
-
- ret = rknpu_get_rw_amount(rknpu_dev, &dt_wr, &dt_rd, &wd_rd);
-
- if (amount != NULL)
- *amount = dt_wr + dt_rd + wd_rd;
-
- return ret;
-}
diff --git a/rknpu-driver/driver-0.9.6/rknpu_mem.c b/rknpu-driver/driver-0.9.6/rknpu_mem.c
deleted file mode 100644
index 31aede9..0000000
--- a/rknpu-driver/driver-0.9.6/rknpu_mem.c
+++ /dev/null
@@ -1,353 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) Rockchip Electronics Co.Ltd
- * Author: Felix Zeng
- */
-
-#include
-#include
-
-#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
-#include
-#endif
-
-#include "rknpu_drv.h"
-#include "rknpu_ioctl.h"
-#include "rknpu_mem.h"
-
-#ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
-
-int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, struct file *file,
- unsigned int cmd, unsigned long data)
-{
- struct rknpu_mem_create args;
- int ret = -EINVAL;
- struct dma_buf_attachment *attachment;
- struct sg_table *table;
- struct scatterlist *sgl;
- dma_addr_t phys;
- struct dma_buf *dmabuf;
- struct page **pages;
- struct page *page;
- struct rknpu_mem_object *rknpu_obj = NULL;
- struct rknpu_session *session = NULL;
- int i, fd;
- unsigned int length, page_count;
- unsigned int in_size = _IOC_SIZE(cmd);
- unsigned int k_size = sizeof(struct rknpu_mem_create);
- char *k_data = (char *)&args;
-
- if (unlikely(copy_from_user(&args, (struct rknpu_mem_create *)data,
- in_size))) {
- LOG_ERROR("%s: copy_from_user failed\n", __func__);
- ret = -EFAULT;
- return ret;
- }
-
- if (k_size > in_size)
- memset(k_data + in_size, 0, k_size - in_size);
-
- if (args.flags & RKNPU_MEM_NON_CONTIGUOUS) {
- LOG_ERROR("%s: malloc iommu memory unsupported in current!\n",
- __func__);
- ret = -EINVAL;
- return ret;
- }
-
- rknpu_obj = kzalloc(sizeof(*rknpu_obj), GFP_KERNEL);
- if (!rknpu_obj)
- return -ENOMEM;
-
- if (args.handle > 0) {
- fd = args.handle;
-
- dmabuf = dma_buf_get(fd);
- if (IS_ERR(dmabuf)) {
- ret = PTR_ERR(dmabuf);
- goto err_free_obj;
- }
-
- rknpu_obj->dmabuf = dmabuf;
- rknpu_obj->owner = 0;
- } else {
- /* Start test kernel alloc/free dma buf */
- dmabuf = rk_dma_heap_buffer_alloc(rknpu_dev->heap, args.size,
- O_CLOEXEC | O_RDWR, 0x0,
- dev_name(rknpu_dev->dev));
- if (IS_ERR(dmabuf)) {
- LOG_ERROR("dmabuf alloc failed, args.size = %llu\n",
- args.size);
- ret = PTR_ERR(dmabuf);
- goto err_free_obj;
- }
-
- rknpu_obj->dmabuf = dmabuf;
- rknpu_obj->owner = 1;
-
- fd = dma_buf_fd(dmabuf, O_CLOEXEC | O_RDWR);
- if (fd < 0) {
- LOG_ERROR("dmabuf fd get failed\n");
- ret = -EFAULT;
- goto err_free_dma_buf;
- }
- }
-
- attachment = dma_buf_attach(dmabuf, rknpu_dev->dev);
- if (IS_ERR(attachment)) {
- LOG_ERROR("dma_buf_attach failed\n");
- ret = PTR_ERR(attachment);
- goto err_free_dma_buf;
- }
-
- table = dma_buf_map_attachment(attachment, DMA_BIDIRECTIONAL);
- if (IS_ERR(table)) {
- LOG_ERROR("dma_buf_attach failed\n");
- dma_buf_detach(dmabuf, attachment);
- ret = PTR_ERR(table);
- goto err_free_dma_buf;
- }
-
- for_each_sgtable_sg(table, sgl, i) {
- phys = sg_dma_address(sgl);
- page = sg_page(sgl);
- length = sg_dma_len(sgl);
- LOG_DEBUG("%s, %d, phys: %pad, length: %u\n", __func__,
- __LINE__, &phys, length);
- }
-
- if (args.flags & RKNPU_MEM_KERNEL_MAPPING) {
- page_count = length >> PAGE_SHIFT;
- pages = vmalloc(page_count * sizeof(struct page));
- if (!pages) {
- LOG_ERROR("alloc pages failed\n");
- ret = -ENOMEM;
- goto err_detach_dma_buf;
- }
-
- for (i = 0; i < page_count; i++)
- pages[i] = &page[i];
-
- rknpu_obj->kv_addr =
- vmap(pages, page_count, VM_MAP, PAGE_KERNEL);
- if (!rknpu_obj->kv_addr) {
- LOG_ERROR("vmap pages addr failed\n");
- ret = -ENOMEM;
- goto err_free_pages;
- }
- vfree(pages);
- pages = NULL;
- }
-
- rknpu_obj->size = PAGE_ALIGN(args.size);
- rknpu_obj->dma_addr = phys;
- rknpu_obj->sgt = table;
-
- args.size = rknpu_obj->size;
- args.obj_addr = (__u64)(uintptr_t)rknpu_obj;
- args.dma_addr = rknpu_obj->dma_addr;
- args.handle = fd;
-
- LOG_DEBUG(
- "args.handle: %d, args.size: %lld, rknpu_obj: %#llx, rknpu_obj->dma_addr: %#llx\n",
- args.handle, args.size, (__u64)(uintptr_t)rknpu_obj,
- (__u64)rknpu_obj->dma_addr);
-
- if (unlikely(copy_to_user((struct rknpu_mem_create *)data, &args,
- in_size))) {
- LOG_ERROR("%s: copy_to_user failed\n", __func__);
- ret = -EFAULT;
- goto err_unmap_kv_addr;
- }
-
- dma_buf_unmap_attachment(attachment, table, DMA_BIDIRECTIONAL);
- dma_buf_detach(dmabuf, attachment);
-
- spin_lock(&rknpu_dev->lock);
-
- session = file->private_data;
- if (!session) {
- spin_unlock(&rknpu_dev->lock);
- ret = -EFAULT;
- goto err_unmap_kv_addr;
- }
- list_add_tail(&rknpu_obj->head, &session->list);
-
- spin_unlock(&rknpu_dev->lock);
-
- return 0;
-
-err_unmap_kv_addr:
- vunmap(rknpu_obj->kv_addr);
- rknpu_obj->kv_addr = NULL;
-
-err_free_pages:
- vfree(pages);
- pages = NULL;
-
-err_detach_dma_buf:
- dma_buf_unmap_attachment(attachment, table, DMA_BIDIRECTIONAL);
- dma_buf_detach(dmabuf, attachment);
-
-err_free_dma_buf:
- if (rknpu_obj->owner)
- rk_dma_heap_buffer_free(dmabuf);
- else
- dma_buf_put(dmabuf);
-
-err_free_obj:
- kfree(rknpu_obj);
-
- return ret;
-}
-
-int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, struct file *file,
- unsigned long data)
-{
- struct rknpu_mem_object *rknpu_obj, *entry, *q;
- struct rknpu_session *session = NULL;
- struct rknpu_mem_destroy args;
- int ret = -EFAULT;
-
- if (unlikely(copy_from_user(&args, (struct rknpu_mem_destroy *)data,
- sizeof(struct rknpu_mem_destroy)))) {
- LOG_ERROR("%s: copy_from_user failed\n", __func__);
- ret = -EFAULT;
- return ret;
- }
-
- if (!kern_addr_valid(args.obj_addr)) {
- LOG_ERROR("%s: invalid obj_addr: %#llx\n", __func__,
- (__u64)(uintptr_t)args.obj_addr);
- ret = -EINVAL;
- return ret;
- }
-
- rknpu_obj = (struct rknpu_mem_object *)(uintptr_t)args.obj_addr;
- LOG_DEBUG(
- "free args.handle: %d, rknpu_obj: %#llx, rknpu_obj->dma_addr: %#llx\n",
- args.handle, (__u64)(uintptr_t)rknpu_obj,
- (__u64)rknpu_obj->dma_addr);
-
- spin_lock(&rknpu_dev->lock);
- session = file->private_data;
- if (!session) {
- spin_unlock(&rknpu_dev->lock);
- ret = -EFAULT;
- return ret;
- }
- list_for_each_entry_safe(entry, q, &session->list, head) {
- if (entry == rknpu_obj) {
- list_del(&entry->head);
- break;
- }
- }
- spin_unlock(&rknpu_dev->lock);
-
- if (rknpu_obj == entry) {
- vunmap(rknpu_obj->kv_addr);
- rknpu_obj->kv_addr = NULL;
-
- if (!rknpu_obj->owner)
- dma_buf_put(rknpu_obj->dmabuf);
-
- kfree(rknpu_obj);
- }
-
- return 0;
-}
-
-/*
- * begin cpu access => for_cpu = true
- * end cpu access => for_cpu = false
- */
-static void __maybe_unused rknpu_dma_buf_sync(
- struct rknpu_device *rknpu_dev, struct rknpu_mem_object *rknpu_obj,
- u32 offset, u32 length, enum dma_data_direction dir, bool for_cpu)
-{
- struct device *dev = rknpu_dev->dev;
- struct sg_table *sgt = rknpu_obj->sgt;
- struct scatterlist *sg = sgt->sgl;
- dma_addr_t sg_dma_addr = sg_dma_address(sg);
- unsigned int len = 0;
- int i;
-
- for_each_sgtable_sg(sgt, sg, i) {
- unsigned int sg_offset, sg_left, size = 0;
-
- len += sg->length;
- if (len <= offset) {
- sg_dma_addr += sg->length;
- continue;
- }
-
- sg_left = len - offset;
- sg_offset = sg->length - sg_left;
-
- size = (length < sg_left) ? length : sg_left;
-
- if (for_cpu)
- dma_sync_single_range_for_cpu(dev, sg_dma_addr,
- sg_offset, size, dir);
- else
- dma_sync_single_range_for_device(dev, sg_dma_addr,
- sg_offset, size, dir);
-
- offset += size;
- length -= size;
- sg_dma_addr += sg->length;
-
- if (length == 0)
- break;
- }
-}
-
-int rknpu_mem_sync_ioctl(struct rknpu_device *rknpu_dev, unsigned long data)
-{
- struct rknpu_mem_object *rknpu_obj = NULL;
- struct rknpu_mem_sync args;
-#ifdef CONFIG_DMABUF_PARTIAL
- struct dma_buf *dmabuf;
-#endif
- int ret = -EFAULT;
-
- if (unlikely(copy_from_user(&args, (struct rknpu_mem_sync *)data,
- sizeof(struct rknpu_mem_sync)))) {
- LOG_ERROR("%s: copy_from_user failed\n", __func__);
- ret = -EFAULT;
- return ret;
- }
-
- if (!kern_addr_valid(args.obj_addr)) {
- LOG_ERROR("%s: invalid obj_addr: %#llx\n", __func__,
- (__u64)(uintptr_t)args.obj_addr);
- ret = -EINVAL;
- return ret;
- }
-
- rknpu_obj = (struct rknpu_mem_object *)(uintptr_t)args.obj_addr;
-
-#ifndef CONFIG_DMABUF_PARTIAL
- if (args.flags & RKNPU_MEM_SYNC_TO_DEVICE) {
- rknpu_dma_buf_sync(rknpu_dev, rknpu_obj, args.offset, args.size,
- DMA_TO_DEVICE, false);
- }
- if (args.flags & RKNPU_MEM_SYNC_FROM_DEVICE) {
- rknpu_dma_buf_sync(rknpu_dev, rknpu_obj, args.offset, args.size,
- DMA_FROM_DEVICE, true);
- }
-#else
- dmabuf = rknpu_obj->dmabuf;
- if (args.flags & RKNPU_MEM_SYNC_TO_DEVICE) {
- dmabuf->ops->end_cpu_access_partial(dmabuf, DMA_TO_DEVICE,
- args.offset, args.size);
- }
- if (args.flags & RKNPU_MEM_SYNC_FROM_DEVICE) {
- dmabuf->ops->begin_cpu_access_partial(dmabuf, DMA_FROM_DEVICE,
- args.offset, args.size);
- }
-#endif
-
- return 0;
-}
-
-#endif
diff --git a/rknpu-driver/driver-0.9.6/rknpu_mm.c b/rknpu-driver/driver-0.9.6/rknpu_mm.c
deleted file mode 100644
index a21bb6d..0000000
--- a/rknpu-driver/driver-0.9.6/rknpu_mm.c
+++ /dev/null
@@ -1,238 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) Rockchip Electronics Co.Ltd
- * Author: Felix Zeng
- */
-
-#include "rknpu_debugger.h"
-#include "rknpu_mm.h"
-
-int rknpu_mm_create(unsigned int mem_size, unsigned int chunk_size,
- struct rknpu_mm **mm)
-{
- unsigned int num_of_longs;
- int ret = -EINVAL;
-
- if (WARN_ON(mem_size < chunk_size))
- return -EINVAL;
- if (WARN_ON(mem_size == 0))
- return -EINVAL;
- if (WARN_ON(chunk_size == 0))
- return -EINVAL;
-
- *mm = kzalloc(sizeof(struct rknpu_mm), GFP_KERNEL);
- if (!(*mm))
- return -ENOMEM;
-
- (*mm)->chunk_size = chunk_size;
- (*mm)->total_chunks = mem_size / chunk_size;
- (*mm)->free_chunks = (*mm)->total_chunks;
-
- num_of_longs =
- ((*mm)->total_chunks + BITS_PER_LONG - 1) / BITS_PER_LONG;
-
- (*mm)->bitmap = kcalloc(num_of_longs, sizeof(long), GFP_KERNEL);
- if (!(*mm)->bitmap) {
- ret = -ENOMEM;
- goto free_mm;
- }
-
- mutex_init(&(*mm)->lock);
-
- LOG_DEBUG("total_chunks: %d, bitmap: %p\n", (*mm)->total_chunks,
- (*mm)->bitmap);
-
- return 0;
-
-free_mm:
- kfree(mm);
- return ret;
-}
-
-void rknpu_mm_destroy(struct rknpu_mm *mm)
-{
- if (mm != NULL) {
- mutex_destroy(&mm->lock);
- kfree(mm->bitmap);
- kfree(mm);
- }
-}
-
-int rknpu_mm_alloc(struct rknpu_mm *mm, unsigned int size,
- struct rknpu_mm_obj **mm_obj)
-{
- unsigned int found, start_search, cur_size;
-
- if (size == 0)
- return -EINVAL;
-
- if (size > mm->total_chunks * mm->chunk_size)
- return -ENOMEM;
-
- *mm_obj = kzalloc(sizeof(struct rknpu_mm_obj), GFP_KERNEL);
- if (!(*mm_obj))
- return -ENOMEM;
-
- start_search = 0;
-
- mutex_lock(&mm->lock);
-
-mm_restart_search:
- /* Find the first chunk that is free */
- found = find_next_zero_bit(mm->bitmap, mm->total_chunks, start_search);
-
- /* If there wasn't any free chunk, bail out */
- if (found == mm->total_chunks)
- goto mm_no_free_chunk;
-
- /* Update fields of mm_obj */
- (*mm_obj)->range_start = found;
- (*mm_obj)->range_end = found;
-
- /* If we need only one chunk, mark it as allocated and get out */
- if (size <= mm->chunk_size) {
- set_bit(found, mm->bitmap);
- goto mm_out;
- }
-
- /* Otherwise, try to see if we have enough contiguous chunks */
- cur_size = size - mm->chunk_size;
- do {
- (*mm_obj)->range_end = find_next_zero_bit(
- mm->bitmap, mm->total_chunks, ++found);
- /*
- * If next free chunk is not contiguous than we need to
- * restart our search from the last free chunk we found (which
- * wasn't contiguous to the previous ones
- */
- if ((*mm_obj)->range_end != found) {
- start_search = found;
- goto mm_restart_search;
- }
-
- /*
- * If we reached end of buffer, bail out with error
- */
- if (found == mm->total_chunks)
- goto mm_no_free_chunk;
-
- /* Check if we don't need another chunk */
- if (cur_size <= mm->chunk_size)
- cur_size = 0;
- else
- cur_size -= mm->chunk_size;
-
- } while (cur_size > 0);
-
- /* Mark the chunks as allocated */
- for (found = (*mm_obj)->range_start; found <= (*mm_obj)->range_end;
- found++)
- set_bit(found, mm->bitmap);
-
-mm_out:
- mm->free_chunks -= ((*mm_obj)->range_end - (*mm_obj)->range_start + 1);
- mutex_unlock(&mm->lock);
-
- LOG_DEBUG("mm allocate, mm_obj: %p, range_start: %d, range_end: %d\n",
- *mm_obj, (*mm_obj)->range_start, (*mm_obj)->range_end);
-
- return 0;
-
-mm_no_free_chunk:
- mutex_unlock(&mm->lock);
- kfree(*mm_obj);
-
- return -ENOMEM;
-}
-
-int rknpu_mm_free(struct rknpu_mm *mm, struct rknpu_mm_obj *mm_obj)
-{
- unsigned int bit;
-
- /* Act like kfree when trying to free a NULL object */
- if (!mm_obj)
- return 0;
-
- LOG_DEBUG("mm free, mem_obj: %p, range_start: %d, range_end: %d\n",
- mm_obj, mm_obj->range_start, mm_obj->range_end);
-
- mutex_lock(&mm->lock);
-
- /* Mark the chunks as free */
- for (bit = mm_obj->range_start; bit <= mm_obj->range_end; bit++)
- clear_bit(bit, mm->bitmap);
-
- mm->free_chunks += (mm_obj->range_end - mm_obj->range_start + 1);
-
- mutex_unlock(&mm->lock);
-
- kfree(mm_obj);
-
- return 0;
-}
-
-int rknpu_mm_dump(struct seq_file *m, void *data)
-{
- struct rknpu_debugger_node *node = m->private;
- struct rknpu_debugger *debugger = node->debugger;
- struct rknpu_device *rknpu_dev =
- container_of(debugger, struct rknpu_device, debugger);
- struct rknpu_mm *mm = NULL;
- int cur = 0, rbot = 0, rtop = 0;
- size_t ret = 0;
- char buf[64];
- size_t size = sizeof(buf);
- int seg_chunks = 32, seg_id = 0;
- int free_size = 0;
- int i = 0;
-
- mm = rknpu_dev->sram_mm;
- if (mm == NULL)
- return 0;
-
- seq_printf(m, "SRAM bitmap: \"*\" - used, \".\" - free (1bit = %dKB)\n",
- mm->chunk_size / 1024);
-
- rbot = cur = find_first_bit(mm->bitmap, mm->total_chunks);
- for (i = 0; i < cur; ++i) {
- ret += scnprintf(buf + ret, size - ret, ".");
- if (ret >= seg_chunks) {
- seq_printf(m, "[%03d] [%s]\n", seg_id++, buf);
- ret = 0;
- }
- }
- while (cur < mm->total_chunks) {
- rtop = cur;
- cur = find_next_bit(mm->bitmap, mm->total_chunks, cur + 1);
- if (cur < mm->total_chunks && cur <= rtop + 1)
- continue;
-
- for (i = rbot; i <= rtop; ++i) {
- ret += scnprintf(buf + ret, size - ret, "*");
- if (ret >= seg_chunks) {
- seq_printf(m, "[%03d] [%s]\n", seg_id++, buf);
- ret = 0;
- }
- }
-
- for (i = rtop + 1; i < cur; ++i) {
- ret += scnprintf(buf + ret, size - ret, ".");
- if (ret >= seg_chunks) {
- seq_printf(m, "[%03d] [%s]\n", seg_id++, buf);
- ret = 0;
- }
- }
-
- rbot = cur;
- }
-
- if (ret > 0)
- seq_printf(m, "[%03d] [%s]\n", seg_id++, buf);
-
- free_size = mm->free_chunks * mm->chunk_size;
- seq_printf(m, "SRAM total size: %d, used: %d, free: %d\n",
- rknpu_dev->sram_size, rknpu_dev->sram_size - free_size,
- free_size);
-
- return 0;
-}
diff --git a/rknpu-driver/driver-0.9.6/rknpu_reset.c b/rknpu-driver/driver-0.9.6/rknpu_reset.c
deleted file mode 100644
index 36657a7..0000000
--- a/rknpu-driver/driver-0.9.6/rknpu_reset.c
+++ /dev/null
@@ -1,155 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) Rockchip Electronics Co.Ltd
- * Author: Felix Zeng
- */
-
-#include
-#include
-
-#include "rknpu_reset.h"
-
-#ifndef FPGA_PLATFORM
-static inline struct reset_control *rknpu_reset_control_get(struct device *dev,
- const char *name)
-{
- struct reset_control *rst = NULL;
-
- rst = devm_reset_control_get(dev, name);
- if (IS_ERR(rst))
- LOG_DEV_ERROR(dev,
- "failed to get rknpu reset control: %s, %ld\n",
- name, PTR_ERR(rst));
-
- return rst;
-}
-#endif
-
-int rknpu_reset_get(struct rknpu_device *rknpu_dev)
-{
-#ifndef FPGA_PLATFORM
- int i = 0;
- int num_srsts = 0;
-
- num_srsts = of_count_phandle_with_args(rknpu_dev->dev->of_node,
- "resets", "#reset-cells");
- if (num_srsts <= 0) {
- LOG_DEV_ERROR(rknpu_dev->dev,
- "failed to get rknpu resets from dtb\n");
- return num_srsts;
- }
-
- rknpu_dev->srsts = devm_kcalloc(rknpu_dev->dev, num_srsts,
- sizeof(*rknpu_dev->srsts), GFP_KERNEL);
- if (!rknpu_dev->srsts)
- return -ENOMEM;
-
- for (i = 0; i < num_srsts; ++i) {
- rknpu_dev->srsts[i] = devm_reset_control_get_exclusive_by_index(
- rknpu_dev->dev, i);
- if (IS_ERR(rknpu_dev->srsts[i])) {
- rknpu_dev->num_srsts = i;
- return PTR_ERR(rknpu_dev->srsts[i]);
- }
- }
-
- rknpu_dev->num_srsts = num_srsts;
-
- return num_srsts;
-#endif
-
- return 0;
-}
-
-#ifndef FPGA_PLATFORM
-static int rknpu_reset_assert(struct reset_control *rst)
-{
- int ret = -EINVAL;
-
- if (!rst)
- return -EINVAL;
-
- ret = reset_control_assert(rst);
- if (ret < 0) {
- LOG_ERROR("failed to assert rknpu reset: %d\n", ret);
- return ret;
- }
-
- return 0;
-}
-
-static int rknpu_reset_deassert(struct reset_control *rst)
-{
- int ret = -EINVAL;
-
- if (!rst)
- return -EINVAL;
-
- ret = reset_control_deassert(rst);
- if (ret < 0) {
- LOG_ERROR("failed to deassert rknpu reset: %d\n", ret);
- return ret;
- }
-
- return 0;
-}
-#endif
-
-int rknpu_soft_reset(struct rknpu_device *rknpu_dev)
-{
-#ifndef FPGA_PLATFORM
- struct iommu_domain *domain = NULL;
- struct rknpu_subcore_data *subcore_data = NULL;
- int ret = 0, i = 0;
-
- if (rknpu_dev->bypass_soft_reset) {
- LOG_WARN("bypass soft reset\n");
- return 0;
- }
-
- if (!mutex_trylock(&rknpu_dev->reset_lock))
- return 0;
-
- rknpu_dev->soft_reseting = true;
-
- msleep(100);
-
- for (i = 0; i < rknpu_dev->config->num_irqs; ++i) {
- subcore_data = &rknpu_dev->subcore_datas[i];
- wake_up(&subcore_data->job_done_wq);
- }
-
- LOG_INFO("soft reset, num: %d\n", rknpu_dev->num_srsts);
-
- for (i = 0; i < rknpu_dev->num_srsts; ++i)
- ret |= rknpu_reset_assert(rknpu_dev->srsts[i]);
-
- udelay(10);
-
- for (i = 0; i < rknpu_dev->num_srsts; ++i)
- ret |= rknpu_reset_deassert(rknpu_dev->srsts[i]);
-
- udelay(10);
-
- if (ret) {
- LOG_DEV_ERROR(rknpu_dev->dev,
- "failed to soft reset for rknpu: %d\n", ret);
- mutex_unlock(&rknpu_dev->reset_lock);
- return ret;
- }
-
- if (rknpu_dev->iommu_en)
- domain = iommu_get_domain_for_dev(rknpu_dev->dev);
-
- if (domain) {
- iommu_detach_device(domain, rknpu_dev->dev);
- iommu_attach_device(domain, rknpu_dev->dev);
- }
-
- rknpu_dev->soft_reseting = false;
-
- mutex_unlock(&rknpu_dev->reset_lock);
-#endif
-
- return 0;
-}