1	// SPDX-License-Identifier: GPL-2.0 OR MIT
2	/*
3	* Copyright 2014-2022 Advanced Micro Devices, Inc.
4	*
5	* Permission is hereby granted, free of charge, to any person obtaining a
6	* copy of this software and associated documentation files (the "Software"),
7	* to deal in the Software without restriction, including without limitation
8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
9	* and/or sell copies of the Software, and to permit persons to whom the
10	* Software is furnished to do so, subject to the following conditions:
11	*
12	* The above copyright notice and this permission notice shall be included in
13	* all copies or substantial portions of the Software.
14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21	* OTHER DEALINGS IN THE SOFTWARE.
22	*/
23
24	#include <linux/device.h>
25	#include <linux/err.h>
26	#include <linux/fs.h>
27	#include <linux/file.h>
28	#include <linux/sched.h>
29	#include <linux/slab.h>
30	#include <linux/uaccess.h>
31	#include <linux/compat.h>
32	#include <uapi/linux/kfd_ioctl.h>
33	#include <linux/time.h>
34	#include <linux/mm.h>
35	#include <linux/mman.h>
36	#include <linux/ptrace.h>
37	#include <linux/dma-buf.h>
38	#include <linux/processor.h>
39	#include "kfd_priv.h"
40	#include "kfd_device_queue_manager.h"
41	#include "kfd_svm.h"
42	#include "amdgpu_amdkfd.h"
43	#include "kfd_smi_events.h"
44	#include "amdgpu_dma_buf.h"
45	#include "kfd_debug.h"
46
47	static long kfd_ioctl(struct file *, unsigned int, unsigned long);
48	static int kfd_open(struct inode *, struct file *);
49	static int kfd_release(struct inode *, struct file *);
50	static int kfd_mmap(struct file *, struct vm_area_struct *);
51
52	static const char kfd_dev_name[] = "kfd";
53
54	static const struct file_operations kfd_fops = {
55	.owner = THIS_MODULE,
56	.unlocked_ioctl = kfd_ioctl,
57	.compat_ioctl = compat_ptr_ioctl,
58	.open = kfd_open,
59	.release = kfd_release,
60	.mmap = kfd_mmap,
61	};
62
63	static int kfd_char_dev_major = -1;
64	struct device *kfd_device;
65	static const struct class kfd_class = {
66	.name = kfd_dev_name,
67	};
68
69	static inline struct kfd_process_device *kfd_lock_pdd_by_id(struct kfd_process *p, __u32 gpu_id)
70	{
71	struct kfd_process_device *pdd;
72
73	mutex_lock(&p->mutex);
74	pdd = kfd_process_device_data_by_id(process: p, gpu_id);
75
76	if (pdd)
77	return pdd;
78
79	mutex_unlock(lock: &p->mutex);
80	return NULL;
81	}
82
83	static inline void kfd_unlock_pdd(struct kfd_process_device *pdd)
84	{
85	mutex_unlock(lock: &pdd->process->mutex);
86	}
87
88	int kfd_chardev_init(void)
89	{
90	int err = 0;
91
92	kfd_char_dev_major = register_chrdev(major: 0, name: kfd_dev_name, fops: &kfd_fops);
93	err = kfd_char_dev_major;
94	if (err < 0)
95	goto err_register_chrdev;
96
97	err = class_register(class: &kfd_class);
98	if (err)
99	goto err_class_create;
100
101	kfd_device = device_create(cls: &kfd_class, NULL,
102	MKDEV(kfd_char_dev_major, 0),
103	NULL, fmt: kfd_dev_name);
104	err = PTR_ERR(ptr: kfd_device);
105	if (IS_ERR(ptr: kfd_device))
106	goto err_device_create;
107
108	return 0;
109
110	err_device_create:
111	class_unregister(class: &kfd_class);
112	err_class_create:
113	unregister_chrdev(major: kfd_char_dev_major, name: kfd_dev_name);
114	err_register_chrdev:
115	return err;
116	}
117
118	void kfd_chardev_exit(void)
119	{
120	device_destroy(cls: &kfd_class, MKDEV(kfd_char_dev_major, 0));
121	class_unregister(class: &kfd_class);
122	unregister_chrdev(major: kfd_char_dev_major, name: kfd_dev_name);
123	kfd_device = NULL;
124	}
125
126
127	static int kfd_open(struct inode *inode, struct file *filep)
128	{
129	struct kfd_process *process;
130	bool is_32bit_user_mode;
131
132	if (iminor(inode) != 0)
133	return -ENODEV;
134
135	is_32bit_user_mode = in_compat_syscall();
136
137	if (is_32bit_user_mode) {
138	dev_warn(kfd_device,
139	"Process %d (32-bit) failed to open /dev/kfd\n"
140	"32-bit processes are not supported by amdkfd\n",
141	current->pid);
142	return -EPERM;
143	}
144
145	process = kfd_create_process(current);
146	if (IS_ERR(ptr: process))
147	return PTR_ERR(ptr: process);
148
149	if (kfd_process_init_cwsr_apu(process, filep)) {
150	kfd_unref_process(p: process);
151	return -EFAULT;
152	}
153
154	/* filep now owns the reference returned by kfd_create_process */
155	filep->private_data = process;
156
157	dev_dbg(kfd_device, "process pid %d opened kfd node, compat mode (32 bit) - %d\n",
158	process->lead_thread->pid, process->is_32bit_user_mode);
159
160	return 0;
161	}
162
163	static int kfd_release(struct inode *inode, struct file *filep)
164	{
165	struct kfd_process *process = filep->private_data;
166
167	if (process)
168	kfd_unref_process(p: process);
169
170	return 0;
171	}
172
173	static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p,
174	void *data)
175	{
176	struct kfd_ioctl_get_version_args *args = data;
177
178	args->major_version = KFD_IOCTL_MAJOR_VERSION;
179	args->minor_version = KFD_IOCTL_MINOR_VERSION;
180
181	return 0;
182	}
183
184	static int set_queue_properties_from_user(struct queue_properties *q_properties,
185	struct kfd_ioctl_create_queue_args *args)
186	{
187	/*
188	* Repurpose queue percentage to accommodate new features:
189	* bit 0-7: queue percentage
190	* bit 8-15: pm4_target_xcc
191	*/
192	if ((args->queue_percentage & 0xFF) > KFD_MAX_QUEUE_PERCENTAGE) {
193	pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
194	return -EINVAL;
195	}
196
197	if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
198	pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
199	return -EINVAL;
200	}
201
202	if ((args->ring_base_address) &&
203	(!access_ok((const void __user *) args->ring_base_address,
204	sizeof(uint64_t)))) {
205	pr_err("Can't access ring base address\n");
206	return -EFAULT;
207	}
208
209	if (!is_power_of_2(n: args->ring_size) && (args->ring_size != 0)) {
210	pr_err("Ring size must be a power of 2 or 0\n");
211	return -EINVAL;
212	}
213
214	if (args->ring_size < KFD_MIN_QUEUE_RING_SIZE) {
215	args->ring_size = KFD_MIN_QUEUE_RING_SIZE;
216	pr_debug("Size lower. clamped to KFD_MIN_QUEUE_RING_SIZE");
217	}
218
219	if (!access_ok((const void __user *) args->read_pointer_address,
220	sizeof(uint32_t))) {
221	pr_err("Can't access read pointer\n");
222	return -EFAULT;
223	}
224
225	if (!access_ok((const void __user *) args->write_pointer_address,
226	sizeof(uint32_t))) {
227	pr_err("Can't access write pointer\n");
228	return -EFAULT;
229	}
230
231	if (args->eop_buffer_address &&
232	!access_ok((const void __user *) args->eop_buffer_address,
233	sizeof(uint32_t))) {
234	pr_debug("Can't access eop buffer");
235	return -EFAULT;
236	}
237
238	if (args->ctx_save_restore_address &&
239	!access_ok((const void __user *) args->ctx_save_restore_address,
240	sizeof(uint32_t))) {
241	pr_debug("Can't access ctx save restore buffer");
242	return -EFAULT;
243	}
244
245	q_properties->is_interop = false;
246	q_properties->is_gws = false;
247	q_properties->queue_percent = args->queue_percentage & 0xFF;
248	/* bit 8-15 are repurposed to be PM4 target XCC */
249	q_properties->pm4_target_xcc = (args->queue_percentage >> 8) & 0xFF;
250	q_properties->priority = args->queue_priority;
251	q_properties->queue_address = args->ring_base_address;
252	q_properties->queue_size = args->ring_size;
253	q_properties->read_ptr = (void __user *)args->read_pointer_address;
254	q_properties->write_ptr = (void __user *)args->write_pointer_address;
255	q_properties->eop_ring_buffer_address = args->eop_buffer_address;
256	q_properties->eop_ring_buffer_size = args->eop_buffer_size;
257	q_properties->ctx_save_restore_area_address =
258	args->ctx_save_restore_address;
259	q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size;
260	q_properties->ctl_stack_size = args->ctl_stack_size;
261	q_properties->sdma_engine_id = args->sdma_engine_id;
262	if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE ||
263	args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
264	q_properties->type = KFD_QUEUE_TYPE_COMPUTE;
265	else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA)
266	q_properties->type = KFD_QUEUE_TYPE_SDMA;
267	else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_XGMI)
268	q_properties->type = KFD_QUEUE_TYPE_SDMA_XGMI;
269	else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_BY_ENG_ID)
270	q_properties->type = KFD_QUEUE_TYPE_SDMA_BY_ENG_ID;
271	else
272	return -ENOTSUPP;
273
274	if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
275	q_properties->format = KFD_QUEUE_FORMAT_AQL;
276	else
277	q_properties->format = KFD_QUEUE_FORMAT_PM4;
278
279	pr_debug("Queue Percentage: %d, %d\n",
280	q_properties->queue_percent, args->queue_percentage);
281
282	pr_debug("Queue Priority: %d, %d\n",
283	q_properties->priority, args->queue_priority);
284
285	pr_debug("Queue Address: 0x%llX, 0x%llX\n",
286	q_properties->queue_address, args->ring_base_address);
287
288	pr_debug("Queue Size: 0x%llX, %u\n",
289	q_properties->queue_size, args->ring_size);
290
291	pr_debug("Queue r/w Pointers: %px, %px\n",
292	q_properties->read_ptr,
293	q_properties->write_ptr);
294
295	pr_debug("Queue Format: %d\n", q_properties->format);
296
297	pr_debug("Queue EOP: 0x%llX\n", q_properties->eop_ring_buffer_address);
298
299	pr_debug("Queue CTX save area: 0x%llX\n",
300	q_properties->ctx_save_restore_area_address);
301
302	return 0;
303	}
304
305	static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
306	void *data)
307	{
308	struct kfd_ioctl_create_queue_args *args = data;
309	struct kfd_node *dev;
310	int err = 0;
311	unsigned int queue_id;
312	struct kfd_process_device *pdd;
313	struct queue_properties q_properties;
314	uint32_t doorbell_offset_in_process = 0;
315
316	memset(&q_properties, 0, sizeof(struct queue_properties));
317
318	pr_debug("Creating queue ioctl\n");
319
320	err = set_queue_properties_from_user(q_properties: &q_properties, args);
321	if (err)
322	return err;
323
324	pr_debug("Looking for gpu id 0x%x\n", args->gpu_id);
325
326	mutex_lock(&p->mutex);
327
328	pdd = kfd_process_device_data_by_id(process: p, gpu_id: args->gpu_id);
329	if (!pdd) {
330	pr_debug("Could not find gpu id 0x%x\n", args->gpu_id);
331	err = -EINVAL;
332	goto err_pdd;
333	}
334	dev = pdd->dev;
335
336	pdd = kfd_bind_process_to_device(dev, p);
337	if (IS_ERR(ptr: pdd)) {
338	err = -ESRCH;
339	goto err_bind_process;
340	}
341
342	if (q_properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) {
343	int max_sdma_eng_id = kfd_get_num_sdma_engines(kdev: dev) +
344	kfd_get_num_xgmi_sdma_engines(kdev: dev) - 1;
345
346	if (q_properties.sdma_engine_id > max_sdma_eng_id) {
347	err = -EINVAL;
348	pr_err("sdma_engine_id %i exceeds maximum id of %i\n",
349	q_properties.sdma_engine_id, max_sdma_eng_id);
350	goto err_sdma_engine_id;
351	}
352	}
353
354	if (!pdd->qpd.proc_doorbells) {
355	err = kfd_alloc_process_doorbells(kfd: dev->kfd, pdd);
356	if (err) {
357	pr_debug("failed to allocate process doorbells\n");
358	goto err_bind_process;
359	}
360	}
361
362	err = kfd_queue_acquire_buffers(pdd, properties: &q_properties);
363	if (err) {
364	pr_debug("failed to acquire user queue buffers\n");
365	goto err_acquire_queue_buf;
366	}
367
368	pr_debug("Creating queue for process pid %d on gpu 0x%x\n",
369	p->lead_thread->pid,
370	dev->id);
371
372	err = pqm_create_queue(pqm: &p->pqm, dev, properties: &q_properties, qid: &queue_id,
373	NULL, NULL, NULL, p_doorbell_offset_in_process: &doorbell_offset_in_process);
374	if (err != 0)
375	goto err_create_queue;
376
377	args->queue_id = queue_id;
378
379
380	/* Return gpu_id as doorbell offset for mmap usage */
381	args->doorbell_offset = KFD_MMAP_TYPE_DOORBELL;
382	args->doorbell_offset |= KFD_MMAP_GPU_ID(args->gpu_id);
383	if (KFD_IS_SOC15(dev))
384	/* On SOC15 ASICs, include the doorbell offset within the
385	* process doorbell frame, which is 2 pages.
386	*/
387	args->doorbell_offset |= doorbell_offset_in_process;
388
389	mutex_unlock(lock: &p->mutex);
390
391	pr_debug("Queue id %d was created successfully\n", args->queue_id);
392
393	pr_debug("Ring buffer address == 0x%016llX\n",
394	args->ring_base_address);
395
396	pr_debug("Read ptr address == 0x%016llX\n",
397	args->read_pointer_address);
398
399	pr_debug("Write ptr address == 0x%016llX\n",
400	args->write_pointer_address);
401
402	kfd_dbg_ev_raise(KFD_EC_MASK(EC_QUEUE_NEW), process: p, dev, source_id: queue_id, use_worker: false, NULL, exception_data_size: 0);
403	return 0;
404
405	err_create_queue:
406	kfd_queue_unref_bo_vas(pdd, properties: &q_properties);
407	kfd_queue_release_buffers(pdd, properties: &q_properties);
408	err_acquire_queue_buf:
409	err_sdma_engine_id:
410	err_bind_process:
411	err_pdd:
412	mutex_unlock(lock: &p->mutex);
413	return err;
414	}
415
416	static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p,
417	void *data)
418	{
419	int retval;
420	struct kfd_ioctl_destroy_queue_args *args = data;
421
422	pr_debug("Destroying queue id %d for process pid %d\n",
423	args->queue_id,
424	p->lead_thread->pid);
425
426	mutex_lock(&p->mutex);
427
428	retval = pqm_destroy_queue(pqm: &p->pqm, qid: args->queue_id);
429
430	mutex_unlock(lock: &p->mutex);
431	return retval;
432	}
433
434	static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
435	void *data)
436	{
437	int retval;
438	struct kfd_ioctl_update_queue_args *args = data;
439	struct queue_properties properties;
440
441	/*
442	* Repurpose queue percentage to accommodate new features:
443	* bit 0-7: queue percentage
444	* bit 8-15: pm4_target_xcc
445	*/
446	if ((args->queue_percentage & 0xFF) > KFD_MAX_QUEUE_PERCENTAGE) {
447	pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
448	return -EINVAL;
449	}
450
451	if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
452	pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
453	return -EINVAL;
454	}
455
456	if ((args->ring_base_address) &&
457	(!access_ok((const void __user *) args->ring_base_address,
458	sizeof(uint64_t)))) {
459	pr_err("Can't access ring base address\n");
460	return -EFAULT;
461	}
462
463	if (!is_power_of_2(n: args->ring_size) && (args->ring_size != 0)) {
464	pr_err("Ring size must be a power of 2 or 0\n");
465	return -EINVAL;
466	}
467
468	if (args->ring_size < KFD_MIN_QUEUE_RING_SIZE) {
469	args->ring_size = KFD_MIN_QUEUE_RING_SIZE;
470	pr_debug("Size lower. clamped to KFD_MIN_QUEUE_RING_SIZE");
471	}
472
473	properties.queue_address = args->ring_base_address;
474	properties.queue_size = args->ring_size;
475	properties.queue_percent = args->queue_percentage & 0xFF;
476	/* bit 8-15 are repurposed to be PM4 target XCC */
477	properties.pm4_target_xcc = (args->queue_percentage >> 8) & 0xFF;
478	properties.priority = args->queue_priority;
479
480	pr_debug("Updating queue id %d for process pid %d\n",
481	args->queue_id, p->lead_thread->pid);
482
483	mutex_lock(&p->mutex);
484
485	retval = pqm_update_queue_properties(pqm: &p->pqm, qid: args->queue_id, p: &properties);
486
487	mutex_unlock(lock: &p->mutex);
488
489	return retval;
490	}
491
492	static int kfd_ioctl_set_cu_mask(struct file *filp, struct kfd_process *p,
493	void *data)
494	{
495	int retval;
496	const int max_num_cus = 1024;
497	struct kfd_ioctl_set_cu_mask_args *args = data;
498	struct mqd_update_info minfo = {0};
499	uint32_t __user *cu_mask_ptr = (uint32_t __user *)args->cu_mask_ptr;
500	size_t cu_mask_size = sizeof(uint32_t) * (args->num_cu_mask / 32);
501
502	if ((args->num_cu_mask % 32) != 0) {
503	pr_debug("num_cu_mask 0x%x must be a multiple of 32",
504	args->num_cu_mask);
505	return -EINVAL;
506	}
507
508	minfo.cu_mask.count = args->num_cu_mask;
509	if (minfo.cu_mask.count == 0) {
510	pr_debug("CU mask cannot be 0");
511	return -EINVAL;
512	}
513
514	/* To prevent an unreasonably large CU mask size, set an arbitrary
515	* limit of max_num_cus bits. We can then just drop any CU mask bits
516	* past max_num_cus bits and just use the first max_num_cus bits.
517	*/
518	if (minfo.cu_mask.count > max_num_cus) {
519	pr_debug("CU mask cannot be greater than 1024 bits");
520	minfo.cu_mask.count = max_num_cus;
521	cu_mask_size = sizeof(uint32_t) * (max_num_cus/32);
522	}
523
524	minfo.cu_mask.ptr = memdup_user(cu_mask_ptr, cu_mask_size);
525	if (IS_ERR(ptr: minfo.cu_mask.ptr)) {
526	pr_debug("Could not copy CU mask from userspace");
527	return PTR_ERR(ptr: minfo.cu_mask.ptr);
528	}
529
530	mutex_lock(&p->mutex);
531
532	retval = pqm_update_mqd(pqm: &p->pqm, qid: args->queue_id, minfo: &minfo);
533
534	mutex_unlock(lock: &p->mutex);
535
536	kfree(objp: minfo.cu_mask.ptr);
537	return retval;
538	}
539
540	static int kfd_ioctl_get_queue_wave_state(struct file *filep,
541	struct kfd_process *p, void *data)
542	{
543	struct kfd_ioctl_get_queue_wave_state_args *args = data;
544	int r;
545
546	mutex_lock(&p->mutex);
547
548	r = pqm_get_wave_state(pqm: &p->pqm, qid: args->queue_id,
549	ctl_stack: (void __user *)args->ctl_stack_address,
550	ctl_stack_used_size: &args->ctl_stack_used_size,
551	save_area_used_size: &args->save_area_used_size);
552
553	mutex_unlock(lock: &p->mutex);
554
555	return r;
556	}
557
558	static int kfd_ioctl_set_memory_policy(struct file *filep,
559	struct kfd_process *p, void *data)
560	{
561	struct kfd_ioctl_set_memory_policy_args *args = data;
562	int err = 0;
563	struct kfd_process_device *pdd;
564	enum cache_policy default_policy, alternate_policy;
565
566	if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT
567	&& args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
568	return -EINVAL;
569	}
570
571	if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT
572	&& args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
573	return -EINVAL;
574	}
575
576	mutex_lock(&p->mutex);
577	pdd = kfd_process_device_data_by_id(process: p, gpu_id: args->gpu_id);
578	if (!pdd) {
579	pr_debug("Could not find gpu id 0x%x\n", args->gpu_id);
580	err = -EINVAL;
581	goto err_pdd;
582	}
583
584	pdd = kfd_bind_process_to_device(dev: pdd->dev, p);
585	if (IS_ERR(ptr: pdd)) {
586	err = -ESRCH;
587	goto out;
588	}
589
590	default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT)
591	? cache_policy_coherent : cache_policy_noncoherent;
592
593	alternate_policy =
594	(args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT)
595	? cache_policy_coherent : cache_policy_noncoherent;
596
597	if (!pdd->dev->dqm->ops.set_cache_memory_policy(pdd->dev->dqm,
598	&pdd->qpd,
599	default_policy,
600	alternate_policy,
601	(void __user *)args->alternate_aperture_base,
602	args->alternate_aperture_size,
603	args->misc_process_flag))
604	err = -EINVAL;
605
606	out:
607	err_pdd:
608	mutex_unlock(lock: &p->mutex);
609
610	return err;
611	}
612
613	static int kfd_ioctl_set_trap_handler(struct file *filep,
614	struct kfd_process *p, void *data)
615	{
616	struct kfd_ioctl_set_trap_handler_args *args = data;
617	int err = 0;
618	struct kfd_process_device *pdd;
619
620	mutex_lock(&p->mutex);
621
622	pdd = kfd_process_device_data_by_id(process: p, gpu_id: args->gpu_id);
623	if (!pdd) {
624	err = -EINVAL;
625	goto err_pdd;
626	}
627
628	pdd = kfd_bind_process_to_device(dev: pdd->dev, p);
629	if (IS_ERR(ptr: pdd)) {
630	err = -ESRCH;
631	goto out;
632	}
633
634	kfd_process_set_trap_handler(qpd: &pdd->qpd, tba_addr: args->tba_addr, tma_addr: args->tma_addr);
635
636	out:
637	err_pdd:
638	mutex_unlock(lock: &p->mutex);
639
640	return err;
641	}
642
643	static int kfd_ioctl_dbg_register(struct file *filep,
644	struct kfd_process *p, void *data)
645	{
646	return -EPERM;
647	}
648
649	static int kfd_ioctl_dbg_unregister(struct file *filep,
650	struct kfd_process *p, void *data)
651	{
652	return -EPERM;
653	}
654
655	static int kfd_ioctl_dbg_address_watch(struct file *filep,
656	struct kfd_process *p, void *data)
657	{
658	return -EPERM;
659	}
660
661	/* Parse and generate fixed size data structure for wave control */
662	static int kfd_ioctl_dbg_wave_control(struct file *filep,
663	struct kfd_process *p, void *data)
664	{
665	return -EPERM;
666	}
667
668	static int kfd_ioctl_get_clock_counters(struct file *filep,
669	struct kfd_process *p, void *data)
670	{
671	struct kfd_ioctl_get_clock_counters_args *args = data;
672	struct kfd_process_device *pdd;
673
674	mutex_lock(&p->mutex);
675	pdd = kfd_process_device_data_by_id(process: p, gpu_id: args->gpu_id);
676	mutex_unlock(lock: &p->mutex);
677	if (pdd)
678	/* Reading GPU clock counter from KGD */
679	args->gpu_clock_counter = amdgpu_amdkfd_get_gpu_clock_counter(adev: pdd->dev->adev);
680	else
681	/* Node without GPU resource */
682	args->gpu_clock_counter = 0;
683
684	/* No access to rdtsc. Using raw monotonic time */
685	args->cpu_clock_counter = ktime_get_raw_ns();
686	args->system_clock_counter = ktime_get_boottime_ns();
687
688	/* Since the counter is in nano-seconds we use 1GHz frequency */
689	args->system_clock_freq = 1000000000;
690
691	return 0;
692	}
693
694
695	static int kfd_ioctl_get_process_apertures(struct file *filp,
696	struct kfd_process *p, void *data)
697	{
698	struct kfd_ioctl_get_process_apertures_args *args = data;
699	struct kfd_process_device_apertures *pAperture;
700	int i;
701
702	dev_dbg(kfd_device, "get apertures for process pid %d", p->lead_thread->pid);
703
704	args->num_of_nodes = 0;
705
706	mutex_lock(&p->mutex);
707	/* Run over all pdd of the process */
708	for (i = 0; i < p->n_pdds; i++) {
709	struct kfd_process_device *pdd = p->pdds[i];
710
711	pAperture =
712	&args->process_apertures[args->num_of_nodes];
713	pAperture->gpu_id = pdd->dev->id;
714	pAperture->lds_base = pdd->lds_base;
715	pAperture->lds_limit = pdd->lds_limit;
716	pAperture->gpuvm_base = pdd->gpuvm_base;
717	pAperture->gpuvm_limit = pdd->gpuvm_limit;
718	pAperture->scratch_base = pdd->scratch_base;
719	pAperture->scratch_limit = pdd->scratch_limit;
720
721	dev_dbg(kfd_device,
722	"node id %u\n", args->num_of_nodes);
723	dev_dbg(kfd_device,
724	"gpu id %u\n", pdd->dev->id);
725	dev_dbg(kfd_device,
726	"lds_base %llX\n", pdd->lds_base);
727	dev_dbg(kfd_device,
728	"lds_limit %llX\n", pdd->lds_limit);
729	dev_dbg(kfd_device,
730	"gpuvm_base %llX\n", pdd->gpuvm_base);
731	dev_dbg(kfd_device,
732	"gpuvm_limit %llX\n", pdd->gpuvm_limit);
733	dev_dbg(kfd_device,
734	"scratch_base %llX\n", pdd->scratch_base);
735	dev_dbg(kfd_device,
736	"scratch_limit %llX\n", pdd->scratch_limit);
737
738	if (++args->num_of_nodes >= NUM_OF_SUPPORTED_GPUS)
739	break;
740	}
741	mutex_unlock(lock: &p->mutex);
742
743	return 0;
744	}
745
746	static int kfd_ioctl_get_process_apertures_new(struct file *filp,
747	struct kfd_process *p, void *data)
748	{
749	struct kfd_ioctl_get_process_apertures_new_args *args = data;
750	struct kfd_process_device_apertures *pa;
751	int ret;
752	int i;
753
754	dev_dbg(kfd_device, "get apertures for process pid %d",
755	p->lead_thread->pid);
756
757	if (args->num_of_nodes == 0) {
758	/* Return number of nodes, so that user space can alloacate
759	* sufficient memory
760	*/
761	mutex_lock(&p->mutex);
762	args->num_of_nodes = p->n_pdds;
763	goto out_unlock;
764	}
765
766	/* Fill in process-aperture information for all available
767	* nodes, but not more than args->num_of_nodes as that is
768	* the amount of memory allocated by user
769	*/
770	pa = kcalloc(args->num_of_nodes, sizeof(struct kfd_process_device_apertures),
771	GFP_KERNEL);
772	if (!pa)
773	return -ENOMEM;
774
775	mutex_lock(&p->mutex);
776
777	if (!p->n_pdds) {
778	args->num_of_nodes = 0;
779	kfree(objp: pa);
780	goto out_unlock;
781	}
782
783	/* Run over all pdd of the process */
784	for (i = 0; i < min(p->n_pdds, args->num_of_nodes); i++) {
785	struct kfd_process_device *pdd = p->pdds[i];
786
787	pa[i].gpu_id = pdd->dev->id;
788	pa[i].lds_base = pdd->lds_base;
789	pa[i].lds_limit = pdd->lds_limit;
790	pa[i].gpuvm_base = pdd->gpuvm_base;
791	pa[i].gpuvm_limit = pdd->gpuvm_limit;
792	pa[i].scratch_base = pdd->scratch_base;
793	pa[i].scratch_limit = pdd->scratch_limit;
794
795	dev_dbg(kfd_device,
796	"gpu id %u\n", pdd->dev->id);
797	dev_dbg(kfd_device,
798	"lds_base %llX\n", pdd->lds_base);
799	dev_dbg(kfd_device,
800	"lds_limit %llX\n", pdd->lds_limit);
801	dev_dbg(kfd_device,
802	"gpuvm_base %llX\n", pdd->gpuvm_base);
803	dev_dbg(kfd_device,
804	"gpuvm_limit %llX\n", pdd->gpuvm_limit);
805	dev_dbg(kfd_device,
806	"scratch_base %llX\n", pdd->scratch_base);
807	dev_dbg(kfd_device,
808	"scratch_limit %llX\n", pdd->scratch_limit);
809	}
810	mutex_unlock(lock: &p->mutex);
811
812	args->num_of_nodes = i;
813	ret = copy_to_user(
814	to: (void __user *)args->kfd_process_device_apertures_ptr,
815	from: pa,
816	n: (i * sizeof(struct kfd_process_device_apertures)));
817	kfree(objp: pa);
818	return ret ? -EFAULT : 0;
819
820	out_unlock:
821	mutex_unlock(lock: &p->mutex);
822	return 0;
823	}
824
825	static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
826	void *data)
827	{
828	struct kfd_ioctl_create_event_args *args = data;
829	int err;
830
831	/* For dGPUs the event page is allocated in user mode. The
832	* handle is passed to KFD with the first call to this IOCTL
833	* through the event_page_offset field.
834	*/
835	if (args->event_page_offset) {
836	mutex_lock(&p->mutex);
837	err = kfd_kmap_event_page(p, event_page_offset: args->event_page_offset);
838	mutex_unlock(lock: &p->mutex);
839	if (err)
840	return err;
841	}
842
843	err = kfd_event_create(devkfd: filp, p, event_type: args->event_type,
844	auto_reset: args->auto_reset != 0, node_id: args->node_id,
845	event_id: &args->event_id, event_trigger_data: &args->event_trigger_data,
846	event_page_offset: &args->event_page_offset,
847	event_slot_index: &args->event_slot_index);
848
849	pr_debug("Created event (id:0x%08x) (%s)\n", args->event_id, __func__);
850	return err;
851	}
852
853	static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p,
854	void *data)
855	{
856	struct kfd_ioctl_destroy_event_args *args = data;
857
858	return kfd_event_destroy(p, event_id: args->event_id);
859	}
860
861	static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p,
862	void *data)
863	{
864	struct kfd_ioctl_set_event_args *args = data;
865
866	return kfd_set_event(p, event_id: args->event_id);
867	}
868
869	static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p,
870	void *data)
871	{
872	struct kfd_ioctl_reset_event_args *args = data;
873
874	return kfd_reset_event(p, event_id: args->event_id);
875	}
876
877	static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
878	void *data)
879	{
880	struct kfd_ioctl_wait_events_args *args = data;
881
882	return kfd_wait_on_events(p, num_events: args->num_events,
883	data: (void __user *)args->events_ptr,
884	all: (args->wait_for_all != 0),
885	user_timeout_ms: &args->timeout, wait_result: &args->wait_result);
886	}
887	static int kfd_ioctl_set_scratch_backing_va(struct file *filep,
888	struct kfd_process *p, void *data)
889	{
890	struct kfd_ioctl_set_scratch_backing_va_args *args = data;
891	struct kfd_process_device *pdd;
892	struct kfd_node *dev;
893	long err;
894
895	mutex_lock(&p->mutex);
896	pdd = kfd_process_device_data_by_id(process: p, gpu_id: args->gpu_id);
897	if (!pdd) {
898	err = -EINVAL;
899	goto err_pdd;
900	}
901	dev = pdd->dev;
902
903	pdd = kfd_bind_process_to_device(dev, p);
904	if (IS_ERR(ptr: pdd)) {
905	err = PTR_ERR(ptr: pdd);
906	goto bind_process_to_device_fail;
907	}
908
909	pdd->qpd.sh_hidden_private_base = args->va_addr;
910
911	mutex_unlock(lock: &p->mutex);
912
913	if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS &&
914	pdd->qpd.vmid != 0 && dev->kfd2kgd->set_scratch_backing_va)
915	dev->kfd2kgd->set_scratch_backing_va(
916	dev->adev, args->va_addr, pdd->qpd.vmid);
917
918	return 0;
919
920	bind_process_to_device_fail:
921	err_pdd:
922	mutex_unlock(lock: &p->mutex);
923	return err;
924	}
925
926	static int kfd_ioctl_get_tile_config(struct file *filep,
927	struct kfd_process *p, void *data)
928	{
929	struct kfd_ioctl_get_tile_config_args *args = data;
930	struct kfd_process_device *pdd;
931	struct tile_config config;
932	int err = 0;
933
934	mutex_lock(&p->mutex);
935	pdd = kfd_process_device_data_by_id(process: p, gpu_id: args->gpu_id);
936	mutex_unlock(lock: &p->mutex);
937	if (!pdd)
938	return -EINVAL;
939
940	amdgpu_amdkfd_get_tile_config(adev: pdd->dev->adev, config: &config);
941
942	args->gb_addr_config = config.gb_addr_config;
943	args->num_banks = config.num_banks;
944	args->num_ranks = config.num_ranks;
945
946	if (args->num_tile_configs > config.num_tile_configs)
947	args->num_tile_configs = config.num_tile_configs;
948	err = copy_to_user(to: (void __user *)args->tile_config_ptr,
949	from: config.tile_config_ptr,
950	n: args->num_tile_configs * sizeof(uint32_t));
951	if (err) {
952	args->num_tile_configs = 0;
953	return -EFAULT;
954	}
955
956	if (args->num_macro_tile_configs > config.num_macro_tile_configs)
957	args->num_macro_tile_configs =
958	config.num_macro_tile_configs;
959	err = copy_to_user(to: (void __user *)args->macro_tile_config_ptr,
960	from: config.macro_tile_config_ptr,
961	n: args->num_macro_tile_configs * sizeof(uint32_t));
962	if (err) {
963	args->num_macro_tile_configs = 0;
964	return -EFAULT;
965	}
966
967	return 0;
968	}
969
970	static int kfd_ioctl_acquire_vm(struct file *filep, struct kfd_process *p,
971	void *data)
972	{
973	struct kfd_ioctl_acquire_vm_args *args = data;
974	struct kfd_process_device *pdd;
975	struct file *drm_file;
976	int ret;
977
978	drm_file = fget(fd: args->drm_fd);
979	if (!drm_file)
980	return -EINVAL;
981
982	mutex_lock(&p->mutex);
983	pdd = kfd_process_device_data_by_id(process: p, gpu_id: args->gpu_id);
984	if (!pdd) {
985	ret = -EINVAL;
986	goto err_pdd;
987	}
988
989	if (pdd->drm_file) {
990	ret = pdd->drm_file == drm_file ? 0 : -EBUSY;
991	goto err_drm_file;
992	}
993
994	ret = kfd_process_device_init_vm(pdd, drm_file);
995	if (ret)
996	goto err_unlock;
997
998	/* On success, the PDD keeps the drm_file reference */
999	mutex_unlock(lock: &p->mutex);
1000
1001	return 0;
1002
1003	err_unlock:
1004	err_pdd:
1005	err_drm_file:
1006	mutex_unlock(lock: &p->mutex);
1007	fput(drm_file);
1008	return ret;
1009	}
1010
1011	bool kfd_dev_is_large_bar(struct kfd_node *dev)
1012	{
1013	if (dev->kfd->adev->debug_largebar) {
1014	pr_debug("Simulate large-bar allocation on non large-bar machine\n");
1015	return true;
1016	}
1017
1018	if (dev->local_mem_info.local_mem_size_private == 0 &&
1019	dev->local_mem_info.local_mem_size_public > 0)
1020	return true;
1021
1022	if (dev->local_mem_info.local_mem_size_public == 0 &&
1023	dev->kfd->adev->gmc.is_app_apu) {
1024	pr_debug("APP APU, Consider like a large bar system\n");
1025	return true;
1026	}
1027
1028	return false;
1029	}
1030
1031	static int kfd_ioctl_get_available_memory(struct file *filep,
1032	struct kfd_process *p, void *data)
1033	{
1034	struct kfd_ioctl_get_available_memory_args *args = data;
1035	struct kfd_process_device *pdd = kfd_lock_pdd_by_id(p, gpu_id: args->gpu_id);
1036
1037	if (!pdd)
1038	return -EINVAL;
1039	args->available = amdgpu_amdkfd_get_available_memory(adev: pdd->dev->adev,
1040	xcp_id: pdd->dev->node_id);
1041	kfd_unlock_pdd(pdd);
1042	return 0;
1043	}
1044
1045	static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
1046	struct kfd_process *p, void *data)
1047	{
1048	struct kfd_ioctl_alloc_memory_of_gpu_args *args = data;
1049	struct kfd_process_device *pdd;
1050	void *mem;
1051	struct kfd_node *dev;
1052	int idr_handle;
1053	long err;
1054	uint64_t offset = args->mmap_offset;
1055	uint32_t flags = args->flags;
1056
1057	if (args->size == 0)
1058	return -EINVAL;
1059
1060	#if IS_ENABLED(CONFIG_HSA_AMD_SVM)
1061	/* Flush pending deferred work to avoid racing with deferred actions
1062	* from previous memory map changes (e.g. munmap).
1063	*/
1064	svm_range_list_lock_and_flush_work(svms: &p->svms, current->mm);
1065	mutex_lock(&p->svms.lock);
1066	mmap_write_unlock(current->mm);
1067
1068	/* Skip a special case that allocates VRAM without VA,
1069	* VA will be invalid of 0.
1070	*/
1071	if (!(!args->va_addr && (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)) &&
1072	interval_tree_iter_first(root: &p->svms.objects,
1073	start: args->va_addr >> PAGE_SHIFT,
1074	last: (args->va_addr + args->size - 1) >> PAGE_SHIFT)) {
1075	pr_err("Address: 0x%llx already allocated by SVM\n",
1076	args->va_addr);
1077	mutex_unlock(lock: &p->svms.lock);
1078	return -EADDRINUSE;
1079	}
1080
1081	/* When register user buffer check if it has been registered by svm by
1082	* buffer cpu virtual address.
1083	*/
1084	if ((flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) &&
1085	interval_tree_iter_first(root: &p->svms.objects,
1086	start: args->mmap_offset >> PAGE_SHIFT,
1087	last: (args->mmap_offset + args->size - 1) >> PAGE_SHIFT)) {
1088	pr_err("User Buffer Address: 0x%llx already allocated by SVM\n",
1089	args->mmap_offset);
1090	mutex_unlock(lock: &p->svms.lock);
1091	return -EADDRINUSE;
1092	}
1093
1094	mutex_unlock(lock: &p->svms.lock);
1095	#endif
1096	mutex_lock(&p->mutex);
1097	pdd = kfd_process_device_data_by_id(process: p, gpu_id: args->gpu_id);
1098	if (!pdd) {
1099	err = -EINVAL;
1100	goto err_pdd;
1101	}
1102
1103	dev = pdd->dev;
1104
1105	if ((flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) &&
1106	(flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) &&
1107	!kfd_dev_is_large_bar(dev)) {
1108	pr_err("Alloc host visible vram on small bar is not allowed\n");
1109	err = -EINVAL;
1110	goto err_large_bar;
1111	}
1112
1113	pdd = kfd_bind_process_to_device(dev, p);
1114	if (IS_ERR(ptr: pdd)) {
1115	err = PTR_ERR(ptr: pdd);
1116	goto err_unlock;
1117	}
1118
1119	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
1120	if (args->size != kfd_doorbell_process_slice(kfd: dev->kfd)) {
1121	err = -EINVAL;
1122	goto err_unlock;
1123	}
1124	offset = kfd_get_process_doorbells(pdd);
1125	if (!offset) {
1126	err = -ENOMEM;
1127	goto err_unlock;
1128	}
1129	} else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
1130	if (args->size != PAGE_SIZE) {
1131	err = -EINVAL;
1132	goto err_unlock;
1133	}
1134	offset = dev->adev->rmmio_remap.bus_addr;
1135	if (!offset || (PAGE_SIZE > 4096)) {
1136	err = -ENOMEM;
1137	goto err_unlock;
1138	}
1139	}
1140
1141	err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1142	adev: dev->adev, va: args->va_addr, size: args->size,
1143	drm_priv: pdd->drm_priv, mem: (struct kgd_mem **) &mem, offset: &offset,
1144	flags, criu_resume: false);
1145
1146	if (err)
1147	goto err_unlock;
1148
1149	idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1150	if (idr_handle < 0) {
1151	err = -EFAULT;
1152	goto err_free;
1153	}
1154
1155	/* Update the VRAM usage count */
1156	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
1157	uint64_t size = args->size;
1158
1159	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM)
1160	size >>= 1;
1161	atomic64_add(PAGE_ALIGN(size), v: &pdd->vram_usage);
1162	}
1163
1164	mutex_unlock(lock: &p->mutex);
1165
1166	args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1167	args->mmap_offset = offset;
1168
1169	/* MMIO is mapped through kfd device
1170	* Generate a kfd mmap offset
1171	*/
1172	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)
1173	args->mmap_offset = KFD_MMAP_TYPE_MMIO
1174	| KFD_MMAP_GPU_ID(args->gpu_id);
1175
1176	return 0;
1177
1178	err_free:
1179	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(adev: dev->adev, mem: (struct kgd_mem *)mem,
1180	drm_priv: pdd->drm_priv, NULL);
1181	err_unlock:
1182	err_pdd:
1183	err_large_bar:
1184	mutex_unlock(lock: &p->mutex);
1185	return err;
1186	}
1187
1188	static int kfd_ioctl_free_memory_of_gpu(struct file *filep,
1189	struct kfd_process *p, void *data)
1190	{
1191	struct kfd_ioctl_free_memory_of_gpu_args *args = data;
1192	struct kfd_process_device *pdd;
1193	void *mem;
1194	int ret;
1195	uint64_t size = 0;
1196
1197	mutex_lock(&p->mutex);
1198	/*
1199	* Safeguard to prevent user space from freeing signal BO.
1200	* It will be freed at process termination.
1201	*/
1202	if (p->signal_handle && (p->signal_handle == args->handle)) {
1203	pr_err("Free signal BO is not allowed\n");
1204	ret = -EPERM;
1205	goto err_unlock;
1206	}
1207
1208	pdd = kfd_process_device_data_by_id(process: p, GET_GPU_ID(args->handle));
1209	if (!pdd) {
1210	pr_err("Process device data doesn't exist\n");
1211	ret = -EINVAL;
1212	goto err_pdd;
1213	}
1214
1215	mem = kfd_process_device_translate_handle(
1216	p: pdd, GET_IDR_HANDLE(args->handle));
1217	if (!mem) {
1218	ret = -EINVAL;
1219	goto err_unlock;
1220	}
1221
1222	ret = amdgpu_amdkfd_gpuvm_free_memory_of_gpu(adev: pdd->dev->adev,
1223	mem: (struct kgd_mem *)mem, drm_priv: pdd->drm_priv, size: &size);
1224
1225	/* If freeing the buffer failed, leave the handle in place for
1226	* clean-up during process tear-down.
1227	*/
1228	if (!ret)
1229	kfd_process_device_remove_obj_handle(
1230	pdd, GET_IDR_HANDLE(args->handle));
1231
1232	atomic64_sub(i: size, v: &pdd->vram_usage);
1233
1234	err_unlock:
1235	err_pdd:
1236	mutex_unlock(lock: &p->mutex);
1237	return ret;
1238	}
1239
1240	static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
1241	struct kfd_process *p, void *data)
1242	{
1243	struct kfd_ioctl_map_memory_to_gpu_args *args = data;
1244	struct kfd_process_device *pdd, *peer_pdd;
1245	void *mem;
1246	struct kfd_node *dev;
1247	long err = 0;
1248	int i;
1249	uint32_t *devices_arr = NULL;
1250
1251	if (!args->n_devices) {
1252	pr_debug("Device IDs array empty\n");
1253	return -EINVAL;
1254	}
1255	if (args->n_success > args->n_devices) {
1256	pr_debug("n_success exceeds n_devices\n");
1257	return -EINVAL;
1258	}
1259
1260	devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
1261	GFP_KERNEL);
1262	if (!devices_arr)
1263	return -ENOMEM;
1264
1265	err = copy_from_user(to: devices_arr,
1266	from: (void __user *)args->device_ids_array_ptr,
1267	n: args->n_devices * sizeof(*devices_arr));
1268	if (err != 0) {
1269	err = -EFAULT;
1270	goto copy_from_user_failed;
1271	}
1272
1273	mutex_lock(&p->mutex);
1274	pdd = kfd_process_device_data_by_id(process: p, GET_GPU_ID(args->handle));
1275	if (!pdd) {
1276	err = -EINVAL;
1277	goto get_process_device_data_failed;
1278	}
1279	dev = pdd->dev;
1280
1281	pdd = kfd_bind_process_to_device(dev, p);
1282	if (IS_ERR(ptr: pdd)) {
1283	err = PTR_ERR(ptr: pdd);
1284	goto bind_process_to_device_failed;
1285	}
1286
1287	mem = kfd_process_device_translate_handle(p: pdd,
1288	GET_IDR_HANDLE(args->handle));
1289	if (!mem) {
1290	err = -ENOMEM;
1291	goto get_mem_obj_from_handle_failed;
1292	}
1293
1294	for (i = args->n_success; i < args->n_devices; i++) {
1295	peer_pdd = kfd_process_device_data_by_id(process: p, gpu_id: devices_arr[i]);
1296	if (!peer_pdd) {
1297	pr_debug("Getting device by id failed for 0x%x\n",
1298	devices_arr[i]);
1299	err = -EINVAL;
1300	goto get_mem_obj_from_handle_failed;
1301	}
1302
1303	peer_pdd = kfd_bind_process_to_device(dev: peer_pdd->dev, p);
1304	if (IS_ERR(ptr: peer_pdd)) {
1305	err = PTR_ERR(ptr: peer_pdd);
1306	goto get_mem_obj_from_handle_failed;
1307	}
1308
1309	err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1310	adev: peer_pdd->dev->adev, mem: (struct kgd_mem *)mem,
1311	drm_priv: peer_pdd->drm_priv);
1312	if (err) {
1313	struct pci_dev *pdev = peer_pdd->dev->adev->pdev;
1314
1315	dev_err(dev->adev->dev,
1316	"Failed to map peer:%04x:%02x:%02x.%d mem_domain:%d\n",
1317	pci_domain_nr(pdev->bus),
1318	pdev->bus->number,
1319	PCI_SLOT(pdev->devfn),
1320	PCI_FUNC(pdev->devfn),
1321	((struct kgd_mem *)mem)->domain);
1322	goto map_memory_to_gpu_failed;
1323	}
1324	args->n_success = i+1;
1325	}
1326
1327	err = amdgpu_amdkfd_gpuvm_sync_memory(adev: dev->adev, mem: (struct kgd_mem *) mem, intr: true);
1328	if (err) {
1329	pr_debug("Sync memory failed, wait interrupted by user signal\n");
1330	goto sync_memory_failed;
1331	}
1332
1333	mutex_unlock(lock: &p->mutex);
1334
1335	/* Flush TLBs after waiting for the page table updates to complete */
1336	for (i = 0; i < args->n_devices; i++) {
1337	peer_pdd = kfd_process_device_data_by_id(process: p, gpu_id: devices_arr[i]);
1338	if (WARN_ON_ONCE(!peer_pdd))
1339	continue;
1340	kfd_flush_tlb(pdd: peer_pdd, type: TLB_FLUSH_LEGACY);
1341	}
1342	kfree(objp: devices_arr);
1343
1344	return err;
1345
1346	get_process_device_data_failed:
1347	bind_process_to_device_failed:
1348	get_mem_obj_from_handle_failed:
1349	map_memory_to_gpu_failed:
1350	sync_memory_failed:
1351	mutex_unlock(lock: &p->mutex);
1352	copy_from_user_failed:
1353	kfree(objp: devices_arr);
1354
1355	return err;
1356	}
1357
1358	static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep,
1359	struct kfd_process *p, void *data)
1360	{
1361	struct kfd_ioctl_unmap_memory_from_gpu_args *args = data;
1362	struct kfd_process_device *pdd, *peer_pdd;
1363	void *mem;
1364	long err = 0;
1365	uint32_t *devices_arr = NULL, i;
1366	bool flush_tlb;
1367
1368	if (!args->n_devices) {
1369	pr_debug("Device IDs array empty\n");
1370	return -EINVAL;
1371	}
1372	if (args->n_success > args->n_devices) {
1373	pr_debug("n_success exceeds n_devices\n");
1374	return -EINVAL;
1375	}
1376
1377	devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
1378	GFP_KERNEL);
1379	if (!devices_arr)
1380	return -ENOMEM;
1381
1382	err = copy_from_user(to: devices_arr,
1383	from: (void __user *)args->device_ids_array_ptr,
1384	n: args->n_devices * sizeof(*devices_arr));
1385	if (err != 0) {
1386	err = -EFAULT;
1387	goto copy_from_user_failed;
1388	}
1389
1390	mutex_lock(&p->mutex);
1391	pdd = kfd_process_device_data_by_id(process: p, GET_GPU_ID(args->handle));
1392	if (!pdd) {
1393	err = -EINVAL;
1394	goto bind_process_to_device_failed;
1395	}
1396
1397	mem = kfd_process_device_translate_handle(p: pdd,
1398	GET_IDR_HANDLE(args->handle));
1399	if (!mem) {
1400	err = -ENOMEM;
1401	goto get_mem_obj_from_handle_failed;
1402	}
1403
1404	for (i = args->n_success; i < args->n_devices; i++) {
1405	peer_pdd = kfd_process_device_data_by_id(process: p, gpu_id: devices_arr[i]);
1406	if (!peer_pdd) {
1407	err = -EINVAL;
1408	goto get_mem_obj_from_handle_failed;
1409	}
1410	err = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1411	adev: peer_pdd->dev->adev, mem: (struct kgd_mem *)mem, drm_priv: peer_pdd->drm_priv);
1412	if (err) {
1413	pr_debug("Failed to unmap from gpu %d/%d\n", i, args->n_devices);
1414	goto unmap_memory_from_gpu_failed;
1415	}
1416	args->n_success = i+1;
1417	}
1418
1419	flush_tlb = kfd_flush_tlb_after_unmap(dev: pdd->dev->kfd);
1420	if (flush_tlb) {
1421	err = amdgpu_amdkfd_gpuvm_sync_memory(adev: pdd->dev->adev,
1422	mem: (struct kgd_mem *) mem, intr: true);
1423	if (err) {
1424	pr_debug("Sync memory failed, wait interrupted by user signal\n");
1425	goto sync_memory_failed;
1426	}
1427	}
1428
1429	/* Flush TLBs after waiting for the page table updates to complete */
1430	for (i = 0; i < args->n_devices; i++) {
1431	peer_pdd = kfd_process_device_data_by_id(process: p, gpu_id: devices_arr[i]);
1432	if (WARN_ON_ONCE(!peer_pdd))
1433	continue;
1434	if (flush_tlb)
1435	kfd_flush_tlb(pdd: peer_pdd, type: TLB_FLUSH_HEAVYWEIGHT);
1436
1437	/* Remove dma mapping after tlb flush to avoid IO_PAGE_FAULT */
1438	err = amdgpu_amdkfd_gpuvm_dmaunmap_mem(mem, drm_priv: peer_pdd->drm_priv);
1439	if (err)
1440	goto sync_memory_failed;
1441	}
1442
1443	mutex_unlock(lock: &p->mutex);
1444
1445	kfree(objp: devices_arr);
1446
1447	return 0;
1448
1449	bind_process_to_device_failed:
1450	get_mem_obj_from_handle_failed:
1451	unmap_memory_from_gpu_failed:
1452	sync_memory_failed:
1453	mutex_unlock(lock: &p->mutex);
1454	copy_from_user_failed:
1455	kfree(objp: devices_arr);
1456	return err;
1457	}
1458
1459	static int kfd_ioctl_alloc_queue_gws(struct file *filep,
1460	struct kfd_process *p, void *data)
1461	{
1462	int retval;
1463	struct kfd_ioctl_alloc_queue_gws_args *args = data;
1464	struct queue *q;
1465	struct kfd_node *dev;
1466
1467	mutex_lock(&p->mutex);
1468	q = pqm_get_user_queue(pqm: &p->pqm, qid: args->queue_id);
1469
1470	if (q) {
1471	dev = q->device;
1472	} else {
1473	retval = -EINVAL;
1474	goto out_unlock;
1475	}
1476
1477	if (!dev->gws) {
1478	retval = -ENODEV;
1479	goto out_unlock;
1480	}
1481
1482	if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
1483	retval = -ENODEV;
1484	goto out_unlock;
1485	}
1486
1487	if (p->debug_trap_enabled && (!kfd_dbg_has_gws_support(dev) ||
1488	kfd_dbg_has_cwsr_workaround(dev))) {
1489	retval = -EBUSY;
1490	goto out_unlock;
1491	}
1492
1493	retval = pqm_set_gws(pqm: &p->pqm, qid: args->queue_id, gws: args->num_gws ? dev->gws : NULL);
1494	mutex_unlock(lock: &p->mutex);
1495
1496	args->first_gws = 0;
1497	return retval;
1498
1499	out_unlock:
1500	mutex_unlock(lock: &p->mutex);
1501	return retval;
1502	}
1503
1504	static int kfd_ioctl_get_dmabuf_info(struct file *filep,
1505	struct kfd_process *p, void *data)
1506	{
1507	struct kfd_ioctl_get_dmabuf_info_args *args = data;
1508	struct kfd_node *dev = NULL;
1509	struct amdgpu_device *dmabuf_adev;
1510	void *metadata_buffer = NULL;
1511	uint32_t flags;
1512	int8_t xcp_id;
1513	unsigned int i;
1514	int r;
1515
1516	/* Find a KFD GPU device that supports the get_dmabuf_info query */
1517	for (i = 0; kfd_topology_enum_kfd_devices(idx: i, kdev: &dev) == 0; i++)
1518	if (dev && !kfd_devcgroup_check_permission(node: dev))
1519	break;
1520	if (!dev)
1521	return -EINVAL;
1522
1523	if (args->metadata_ptr) {
1524	metadata_buffer = kzalloc(args->metadata_size, GFP_KERNEL);
1525	if (!metadata_buffer)
1526	return -ENOMEM;
1527	}
1528
1529	/* Get dmabuf info from KGD */
1530	r = amdgpu_amdkfd_get_dmabuf_info(adev: dev->adev, dma_buf_fd: args->dmabuf_fd,
1531	dmabuf_adev: &dmabuf_adev, bo_size: &args->size,
1532	metadata_buffer, buffer_size: args->metadata_size,
1533	metadata_size: &args->metadata_size, flags: &flags, xcp_id: &xcp_id);
1534	if (r)
1535	goto exit;
1536
1537	if (xcp_id >= 0)
1538	args->gpu_id = dmabuf_adev->kfd.dev->nodes[xcp_id]->id;
1539	else
1540	args->gpu_id = dev->id;
1541	args->flags = flags;
1542
1543	/* Copy metadata buffer to user mode */
1544	if (metadata_buffer) {
1545	r = copy_to_user(to: (void __user *)args->metadata_ptr,
1546	from: metadata_buffer, n: args->metadata_size);
1547	if (r != 0)
1548	r = -EFAULT;
1549	}
1550
1551	exit:
1552	kfree(objp: metadata_buffer);
1553
1554	return r;
1555	}
1556
1557	static int kfd_ioctl_import_dmabuf(struct file *filep,
1558	struct kfd_process *p, void *data)
1559	{
1560	struct kfd_ioctl_import_dmabuf_args *args = data;
1561	struct kfd_process_device *pdd;
1562	int idr_handle;
1563	uint64_t size;
1564	void *mem;
1565	int r;
1566
1567	mutex_lock(&p->mutex);
1568	pdd = kfd_process_device_data_by_id(process: p, gpu_id: args->gpu_id);
1569	if (!pdd) {
1570	r = -EINVAL;
1571	goto err_unlock;
1572	}
1573
1574	pdd = kfd_bind_process_to_device(dev: pdd->dev, p);
1575	if (IS_ERR(ptr: pdd)) {
1576	r = PTR_ERR(ptr: pdd);
1577	goto err_unlock;
1578	}
1579
1580	r = amdgpu_amdkfd_gpuvm_import_dmabuf_fd(adev: pdd->dev->adev, fd: args->dmabuf_fd,
1581	va: args->va_addr, drm_priv: pdd->drm_priv,
1582	mem: (struct kgd_mem **)&mem, size: &size,
1583	NULL);
1584	if (r)
1585	goto err_unlock;
1586
1587	idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1588	if (idr_handle < 0) {
1589	r = -EFAULT;
1590	goto err_free;
1591	}
1592
1593	mutex_unlock(lock: &p->mutex);
1594
1595	args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1596
1597	return 0;
1598
1599	err_free:
1600	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(adev: pdd->dev->adev, mem: (struct kgd_mem *)mem,
1601	drm_priv: pdd->drm_priv, NULL);
1602	err_unlock:
1603	mutex_unlock(lock: &p->mutex);
1604	return r;
1605	}
1606
1607	static int kfd_ioctl_export_dmabuf(struct file *filep,
1608	struct kfd_process *p, void *data)
1609	{
1610	struct kfd_ioctl_export_dmabuf_args *args = data;
1611	struct kfd_process_device *pdd;
1612	struct dma_buf *dmabuf;
1613	struct kfd_node *dev;
1614	void *mem;
1615	int ret = 0;
1616
1617	dev = kfd_device_by_id(GET_GPU_ID(args->handle));
1618	if (!dev)
1619	return -EINVAL;
1620
1621	mutex_lock(&p->mutex);
1622
1623	pdd = kfd_get_process_device_data(dev, p);
1624	if (!pdd) {
1625	ret = -EINVAL;
1626	goto err_unlock;
1627	}
1628
1629	mem = kfd_process_device_translate_handle(p: pdd,
1630	GET_IDR_HANDLE(args->handle));
1631	if (!mem) {
1632	ret = -EINVAL;
1633	goto err_unlock;
1634	}
1635
1636	ret = amdgpu_amdkfd_gpuvm_export_dmabuf(mem, dmabuf: &dmabuf);
1637	mutex_unlock(lock: &p->mutex);
1638	if (ret)
1639	goto err_out;
1640
1641	ret = dma_buf_fd(dmabuf, flags: args->flags);
1642	if (ret < 0) {
1643	dma_buf_put(dmabuf);
1644	goto err_out;
1645	}
1646	/* dma_buf_fd assigns the reference count to the fd, no need to
1647	* put the reference here.
1648	*/
1649	args->dmabuf_fd = ret;
1650
1651	return 0;
1652
1653	err_unlock:
1654	mutex_unlock(lock: &p->mutex);
1655	err_out:
1656	return ret;
1657	}
1658
1659	/* Handle requests for watching SMI events */
1660	static int kfd_ioctl_smi_events(struct file *filep,
1661	struct kfd_process *p, void *data)
1662	{
1663	struct kfd_ioctl_smi_events_args *args = data;
1664	struct kfd_process_device *pdd;
1665
1666	mutex_lock(&p->mutex);
1667
1668	pdd = kfd_process_device_data_by_id(process: p, gpu_id: args->gpuid);
1669	mutex_unlock(lock: &p->mutex);
1670	if (!pdd)
1671	return -EINVAL;
1672
1673	return kfd_smi_event_open(dev: pdd->dev, fd: &args->anon_fd);
1674	}
1675
1676	#if IS_ENABLED(CONFIG_HSA_AMD_SVM)
1677
1678	static int kfd_ioctl_set_xnack_mode(struct file *filep,
1679	struct kfd_process *p, void *data)
1680	{
1681	struct kfd_ioctl_set_xnack_mode_args *args = data;
1682	int r = 0;
1683
1684	mutex_lock(&p->mutex);
1685	if (args->xnack_enabled >= 0) {
1686	if (!list_empty(head: &p->pqm.queues)) {
1687	pr_debug("Process has user queues running\n");
1688	r = -EBUSY;
1689	goto out_unlock;
1690	}
1691
1692	if (p->xnack_enabled == args->xnack_enabled)
1693	goto out_unlock;
1694
1695	if (args->xnack_enabled && !kfd_process_xnack_mode(p, supported: true)) {
1696	r = -EPERM;
1697	goto out_unlock;
1698	}
1699
1700	r = svm_range_switch_xnack_reserve_mem(p, xnack_enabled: args->xnack_enabled);
1701	} else {
1702	args->xnack_enabled = p->xnack_enabled;
1703	}
1704
1705	out_unlock:
1706	mutex_unlock(lock: &p->mutex);
1707
1708	return r;
1709	}
1710
1711	static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
1712	{
1713	struct kfd_ioctl_svm_args *args = data;
1714	int r = 0;
1715
1716	pr_debug("start 0x%llx size 0x%llx op 0x%x nattr 0x%x\n",
1717	args->start_addr, args->size, args->op, args->nattr);
1718
1719	if ((args->start_addr & ~PAGE_MASK) || (args->size & ~PAGE_MASK))
1720	return -EINVAL;
1721	if (!args->start_addr || !args->size)
1722	return -EINVAL;
1723
1724	r = svm_ioctl(p, op: args->op, start: args->start_addr, size: args->size, nattrs: args->nattr,
1725	attrs: args->attrs);
1726
1727	return r;
1728	}
1729	#else
1730	static int kfd_ioctl_set_xnack_mode(struct file *filep,
1731	struct kfd_process *p, void *data)
1732	{
1733	return -EPERM;
1734	}
1735	static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
1736	{
1737	return -EPERM;
1738	}
1739	#endif
1740
1741	static int criu_checkpoint_process(struct kfd_process *p,
1742	uint8_t __user *user_priv_data,
1743	uint64_t *priv_offset)
1744	{
1745	struct kfd_criu_process_priv_data process_priv;
1746	int ret;
1747
1748	memset(&process_priv, 0, sizeof(process_priv));
1749
1750	process_priv.version = KFD_CRIU_PRIV_VERSION;
1751	/* For CR, we don't consider negative xnack mode which is used for
1752	* querying without changing it, here 0 simply means disabled and 1
1753	* means enabled so retry for finding a valid PTE.
1754	*/
1755	process_priv.xnack_mode = p->xnack_enabled ? 1 : 0;
1756
1757	ret = copy_to_user(to: user_priv_data + *priv_offset,
1758	from: &process_priv, n: sizeof(process_priv));
1759
1760	if (ret) {
1761	pr_err("Failed to copy process information to user\n");
1762	ret = -EFAULT;
1763	}
1764
1765	*priv_offset += sizeof(process_priv);
1766	return ret;
1767	}
1768
1769	static int criu_checkpoint_devices(struct kfd_process *p,
1770	uint32_t num_devices,
1771	uint8_t __user *user_addr,
1772	uint8_t __user *user_priv_data,
1773	uint64_t *priv_offset)
1774	{
1775	struct kfd_criu_device_priv_data *device_priv = NULL;
1776	struct kfd_criu_device_bucket *device_buckets = NULL;
1777	int ret = 0, i;
1778
1779	device_buckets = kvzalloc(num_devices * sizeof(*device_buckets), GFP_KERNEL);
1780	if (!device_buckets) {
1781	ret = -ENOMEM;
1782	goto exit;
1783	}
1784
1785	device_priv = kvzalloc(num_devices * sizeof(*device_priv), GFP_KERNEL);
1786	if (!device_priv) {
1787	ret = -ENOMEM;
1788	goto exit;
1789	}
1790
1791	for (i = 0; i < num_devices; i++) {
1792	struct kfd_process_device *pdd = p->pdds[i];
1793
1794	device_buckets[i].user_gpu_id = pdd->user_gpu_id;
1795	device_buckets[i].actual_gpu_id = pdd->dev->id;
1796
1797	/*
1798	* priv_data does not contain useful information for now and is reserved for
1799	* future use, so we do not set its contents.
1800	*/
1801	}
1802
1803	ret = copy_to_user(to: user_addr, from: device_buckets, n: num_devices * sizeof(*device_buckets));
1804	if (ret) {
1805	pr_err("Failed to copy device information to user\n");
1806	ret = -EFAULT;
1807	goto exit;
1808	}
1809
1810	ret = copy_to_user(to: user_priv_data + *priv_offset,
1811	from: device_priv,
1812	n: num_devices * sizeof(*device_priv));
1813	if (ret) {
1814	pr_err("Failed to copy device information to user\n");
1815	ret = -EFAULT;
1816	}
1817	*priv_offset += num_devices * sizeof(*device_priv);
1818
1819	exit:
1820	kvfree(addr: device_buckets);
1821	kvfree(addr: device_priv);
1822	return ret;
1823	}
1824
1825	static uint32_t get_process_num_bos(struct kfd_process *p)
1826	{
1827	uint32_t num_of_bos = 0;
1828	int i;
1829
1830	/* Run over all PDDs of the process */
1831	for (i = 0; i < p->n_pdds; i++) {
1832	struct kfd_process_device *pdd = p->pdds[i];
1833	void *mem;
1834	int id;
1835
1836	idr_for_each_entry(&pdd->alloc_idr, mem, id) {
1837	struct kgd_mem *kgd_mem = (struct kgd_mem *)mem;
1838
1839	if (!kgd_mem->va || kgd_mem->va > pdd->gpuvm_base)
1840	num_of_bos++;
1841	}
1842	}
1843	return num_of_bos;
1844	}
1845
1846	static int criu_get_prime_handle(struct kgd_mem *mem,
1847	int flags, u32 *shared_fd,
1848	struct file **file)
1849	{
1850	struct dma_buf *dmabuf;
1851	int ret;
1852
1853	ret = amdgpu_amdkfd_gpuvm_export_dmabuf(mem, dmabuf: &dmabuf);
1854	if (ret) {
1855	pr_err("dmabuf export failed for the BO\n");
1856	return ret;
1857	}
1858
1859	ret = get_unused_fd_flags(flags);
1860	if (ret < 0) {
1861	pr_err("dmabuf create fd failed, ret:%d\n", ret);
1862	goto out_free_dmabuf;
1863	}
1864
1865	*shared_fd = ret;
1866	*file = dmabuf->file;
1867	return 0;
1868
1869	out_free_dmabuf:
1870	dma_buf_put(dmabuf);
1871	return ret;
1872	}
1873
1874	static void commit_files(struct file **files,
1875	struct kfd_criu_bo_bucket *bo_buckets,
1876	unsigned int count,
1877	int err)
1878	{
1879	while (count--) {
1880	struct file *file = files[count];
1881
1882	if (!file)
1883	continue;
1884	if (err) {
1885	fput(file);
1886	put_unused_fd(fd: bo_buckets[count].dmabuf_fd);
1887	} else {
1888	fd_install(fd: bo_buckets[count].dmabuf_fd, file);
1889	}
1890	}
1891	}
1892
1893	static int criu_checkpoint_bos(struct kfd_process *p,
1894	uint32_t num_bos,
1895	uint8_t __user *user_bos,
1896	uint8_t __user *user_priv_data,
1897	uint64_t *priv_offset)
1898	{
1899	struct kfd_criu_bo_bucket *bo_buckets;
1900	struct kfd_criu_bo_priv_data *bo_privs;
1901	struct file **files = NULL;
1902	int ret = 0, pdd_index, bo_index = 0, id;
1903	void *mem;
1904
1905	bo_buckets = kvzalloc(num_bos * sizeof(*bo_buckets), GFP_KERNEL);
1906	if (!bo_buckets)
1907	return -ENOMEM;
1908
1909	bo_privs = kvzalloc(num_bos * sizeof(*bo_privs), GFP_KERNEL);
1910	if (!bo_privs) {
1911	ret = -ENOMEM;
1912	goto exit;
1913	}
1914
1915	files = kvzalloc(num_bos * sizeof(struct file *), GFP_KERNEL);
1916	if (!files) {
1917	ret = -ENOMEM;
1918	goto exit;
1919	}
1920
1921	for (pdd_index = 0; pdd_index < p->n_pdds; pdd_index++) {
1922	struct kfd_process_device *pdd = p->pdds[pdd_index];
1923	struct amdgpu_bo *dumper_bo;
1924	struct kgd_mem *kgd_mem;
1925
1926	idr_for_each_entry(&pdd->alloc_idr, mem, id) {
1927	struct kfd_criu_bo_bucket *bo_bucket;
1928	struct kfd_criu_bo_priv_data *bo_priv;
1929	int i, dev_idx = 0;
1930
1931	kgd_mem = (struct kgd_mem *)mem;
1932	dumper_bo = kgd_mem->bo;
1933
1934	/* Skip checkpointing BOs that are used for Trap handler
1935	* code and state. Currently, these BOs have a VA that
1936	* is less GPUVM Base
1937	*/
1938	if (kgd_mem->va && kgd_mem->va <= pdd->gpuvm_base)
1939	continue;
1940
1941	bo_bucket = &bo_buckets[bo_index];
1942	bo_priv = &bo_privs[bo_index];
1943
1944	bo_bucket->gpu_id = pdd->user_gpu_id;
1945	bo_bucket->addr = (uint64_t)kgd_mem->va;
1946	bo_bucket->size = amdgpu_bo_size(bo: dumper_bo);
1947	bo_bucket->alloc_flags = (uint32_t)kgd_mem->alloc_flags;
1948	bo_priv->idr_handle = id;
1949
1950	if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
1951	ret = amdgpu_ttm_tt_get_userptr(tbo: &dumper_bo->tbo,
1952	user_addr: &bo_priv->user_addr);
1953	if (ret) {
1954	pr_err("Failed to obtain user address for user-pointer bo\n");
1955	goto exit;
1956	}
1957	}
1958	if (bo_bucket->alloc_flags
1959	& (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) {
1960	ret = criu_get_prime_handle(mem: kgd_mem,
1961	flags: bo_bucket->alloc_flags &
1962	KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? DRM_RDWR : 0,
1963	shared_fd: &bo_bucket->dmabuf_fd, file: &files[bo_index]);
1964	if (ret)
1965	goto exit;
1966	} else {
1967	bo_bucket->dmabuf_fd = KFD_INVALID_FD;
1968	}
1969
1970	if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL)
1971	bo_bucket->offset = KFD_MMAP_TYPE_DOORBELL |
1972	KFD_MMAP_GPU_ID(pdd->dev->id);
1973	else if (bo_bucket->alloc_flags &
1974	KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)
1975	bo_bucket->offset = KFD_MMAP_TYPE_MMIO |
1976	KFD_MMAP_GPU_ID(pdd->dev->id);
1977	else
1978	bo_bucket->offset = amdgpu_bo_mmap_offset(bo: dumper_bo);
1979
1980	for (i = 0; i < p->n_pdds; i++) {
1981	if (amdgpu_amdkfd_bo_mapped_to_dev(drm_priv: p->pdds[i]->drm_priv, mem: kgd_mem))
1982	bo_priv->mapped_gpuids[dev_idx++] = p->pdds[i]->user_gpu_id;
1983	}
1984
1985	pr_debug("bo_size = 0x%llx, bo_addr = 0x%llx bo_offset = 0x%llx\n"
1986	"gpu_id = 0x%x alloc_flags = 0x%x idr_handle = 0x%x",
1987	bo_bucket->size,
1988	bo_bucket->addr,
1989	bo_bucket->offset,
1990	bo_bucket->gpu_id,
1991	bo_bucket->alloc_flags,
1992	bo_priv->idr_handle);
1993	bo_index++;
1994	}
1995	}
1996
1997	ret = copy_to_user(to: user_bos, from: bo_buckets, n: num_bos * sizeof(*bo_buckets));
1998	if (ret) {
1999	pr_err("Failed to copy BO information to user\n");
2000	ret = -EFAULT;
2001	goto exit;
2002	}
2003
2004	ret = copy_to_user(to: user_priv_data + *priv_offset, from: bo_privs, n: num_bos * sizeof(*bo_privs));
2005	if (ret) {
2006	pr_err("Failed to copy BO priv information to user\n");
2007	ret = -EFAULT;
2008	goto exit;
2009	}
2010
2011	*priv_offset += num_bos * sizeof(*bo_privs);
2012
2013	exit:
2014	commit_files(files, bo_buckets, count: bo_index, err: ret);
2015	kvfree(addr: files);
2016	kvfree(addr: bo_buckets);
2017	kvfree(addr: bo_privs);
2018	return ret;
2019	}
2020
2021	static int criu_get_process_object_info(struct kfd_process *p,
2022	uint32_t *num_devices,
2023	uint32_t *num_bos,
2024	uint32_t *num_objects,
2025	uint64_t *objs_priv_size)
2026	{
2027	uint64_t queues_priv_data_size, svm_priv_data_size, priv_size;
2028	uint32_t num_queues, num_events, num_svm_ranges;
2029	int ret;
2030
2031	*num_devices = p->n_pdds;
2032	*num_bos = get_process_num_bos(p);
2033
2034	ret = kfd_process_get_queue_info(p, num_queues: &num_queues, priv_data_sizes: &queues_priv_data_size);
2035	if (ret)
2036	return ret;
2037
2038	num_events = kfd_get_num_events(p);
2039
2040	svm_range_get_info(p, num_svm_ranges: &num_svm_ranges, svm_priv_data_size: &svm_priv_data_size);
2041
2042	*num_objects = num_queues + num_events + num_svm_ranges;
2043
2044	if (objs_priv_size) {
2045	priv_size = sizeof(struct kfd_criu_process_priv_data);
2046	priv_size += *num_devices * sizeof(struct kfd_criu_device_priv_data);
2047	priv_size += *num_bos * sizeof(struct kfd_criu_bo_priv_data);
2048	priv_size += queues_priv_data_size;
2049	priv_size += num_events * sizeof(struct kfd_criu_event_priv_data);
2050	priv_size += svm_priv_data_size;
2051	*objs_priv_size = priv_size;
2052	}
2053	return 0;
2054	}
2055
2056	static int criu_checkpoint(struct file *filep,
2057	struct kfd_process *p,
2058	struct kfd_ioctl_criu_args *args)
2059	{
2060	int ret;
2061	uint32_t num_devices, num_bos, num_objects;
2062	uint64_t priv_size, priv_offset = 0, bo_priv_offset;
2063
2064	if (!args->devices || !args->bos || !args->priv_data)
2065	return -EINVAL;
2066
2067	mutex_lock(&p->mutex);
2068
2069	if (!p->n_pdds) {
2070	pr_err("No pdd for given process\n");
2071	ret = -ENODEV;
2072	goto exit_unlock;
2073	}
2074
2075	/* Confirm all process queues are evicted */
2076	if (!p->queues_paused) {
2077	pr_err("Cannot dump process when queues are not in evicted state\n");
2078	/* CRIU plugin did not call op PROCESS_INFO before checkpointing */
2079	ret = -EINVAL;
2080	goto exit_unlock;
2081	}
2082
2083	ret = criu_get_process_object_info(p, num_devices: &num_devices, num_bos: &num_bos, num_objects: &num_objects, objs_priv_size: &priv_size);
2084	if (ret)
2085	goto exit_unlock;
2086
2087	if (num_devices != args->num_devices ||
2088	num_bos != args->num_bos ||
2089	num_objects != args->num_objects ||
2090	priv_size != args->priv_data_size) {
2091
2092	ret = -EINVAL;
2093	goto exit_unlock;
2094	}
2095
2096	/* each function will store private data inside priv_data and adjust priv_offset */
2097	ret = criu_checkpoint_process(p, user_priv_data: (uint8_t __user *)args->priv_data, priv_offset: &priv_offset);
2098	if (ret)
2099	goto exit_unlock;
2100
2101	ret = criu_checkpoint_devices(p, num_devices, user_addr: (uint8_t __user *)args->devices,
2102	user_priv_data: (uint8_t __user *)args->priv_data, priv_offset: &priv_offset);
2103	if (ret)
2104	goto exit_unlock;
2105
2106	/* Leave room for BOs in the private data. They need to be restored
2107	* before events, but we checkpoint them last to simplify the error
2108	* handling.
2109	*/
2110	bo_priv_offset = priv_offset;
2111	priv_offset += num_bos * sizeof(struct kfd_criu_bo_priv_data);
2112
2113	if (num_objects) {
2114	ret = kfd_criu_checkpoint_queues(p, user_priv_data: (uint8_t __user *)args->priv_data,
2115	priv_data_offset: &priv_offset);
2116	if (ret)
2117	goto exit_unlock;
2118
2119	ret = kfd_criu_checkpoint_events(p, user_priv_data: (uint8_t __user *)args->priv_data,
2120	priv_data_offset: &priv_offset);
2121	if (ret)
2122	goto exit_unlock;
2123
2124	ret = kfd_criu_checkpoint_svm(p, user_priv_data: (uint8_t __user *)args->priv_data, priv_offset: &priv_offset);
2125	if (ret)
2126	goto exit_unlock;
2127	}
2128
2129	/* This must be the last thing in this function that can fail.
2130	* Otherwise we leak dmabuf file descriptors.
2131	*/
2132	ret = criu_checkpoint_bos(p, num_bos, user_bos: (uint8_t __user *)args->bos,
2133	user_priv_data: (uint8_t __user *)args->priv_data, priv_offset: &bo_priv_offset);
2134
2135	exit_unlock:
2136	mutex_unlock(lock: &p->mutex);
2137	if (ret)
2138	pr_err("Failed to dump CRIU ret:%d\n", ret);
2139	else
2140	pr_debug("CRIU dump ret:%d\n", ret);
2141
2142	return ret;
2143	}
2144
2145	static int criu_restore_process(struct kfd_process *p,
2146	struct kfd_ioctl_criu_args *args,
2147	uint64_t *priv_offset,
2148	uint64_t max_priv_data_size)
2149	{
2150	int ret = 0;
2151	struct kfd_criu_process_priv_data process_priv;
2152
2153	if (*priv_offset + sizeof(process_priv) > max_priv_data_size)
2154	return -EINVAL;
2155
2156	ret = copy_from_user(to: &process_priv,
2157	from: (void __user *)(args->priv_data + *priv_offset),
2158	n: sizeof(process_priv));
2159	if (ret) {
2160	pr_err("Failed to copy process private information from user\n");
2161	ret = -EFAULT;
2162	goto exit;
2163	}
2164	*priv_offset += sizeof(process_priv);
2165
2166	if (process_priv.version != KFD_CRIU_PRIV_VERSION) {
2167	pr_err("Invalid CRIU API version (checkpointed:%d current:%d)\n",
2168	process_priv.version, KFD_CRIU_PRIV_VERSION);
2169	return -EINVAL;
2170	}
2171
2172	pr_debug("Setting XNACK mode\n");
2173	if (process_priv.xnack_mode && !kfd_process_xnack_mode(p, supported: true)) {
2174	pr_err("xnack mode cannot be set\n");
2175	ret = -EPERM;
2176	goto exit;
2177	} else {
2178	pr_debug("set xnack mode: %d\n", process_priv.xnack_mode);
2179	p->xnack_enabled = process_priv.xnack_mode;
2180	}
2181
2182	exit:
2183	return ret;
2184	}
2185
2186	static int criu_restore_devices(struct kfd_process *p,
2187	struct kfd_ioctl_criu_args *args,
2188	uint64_t *priv_offset,
2189	uint64_t max_priv_data_size)
2190	{
2191	struct kfd_criu_device_bucket *device_buckets;
2192	struct kfd_criu_device_priv_data *device_privs;
2193	int ret = 0;
2194	uint32_t i;
2195
2196	if (args->num_devices != p->n_pdds)
2197	return -EINVAL;
2198
2199	if (*priv_offset + (args->num_devices * sizeof(*device_privs)) > max_priv_data_size)
2200	return -EINVAL;
2201
2202	device_buckets = kmalloc_array(args->num_devices, sizeof(*device_buckets), GFP_KERNEL);
2203	if (!device_buckets)
2204	return -ENOMEM;
2205
2206	ret = copy_from_user(to: device_buckets, from: (void __user *)args->devices,
2207	n: args->num_devices * sizeof(*device_buckets));
2208	if (ret) {
2209	pr_err("Failed to copy devices buckets from user\n");
2210	ret = -EFAULT;
2211	goto exit;
2212	}
2213
2214	for (i = 0; i < args->num_devices; i++) {
2215	struct kfd_node *dev;
2216	struct kfd_process_device *pdd;
2217	struct file *drm_file;
2218
2219	/* device private data is not currently used */
2220
2221	if (!device_buckets[i].user_gpu_id) {
2222	pr_err("Invalid user gpu_id\n");
2223	ret = -EINVAL;
2224	goto exit;
2225	}
2226
2227	dev = kfd_device_by_id(gpu_id: device_buckets[i].actual_gpu_id);
2228	if (!dev) {
2229	pr_err("Failed to find device with gpu_id = %x\n",
2230	device_buckets[i].actual_gpu_id);
2231	ret = -EINVAL;
2232	goto exit;
2233	}
2234
2235	pdd = kfd_get_process_device_data(dev, p);
2236	if (!pdd) {
2237	pr_err("Failed to get pdd for gpu_id = %x\n",
2238	device_buckets[i].actual_gpu_id);
2239	ret = -EINVAL;
2240	goto exit;
2241	}
2242	pdd->user_gpu_id = device_buckets[i].user_gpu_id;
2243
2244	drm_file = fget(fd: device_buckets[i].drm_fd);
2245	if (!drm_file) {
2246	pr_err("Invalid render node file descriptor sent from plugin (%d)\n",
2247	device_buckets[i].drm_fd);
2248	ret = -EINVAL;
2249	goto exit;
2250	}
2251
2252	if (pdd->drm_file) {
2253	ret = -EINVAL;
2254	goto exit;
2255	}
2256
2257	/* create the vm using render nodes for kfd pdd */
2258	if (kfd_process_device_init_vm(pdd, drm_file)) {
2259	pr_err("could not init vm for given pdd\n");
2260	/* On success, the PDD keeps the drm_file reference */
2261	fput(drm_file);
2262	ret = -EINVAL;
2263	goto exit;
2264	}
2265	/*
2266	* pdd now already has the vm bound to render node so below api won't create a new
2267	* exclusive kfd mapping but use existing one with renderDXXX but is still needed
2268	* for iommu v2 binding and runtime pm.
2269	*/
2270	pdd = kfd_bind_process_to_device(dev, p);
2271	if (IS_ERR(ptr: pdd)) {
2272	ret = PTR_ERR(ptr: pdd);
2273	goto exit;
2274	}
2275
2276	if (!pdd->qpd.proc_doorbells) {
2277	ret = kfd_alloc_process_doorbells(kfd: dev->kfd, pdd);
2278	if (ret)
2279	goto exit;
2280	}
2281	}
2282
2283	/*
2284	* We are not copying device private data from user as we are not using the data for now,
2285	* but we still adjust for its private data.
2286	*/
2287	*priv_offset += args->num_devices * sizeof(*device_privs);
2288
2289	exit:
2290	kfree(objp: device_buckets);
2291	return ret;
2292	}
2293
2294	static int criu_restore_memory_of_gpu(struct kfd_process_device *pdd,
2295	struct kfd_criu_bo_bucket *bo_bucket,
2296	struct kfd_criu_bo_priv_data *bo_priv,
2297	struct kgd_mem **kgd_mem)
2298	{
2299	int idr_handle;
2300	int ret;
2301	const bool criu_resume = true;
2302	u64 offset;
2303
2304	if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
2305	if (bo_bucket->size !=
2306	kfd_doorbell_process_slice(kfd: pdd->dev->kfd))
2307	return -EINVAL;
2308
2309	offset = kfd_get_process_doorbells(pdd);
2310	if (!offset)
2311	return -ENOMEM;
2312	} else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
2313	/* MMIO BOs need remapped bus address */
2314	if (bo_bucket->size != PAGE_SIZE) {
2315	pr_err("Invalid page size\n");
2316	return -EINVAL;
2317	}
2318	offset = pdd->dev->adev->rmmio_remap.bus_addr;
2319	if (!offset || (PAGE_SIZE > 4096)) {
2320	pr_err("amdgpu_amdkfd_get_mmio_remap_phys_addr failed\n");
2321	return -ENOMEM;
2322	}
2323	} else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
2324	offset = bo_priv->user_addr;
2325	}
2326	/* Create the BO */
2327	ret = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(adev: pdd->dev->adev, va: bo_bucket->addr,
2328	size: bo_bucket->size, drm_priv: pdd->drm_priv, mem: kgd_mem,
2329	offset: &offset, flags: bo_bucket->alloc_flags, criu_resume);
2330	if (ret) {
2331	pr_err("Could not create the BO\n");
2332	return ret;
2333	}
2334	pr_debug("New BO created: size:0x%llx addr:0x%llx offset:0x%llx\n",
2335	bo_bucket->size, bo_bucket->addr, offset);
2336
2337	/* Restore previous IDR handle */
2338	pr_debug("Restoring old IDR handle for the BO");
2339	idr_handle = idr_alloc(&pdd->alloc_idr, ptr: *kgd_mem, start: bo_priv->idr_handle,
2340	end: bo_priv->idr_handle + 1, GFP_KERNEL);
2341
2342	if (idr_handle < 0) {
2343	pr_err("Could not allocate idr\n");
2344	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(adev: pdd->dev->adev, mem: *kgd_mem, drm_priv: pdd->drm_priv,
2345	NULL);
2346	return -ENOMEM;
2347	}
2348
2349	if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL)
2350	bo_bucket->restored_offset = KFD_MMAP_TYPE_DOORBELL | KFD_MMAP_GPU_ID(pdd->dev->id);
2351	if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
2352	bo_bucket->restored_offset = KFD_MMAP_TYPE_MMIO | KFD_MMAP_GPU_ID(pdd->dev->id);
2353	} else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
2354	bo_bucket->restored_offset = offset;
2355	} else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
2356	bo_bucket->restored_offset = offset;
2357	/* Update the VRAM usage count */
2358	atomic64_add(i: bo_bucket->size, v: &pdd->vram_usage);
2359	}
2360	return 0;
2361	}
2362
2363	static int criu_restore_bo(struct kfd_process *p,
2364	struct kfd_criu_bo_bucket *bo_bucket,
2365	struct kfd_criu_bo_priv_data *bo_priv,
2366	struct file **file)
2367	{
2368	struct kfd_process_device *pdd;
2369	struct kgd_mem *kgd_mem;
2370	int ret;
2371	int j;
2372
2373	pr_debug("Restoring BO size:0x%llx addr:0x%llx gpu_id:0x%x flags:0x%x idr_handle:0x%x\n",
2374	bo_bucket->size, bo_bucket->addr, bo_bucket->gpu_id, bo_bucket->alloc_flags,
2375	bo_priv->idr_handle);
2376
2377	pdd = kfd_process_device_data_by_id(process: p, gpu_id: bo_bucket->gpu_id);
2378	if (!pdd) {
2379	pr_err("Failed to get pdd\n");
2380	return -ENODEV;
2381	}
2382
2383	ret = criu_restore_memory_of_gpu(pdd, bo_bucket, bo_priv, kgd_mem: &kgd_mem);
2384	if (ret)
2385	return ret;
2386
2387	/* now map these BOs to GPU/s */
2388	for (j = 0; j < p->n_pdds; j++) {
2389	struct kfd_node *peer;
2390	struct kfd_process_device *peer_pdd;
2391
2392	if (!bo_priv->mapped_gpuids[j])
2393	break;
2394
2395	peer_pdd = kfd_process_device_data_by_id(process: p, gpu_id: bo_priv->mapped_gpuids[j]);
2396	if (!peer_pdd)
2397	return -EINVAL;
2398
2399	peer = peer_pdd->dev;
2400
2401	peer_pdd = kfd_bind_process_to_device(dev: peer, p);
2402	if (IS_ERR(ptr: peer_pdd))
2403	return PTR_ERR(ptr: peer_pdd);
2404
2405	ret = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(adev: peer->adev, mem: kgd_mem,
2406	drm_priv: peer_pdd->drm_priv);
2407	if (ret) {
2408	pr_err("Failed to map to gpu %d/%d\n", j, p->n_pdds);
2409	return ret;
2410	}
2411	}
2412
2413	pr_debug("map memory was successful for the BO\n");
2414	/* create the dmabuf object and export the bo */
2415	if (bo_bucket->alloc_flags
2416	& (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) {
2417	ret = criu_get_prime_handle(mem: kgd_mem, DRM_RDWR,
2418	shared_fd: &bo_bucket->dmabuf_fd, file);
2419	if (ret)
2420	return ret;
2421	} else {
2422	bo_bucket->dmabuf_fd = KFD_INVALID_FD;
2423	}
2424
2425	return 0;
2426	}
2427
2428	static int criu_restore_bos(struct kfd_process *p,
2429	struct kfd_ioctl_criu_args *args,
2430	uint64_t *priv_offset,
2431	uint64_t max_priv_data_size)
2432	{
2433	struct kfd_criu_bo_bucket *bo_buckets = NULL;
2434	struct kfd_criu_bo_priv_data *bo_privs = NULL;
2435	struct file **files = NULL;
2436	int ret = 0;
2437	uint32_t i = 0;
2438
2439	if (*priv_offset + (args->num_bos * sizeof(*bo_privs)) > max_priv_data_size)
2440	return -EINVAL;
2441
2442	/* Prevent MMU notifications until stage-4 IOCTL (CRIU_RESUME) is received */
2443	amdgpu_amdkfd_block_mmu_notifications(p: p->kgd_process_info);
2444
2445	bo_buckets = kvmalloc_array(args->num_bos, sizeof(*bo_buckets), GFP_KERNEL);
2446	if (!bo_buckets)
2447	return -ENOMEM;
2448
2449	files = kvzalloc(args->num_bos * sizeof(struct file *), GFP_KERNEL);
2450	if (!files) {
2451	ret = -ENOMEM;
2452	goto exit;
2453	}
2454
2455	ret = copy_from_user(to: bo_buckets, from: (void __user *)args->bos,
2456	n: args->num_bos * sizeof(*bo_buckets));
2457	if (ret) {
2458	pr_err("Failed to copy BOs information from user\n");
2459	ret = -EFAULT;
2460	goto exit;
2461	}
2462
2463	bo_privs = kvmalloc_array(args->num_bos, sizeof(*bo_privs), GFP_KERNEL);
2464	if (!bo_privs) {
2465	ret = -ENOMEM;
2466	goto exit;
2467	}
2468
2469	ret = copy_from_user(to: bo_privs, from: (void __user *)args->priv_data + *priv_offset,
2470	n: args->num_bos * sizeof(*bo_privs));
2471	if (ret) {
2472	pr_err("Failed to copy BOs information from user\n");
2473	ret = -EFAULT;
2474	goto exit;
2475	}
2476	*priv_offset += args->num_bos * sizeof(*bo_privs);
2477
2478	/* Create and map new BOs */
2479	for (; i < args->num_bos; i++) {
2480	ret = criu_restore_bo(p, bo_bucket: &bo_buckets[i], bo_priv: &bo_privs[i], file: &files[i]);
2481	if (ret) {
2482	pr_debug("Failed to restore BO[%d] ret%d\n", i, ret);
2483	goto exit;
2484	}
2485	} /* done */
2486
2487	/* Copy only the buckets back so user can read bo_buckets[N].restored_offset */
2488	ret = copy_to_user(to: (void __user *)args->bos,
2489	from: bo_buckets,
2490	n: (args->num_bos * sizeof(*bo_buckets)));
2491	if (ret)
2492	ret = -EFAULT;
2493
2494	exit:
2495	commit_files(files, bo_buckets, count: i, err: ret);
2496	kvfree(addr: files);
2497	kvfree(addr: bo_buckets);
2498	kvfree(addr: bo_privs);
2499	return ret;
2500	}
2501
2502	static int criu_restore_objects(struct file *filep,
2503	struct kfd_process *p,
2504	struct kfd_ioctl_criu_args *args,
2505	uint64_t *priv_offset,
2506	uint64_t max_priv_data_size)
2507	{
2508	int ret = 0;
2509	uint32_t i;
2510
2511	BUILD_BUG_ON(offsetof(struct kfd_criu_queue_priv_data, object_type));
2512	BUILD_BUG_ON(offsetof(struct kfd_criu_event_priv_data, object_type));
2513	BUILD_BUG_ON(offsetof(struct kfd_criu_svm_range_priv_data, object_type));
2514
2515	for (i = 0; i < args->num_objects; i++) {
2516	uint32_t object_type;
2517
2518	if (*priv_offset + sizeof(object_type) > max_priv_data_size) {
2519	pr_err("Invalid private data size\n");
2520	return -EINVAL;
2521	}
2522
2523	ret = get_user(object_type, (uint32_t __user *)(args->priv_data + *priv_offset));
2524	if (ret) {
2525	pr_err("Failed to copy private information from user\n");
2526	goto exit;
2527	}
2528
2529	switch (object_type) {
2530	case KFD_CRIU_OBJECT_TYPE_QUEUE:
2531	ret = kfd_criu_restore_queue(p, user_priv_data: (uint8_t __user *)args->priv_data,
2532	priv_data_offset: priv_offset, max_priv_data_size);
2533	if (ret)
2534	goto exit;
2535	break;
2536	case KFD_CRIU_OBJECT_TYPE_EVENT:
2537	ret = kfd_criu_restore_event(devkfd: filep, p, user_priv_data: (uint8_t __user *)args->priv_data,
2538	priv_data_offset: priv_offset, max_priv_data_size);
2539	if (ret)
2540	goto exit;
2541	break;
2542	case KFD_CRIU_OBJECT_TYPE_SVM_RANGE:
2543	ret = kfd_criu_restore_svm(p, user_priv_ptr: (uint8_t __user *)args->priv_data,
2544	priv_data_offset: priv_offset, max_priv_data_size);
2545	if (ret)
2546	goto exit;
2547	break;
2548	default:
2549	pr_err("Invalid object type:%u at index:%d\n", object_type, i);
2550	ret = -EINVAL;
2551	goto exit;
2552	}
2553	}
2554	exit:
2555	return ret;
2556	}
2557
2558	static int criu_restore(struct file *filep,
2559	struct kfd_process *p,
2560	struct kfd_ioctl_criu_args *args)
2561	{
2562	uint64_t priv_offset = 0;
2563	int ret = 0;
2564
2565	pr_debug("CRIU restore (num_devices:%u num_bos:%u num_objects:%u priv_data_size:%llu)\n",
2566	args->num_devices, args->num_bos, args->num_objects, args->priv_data_size);
2567
2568	if ((args->num_bos > 0 && !args->bos) || !args->devices || !args->priv_data ||
2569	!args->priv_data_size || !args->num_devices)
2570	return -EINVAL;
2571
2572	mutex_lock(&p->mutex);
2573
2574	/*
2575	* Set the process to evicted state to avoid running any new queues before all the memory
2576	* mappings are ready.
2577	*/
2578	ret = kfd_process_evict_queues(p, trigger: KFD_QUEUE_EVICTION_CRIU_RESTORE);
2579	if (ret)
2580	goto exit_unlock;
2581
2582	/* Each function will adjust priv_offset based on how many bytes they consumed */
2583	ret = criu_restore_process(p, args, priv_offset: &priv_offset, max_priv_data_size: args->priv_data_size);
2584	if (ret)
2585	goto exit_unlock;
2586
2587	ret = criu_restore_devices(p, args, priv_offset: &priv_offset, max_priv_data_size: args->priv_data_size);
2588	if (ret)
2589	goto exit_unlock;
2590
2591	ret = criu_restore_bos(p, args, priv_offset: &priv_offset, max_priv_data_size: args->priv_data_size);
2592	if (ret)
2593	goto exit_unlock;
2594
2595	ret = criu_restore_objects(filep, p, args, priv_offset: &priv_offset, max_priv_data_size: args->priv_data_size);
2596	if (ret)
2597	goto exit_unlock;
2598
2599	if (priv_offset != args->priv_data_size) {
2600	pr_err("Invalid private data size\n");
2601	ret = -EINVAL;
2602	}
2603
2604	exit_unlock:
2605	mutex_unlock(lock: &p->mutex);
2606	if (ret)
2607	pr_err("Failed to restore CRIU ret:%d\n", ret);
2608	else
2609	pr_debug("CRIU restore successful\n");
2610
2611	return ret;
2612	}
2613
2614	static int criu_unpause(struct file *filep,
2615	struct kfd_process *p,
2616	struct kfd_ioctl_criu_args *args)
2617	{
2618	int ret;
2619
2620	mutex_lock(&p->mutex);
2621
2622	if (!p->queues_paused) {
2623	mutex_unlock(lock: &p->mutex);
2624	return -EINVAL;
2625	}
2626
2627	ret = kfd_process_restore_queues(p);
2628	if (ret)
2629	pr_err("Failed to unpause queues ret:%d\n", ret);
2630	else
2631	p->queues_paused = false;
2632
2633	mutex_unlock(lock: &p->mutex);
2634
2635	return ret;
2636	}
2637
2638	static int criu_resume(struct file *filep,
2639	struct kfd_process *p,
2640	struct kfd_ioctl_criu_args *args)
2641	{
2642	struct kfd_process *target = NULL;
2643	struct pid *pid = NULL;
2644	int ret = 0;
2645
2646	pr_debug("Inside %s, target pid for criu restore: %d\n", __func__,
2647	args->pid);
2648
2649	pid = find_get_pid(nr: args->pid);
2650	if (!pid) {
2651	pr_err("Cannot find pid info for %i\n", args->pid);
2652	return -ESRCH;
2653	}
2654
2655	pr_debug("calling kfd_lookup_process_by_pid\n");
2656	target = kfd_lookup_process_by_pid(pid);
2657
2658	put_pid(pid);
2659
2660	if (!target) {
2661	pr_debug("Cannot find process info for %i\n", args->pid);
2662	return -ESRCH;
2663	}
2664
2665	mutex_lock(&target->mutex);
2666	ret = kfd_criu_resume_svm(p: target);
2667	if (ret) {
2668	pr_err("kfd_criu_resume_svm failed for %i\n", args->pid);
2669	goto exit;
2670	}
2671
2672	ret = amdgpu_amdkfd_criu_resume(p: target->kgd_process_info);
2673	if (ret)
2674	pr_err("amdgpu_amdkfd_criu_resume failed for %i\n", args->pid);
2675
2676	exit:
2677	mutex_unlock(lock: &target->mutex);
2678
2679	kfd_unref_process(p: target);
2680	return ret;
2681	}
2682
2683	static int criu_process_info(struct file *filep,
2684	struct kfd_process *p,
2685	struct kfd_ioctl_criu_args *args)
2686	{
2687	int ret = 0;
2688
2689	mutex_lock(&p->mutex);
2690
2691	if (!p->n_pdds) {
2692	pr_err("No pdd for given process\n");
2693	ret = -ENODEV;
2694	goto err_unlock;
2695	}
2696
2697	ret = kfd_process_evict_queues(p, trigger: KFD_QUEUE_EVICTION_CRIU_CHECKPOINT);
2698	if (ret)
2699	goto err_unlock;
2700
2701	p->queues_paused = true;
2702
2703	args->pid = task_pid_nr_ns(tsk: p->lead_thread,
2704	ns: task_active_pid_ns(tsk: p->lead_thread));
2705
2706	ret = criu_get_process_object_info(p, num_devices: &args->num_devices, num_bos: &args->num_bos,
2707	num_objects: &args->num_objects, objs_priv_size: &args->priv_data_size);
2708	if (ret)
2709	goto err_unlock;
2710
2711	dev_dbg(kfd_device, "Num of devices:%u bos:%u objects:%u priv_data_size:%lld\n",
2712	args->num_devices, args->num_bos, args->num_objects,
2713	args->priv_data_size);
2714
2715	err_unlock:
2716	if (ret) {
2717	kfd_process_restore_queues(p);
2718	p->queues_paused = false;
2719	}
2720	mutex_unlock(lock: &p->mutex);
2721	return ret;
2722	}
2723
2724	static int kfd_ioctl_criu(struct file *filep, struct kfd_process *p, void *data)
2725	{
2726	struct kfd_ioctl_criu_args *args = data;
2727	int ret;
2728
2729	dev_dbg(kfd_device, "CRIU operation: %d\n", args->op);
2730	switch (args->op) {
2731	case KFD_CRIU_OP_PROCESS_INFO:
2732	ret = criu_process_info(filep, p, args);
2733	break;
2734	case KFD_CRIU_OP_CHECKPOINT:
2735	ret = criu_checkpoint(filep, p, args);
2736	break;
2737	case KFD_CRIU_OP_UNPAUSE:
2738	ret = criu_unpause(filep, p, args);
2739	break;
2740	case KFD_CRIU_OP_RESTORE:
2741	ret = criu_restore(filep, p, args);
2742	break;
2743	case KFD_CRIU_OP_RESUME:
2744	ret = criu_resume(filep, p, args);
2745	break;
2746	default:
2747	dev_dbg(kfd_device, "Unsupported CRIU operation:%d\n", args->op);
2748	ret = -EINVAL;
2749	break;
2750	}
2751
2752	if (ret)
2753	dev_dbg(kfd_device, "CRIU operation:%d err:%d\n", args->op, ret);
2754
2755	return ret;
2756	}
2757
2758	static int runtime_enable(struct kfd_process *p, uint64_t r_debug,
2759	bool enable_ttmp_setup)
2760	{
2761	int i = 0, ret = 0;
2762
2763	if (p->is_runtime_retry)
2764	goto retry;
2765
2766	if (p->runtime_info.runtime_state != DEBUG_RUNTIME_STATE_DISABLED)
2767	return -EBUSY;
2768
2769	for (i = 0; i < p->n_pdds; i++) {
2770	struct kfd_process_device *pdd = p->pdds[i];
2771
2772	if (pdd->qpd.queue_count)
2773	return -EEXIST;
2774
2775	/*
2776	* Setup TTMPs by default.
2777	* Note that this call must remain here for MES ADD QUEUE to
2778	* skip_process_ctx_clear unconditionally as the first call to
2779	* SET_SHADER_DEBUGGER clears any stale process context data
2780	* saved in MES.
2781	*/
2782	if (pdd->dev->kfd->shared_resources.enable_mes)
2783	kfd_dbg_set_mes_debug_mode(pdd, sq_trap_en: !kfd_dbg_has_cwsr_workaround(dev: pdd->dev));
2784	}
2785
2786	p->runtime_info.runtime_state = DEBUG_RUNTIME_STATE_ENABLED;
2787	p->runtime_info.r_debug = r_debug;
2788	p->runtime_info.ttmp_setup = enable_ttmp_setup;
2789
2790	if (p->runtime_info.ttmp_setup) {
2791	for (i = 0; i < p->n_pdds; i++) {
2792	struct kfd_process_device *pdd = p->pdds[i];
2793
2794	if (!kfd_dbg_is_rlc_restore_supported(dev: pdd->dev)) {
2795	amdgpu_gfx_off_ctrl(adev: pdd->dev->adev, enable: false);
2796	pdd->dev->kfd2kgd->enable_debug_trap(
2797	pdd->dev->adev,
2798	true,
2799	pdd->dev->vm_info.last_vmid_kfd);
2800	} else if (kfd_dbg_is_per_vmid_supported(dev: pdd->dev)) {
2801	pdd->spi_dbg_override = pdd->dev->kfd2kgd->enable_debug_trap(
2802	pdd->dev->adev,
2803	false,
2804	0);
2805	}
2806	}
2807	}
2808
2809	retry:
2810	if (p->debug_trap_enabled) {
2811	if (!p->is_runtime_retry) {
2812	kfd_dbg_trap_activate(target: p);
2813	kfd_dbg_ev_raise(KFD_EC_MASK(EC_PROCESS_RUNTIME),
2814	process: p, NULL, source_id: 0, use_worker: false, NULL, exception_data_size: 0);
2815	}
2816
2817	mutex_unlock(lock: &p->mutex);
2818	ret = down_interruptible(sem: &p->runtime_enable_sema);
2819	mutex_lock(&p->mutex);
2820
2821	p->is_runtime_retry = !!ret;
2822	}
2823
2824	return ret;
2825	}
2826
2827	static int runtime_disable(struct kfd_process *p)
2828	{
2829	int i = 0, ret = 0;
2830	bool was_enabled = p->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED;
2831
2832	p->runtime_info.runtime_state = DEBUG_RUNTIME_STATE_DISABLED;
2833	p->runtime_info.r_debug = 0;
2834
2835	if (p->debug_trap_enabled) {
2836	if (was_enabled)
2837	kfd_dbg_trap_deactivate(target: p, unwind: false, unwind_count: 0);
2838
2839	if (!p->is_runtime_retry)
2840	kfd_dbg_ev_raise(KFD_EC_MASK(EC_PROCESS_RUNTIME),
2841	process: p, NULL, source_id: 0, use_worker: false, NULL, exception_data_size: 0);
2842
2843	mutex_unlock(lock: &p->mutex);
2844	ret = down_interruptible(sem: &p->runtime_enable_sema);
2845	mutex_lock(&p->mutex);
2846
2847	p->is_runtime_retry = !!ret;
2848	if (ret)
2849	return ret;
2850	}
2851
2852	if (was_enabled && p->runtime_info.ttmp_setup) {
2853	for (i = 0; i < p->n_pdds; i++) {
2854	struct kfd_process_device *pdd = p->pdds[i];
2855
2856	if (!kfd_dbg_is_rlc_restore_supported(dev: pdd->dev))
2857	amdgpu_gfx_off_ctrl(adev: pdd->dev->adev, enable: true);
2858	}
2859	}
2860
2861	p->runtime_info.ttmp_setup = false;
2862
2863	/* disable ttmp setup */
2864	for (i = 0; i < p->n_pdds; i++) {
2865	struct kfd_process_device *pdd = p->pdds[i];
2866	int last_err = 0;
2867
2868	if (kfd_dbg_is_per_vmid_supported(dev: pdd->dev)) {
2869	pdd->spi_dbg_override =
2870	pdd->dev->kfd2kgd->disable_debug_trap(
2871	pdd->dev->adev,
2872	false,
2873	pdd->dev->vm_info.last_vmid_kfd);
2874
2875	if (!pdd->dev->kfd->shared_resources.enable_mes)
2876	last_err = debug_refresh_runlist(dqm: pdd->dev->dqm);
2877	else
2878	last_err = kfd_dbg_set_mes_debug_mode(pdd,
2879	sq_trap_en: !kfd_dbg_has_cwsr_workaround(dev: pdd->dev));
2880
2881	if (last_err)
2882	ret = last_err;
2883	}
2884	}
2885
2886	return ret;
2887	}
2888
2889	static int kfd_ioctl_runtime_enable(struct file *filep, struct kfd_process *p, void *data)
2890	{
2891	struct kfd_ioctl_runtime_enable_args *args = data;
2892	int r;
2893
2894	mutex_lock(&p->mutex);
2895
2896	if (args->mode_mask & KFD_RUNTIME_ENABLE_MODE_ENABLE_MASK)
2897	r = runtime_enable(p, r_debug: args->r_debug,
2898	enable_ttmp_setup: !!(args->mode_mask & KFD_RUNTIME_ENABLE_MODE_TTMP_SAVE_MASK));
2899	else
2900	r = runtime_disable(p);
2901
2902	mutex_unlock(lock: &p->mutex);
2903
2904	return r;
2905	}
2906
2907	static int kfd_ioctl_set_debug_trap(struct file *filep, struct kfd_process *p, void *data)
2908	{
2909	struct kfd_ioctl_dbg_trap_args *args = data;
2910	struct task_struct *thread = NULL;
2911	struct mm_struct *mm = NULL;
2912	struct pid *pid = NULL;
2913	struct kfd_process *target = NULL;
2914	struct kfd_process_device *pdd = NULL;
2915	int r = 0;
2916
2917	if (sched_policy == KFD_SCHED_POLICY_NO_HWS) {
2918	pr_err("Debugging does not support sched_policy %i", sched_policy);
2919	return -EINVAL;
2920	}
2921
2922	pid = find_get_pid(nr: args->pid);
2923	if (!pid) {
2924	pr_debug("Cannot find pid info for %i\n", args->pid);
2925	r = -ESRCH;
2926	goto out;
2927	}
2928
2929	thread = get_pid_task(pid, PIDTYPE_PID);
2930	if (!thread) {
2931	r = -ESRCH;
2932	goto out;
2933	}
2934
2935	mm = get_task_mm(task: thread);
2936	if (!mm) {
2937	r = -ESRCH;
2938	goto out;
2939	}
2940
2941	if (args->op == KFD_IOC_DBG_TRAP_ENABLE) {
2942	bool create_process;
2943
2944	rcu_read_lock();
2945	create_process = thread && thread != current && ptrace_parent(task: thread) == current;
2946	rcu_read_unlock();
2947
2948	target = create_process ? kfd_create_process(thread) :
2949	kfd_lookup_process_by_pid(pid);
2950	} else {
2951	target = kfd_lookup_process_by_pid(pid);
2952	}
2953
2954	if (IS_ERR_OR_NULL(ptr: target)) {
2955	pr_debug("Cannot find process PID %i to debug\n", args->pid);
2956	r = target ? PTR_ERR(ptr: target) : -ESRCH;
2957	target = NULL;
2958	goto out;
2959	}
2960
2961	/* Check if target is still PTRACED. */
2962	rcu_read_lock();
2963	if (target != p && args->op != KFD_IOC_DBG_TRAP_DISABLE
2964	&& ptrace_parent(task: target->lead_thread) != current) {
2965	pr_err("PID %i is not PTRACED and cannot be debugged\n", args->pid);
2966	r = -EPERM;
2967	}
2968	rcu_read_unlock();
2969
2970	if (r)
2971	goto out;
2972
2973	mutex_lock(&target->mutex);
2974
2975	if (args->op != KFD_IOC_DBG_TRAP_ENABLE && !target->debug_trap_enabled) {
2976	pr_err("PID %i not debug enabled for op %i\n", args->pid, args->op);
2977	r = -EINVAL;
2978	goto unlock_out;
2979	}
2980
2981	if (target->runtime_info.runtime_state != DEBUG_RUNTIME_STATE_ENABLED &&
2982	(args->op == KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE ||
2983	args->op == KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE ||
2984	args->op == KFD_IOC_DBG_TRAP_SUSPEND_QUEUES ||
2985	args->op == KFD_IOC_DBG_TRAP_RESUME_QUEUES ||
2986	args->op == KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH ||
2987	args->op == KFD_IOC_DBG_TRAP_CLEAR_NODE_ADDRESS_WATCH ||
2988	args->op == KFD_IOC_DBG_TRAP_SET_FLAGS)) {
2989	r = -EPERM;
2990	goto unlock_out;
2991	}
2992
2993	if (args->op == KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH ||
2994	args->op == KFD_IOC_DBG_TRAP_CLEAR_NODE_ADDRESS_WATCH) {
2995	int user_gpu_id = kfd_process_get_user_gpu_id(p: target,
2996	actual_gpu_id: args->op == KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH ?
2997	args->set_node_address_watch.gpu_id :
2998	args->clear_node_address_watch.gpu_id);
2999
3000	pdd = kfd_process_device_data_by_id(process: target, gpu_id: user_gpu_id);
3001	if (user_gpu_id == -EINVAL || !pdd) {
3002	r = -ENODEV;
3003	goto unlock_out;
3004	}
3005	}
3006
3007	switch (args->op) {
3008	case KFD_IOC_DBG_TRAP_ENABLE:
3009	if (target != p)
3010	target->debugger_process = p;
3011
3012	r = kfd_dbg_trap_enable(target,
3013	fd: args->enable.dbg_fd,
3014	runtime_info: (void __user *)args->enable.rinfo_ptr,
3015	runtime_info_size: &args->enable.rinfo_size);
3016	if (!r)
3017	target->exception_enable_mask = args->enable.exception_mask;
3018
3019	break;
3020	case KFD_IOC_DBG_TRAP_DISABLE:
3021	r = kfd_dbg_trap_disable(target);
3022	break;
3023	case KFD_IOC_DBG_TRAP_SEND_RUNTIME_EVENT:
3024	r = kfd_dbg_send_exception_to_runtime(p: target,
3025	dev_id: args->send_runtime_event.gpu_id,
3026	queue_id: args->send_runtime_event.queue_id,
3027	error_reason: args->send_runtime_event.exception_mask);
3028	break;
3029	case KFD_IOC_DBG_TRAP_SET_EXCEPTIONS_ENABLED:
3030	kfd_dbg_set_enabled_debug_exception_mask(target,
3031	exception_set_mask: args->set_exceptions_enabled.exception_mask);
3032	break;
3033	case KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE:
3034	r = kfd_dbg_trap_set_wave_launch_override(target,
3035	trap_override: args->launch_override.override_mode,
3036	trap_mask_bits: args->launch_override.enable_mask,
3037	trap_mask_request: args->launch_override.support_request_mask,
3038	trap_mask_prev: &args->launch_override.enable_mask,
3039	trap_mask_supported: &args->launch_override.support_request_mask);
3040	break;
3041	case KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE:
3042	r = kfd_dbg_trap_set_wave_launch_mode(target,
3043	wave_launch_mode: args->launch_mode.launch_mode);
3044	break;
3045	case KFD_IOC_DBG_TRAP_SUSPEND_QUEUES:
3046	r = suspend_queues(p: target,
3047	num_queues: args->suspend_queues.num_queues,
3048	grace_period: args->suspend_queues.grace_period,
3049	exception_clear_mask: args->suspend_queues.exception_mask,
3050	usr_queue_id_array: (uint32_t *)args->suspend_queues.queue_array_ptr);
3051
3052	break;
3053	case KFD_IOC_DBG_TRAP_RESUME_QUEUES:
3054	r = resume_queues(p: target, num_queues: args->resume_queues.num_queues,
3055	usr_queue_id_array: (uint32_t *)args->resume_queues.queue_array_ptr);
3056	break;
3057	case KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH:
3058	r = kfd_dbg_trap_set_dev_address_watch(pdd,
3059	watch_address: args->set_node_address_watch.address,
3060	watch_address_mask: args->set_node_address_watch.mask,
3061	watch_id: &args->set_node_address_watch.id,
3062	watch_mode: args->set_node_address_watch.mode);
3063	break;
3064	case KFD_IOC_DBG_TRAP_CLEAR_NODE_ADDRESS_WATCH:
3065	r = kfd_dbg_trap_clear_dev_address_watch(pdd,
3066	watch_id: args->clear_node_address_watch.id);
3067	break;
3068	case KFD_IOC_DBG_TRAP_SET_FLAGS:
3069	r = kfd_dbg_trap_set_flags(target, flags: &args->set_flags.flags);
3070	break;
3071	case KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT:
3072	r = kfd_dbg_ev_query_debug_event(process: target,
3073	queue_id: &args->query_debug_event.queue_id,
3074	gpu_id: &args->query_debug_event.gpu_id,
3075	exception_clear_mask: args->query_debug_event.exception_mask,
3076	event_status: &args->query_debug_event.exception_mask);
3077	break;
3078	case KFD_IOC_DBG_TRAP_QUERY_EXCEPTION_INFO:
3079	r = kfd_dbg_trap_query_exception_info(target,
3080	source_id: args->query_exception_info.source_id,
3081	exception_code: args->query_exception_info.exception_code,
3082	clear_exception: args->query_exception_info.clear_exception,
3083	info: (void __user *)args->query_exception_info.info_ptr,
3084	info_size: &args->query_exception_info.info_size);
3085	break;
3086	case KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT:
3087	r = pqm_get_queue_snapshot(pqm: &target->pqm,
3088	exception_clear_mask: args->queue_snapshot.exception_mask,
3089	buf: (void __user *)args->queue_snapshot.snapshot_buf_ptr,
3090	num_qss_entries: &args->queue_snapshot.num_queues,
3091	entry_size: &args->queue_snapshot.entry_size);
3092	break;
3093	case KFD_IOC_DBG_TRAP_GET_DEVICE_SNAPSHOT:
3094	r = kfd_dbg_trap_device_snapshot(target,
3095	exception_clear_mask: args->device_snapshot.exception_mask,
3096	user_info: (void __user *)args->device_snapshot.snapshot_buf_ptr,
3097	number_of_device_infos: &args->device_snapshot.num_devices,
3098	entry_size: &args->device_snapshot.entry_size);
3099	break;
3100	default:
3101	pr_err("Invalid option: %i\n", args->op);
3102	r = -EINVAL;
3103	}
3104
3105	unlock_out:
3106	mutex_unlock(lock: &target->mutex);
3107
3108	out:
3109	if (thread)
3110	put_task_struct(t: thread);
3111
3112	if (mm)
3113	mmput(mm);
3114
3115	if (pid)
3116	put_pid(pid);
3117
3118	if (target)
3119	kfd_unref_process(p: target);
3120
3121	return r;
3122	}
3123
3124	#define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
3125	[_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
3126	.cmd_drv = 0, .name = #ioctl}
3127
3128	/** Ioctl table */
3129	static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
3130	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION,
3131	kfd_ioctl_get_version, 0),
3132
3133	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE,
3134	kfd_ioctl_create_queue, 0),
3135
3136	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE,
3137	kfd_ioctl_destroy_queue, 0),
3138
3139	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY,
3140	kfd_ioctl_set_memory_policy, 0),
3141
3142	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS,
3143	kfd_ioctl_get_clock_counters, 0),
3144
3145	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES,
3146	kfd_ioctl_get_process_apertures, 0),
3147
3148	AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE,
3149	kfd_ioctl_update_queue, 0),
3150
3151	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_EVENT,
3152	kfd_ioctl_create_event, 0),
3153
3154	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_EVENT,
3155	kfd_ioctl_destroy_event, 0),
3156
3157	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_EVENT,
3158	kfd_ioctl_set_event, 0),
3159
3160	AMDKFD_IOCTL_DEF(AMDKFD_IOC_RESET_EVENT,
3161	kfd_ioctl_reset_event, 0),
3162
3163	AMDKFD_IOCTL_DEF(AMDKFD_IOC_WAIT_EVENTS,
3164	kfd_ioctl_wait_events, 0),
3165
3166	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_REGISTER_DEPRECATED,
3167	kfd_ioctl_dbg_register, 0),
3168
3169	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_UNREGISTER_DEPRECATED,
3170	kfd_ioctl_dbg_unregister, 0),
3171
3172	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_ADDRESS_WATCH_DEPRECATED,
3173	kfd_ioctl_dbg_address_watch, 0),
3174
3175	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL_DEPRECATED,
3176	kfd_ioctl_dbg_wave_control, 0),
3177
3178	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_SCRATCH_BACKING_VA,
3179	kfd_ioctl_set_scratch_backing_va, 0),
3180
3181	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_TILE_CONFIG,
3182	kfd_ioctl_get_tile_config, 0),
3183
3184	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_TRAP_HANDLER,
3185	kfd_ioctl_set_trap_handler, 0),
3186
3187	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES_NEW,
3188	kfd_ioctl_get_process_apertures_new, 0),
3189
3190	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ACQUIRE_VM,
3191	kfd_ioctl_acquire_vm, 0),
3192
3193	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_MEMORY_OF_GPU,
3194	kfd_ioctl_alloc_memory_of_gpu, 0),
3195
3196	AMDKFD_IOCTL_DEF(AMDKFD_IOC_FREE_MEMORY_OF_GPU,
3197	kfd_ioctl_free_memory_of_gpu, 0),
3198
3199	AMDKFD_IOCTL_DEF(AMDKFD_IOC_MAP_MEMORY_TO_GPU,
3200	kfd_ioctl_map_memory_to_gpu, 0),
3201
3202	AMDKFD_IOCTL_DEF(AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU,
3203	kfd_ioctl_unmap_memory_from_gpu, 0),
3204
3205	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_CU_MASK,
3206	kfd_ioctl_set_cu_mask, 0),
3207
3208	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_QUEUE_WAVE_STATE,
3209	kfd_ioctl_get_queue_wave_state, 0),
3210
3211	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_DMABUF_INFO,
3212	kfd_ioctl_get_dmabuf_info, 0),
3213
3214	AMDKFD_IOCTL_DEF(AMDKFD_IOC_IMPORT_DMABUF,
3215	kfd_ioctl_import_dmabuf, 0),
3216
3217	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_QUEUE_GWS,
3218	kfd_ioctl_alloc_queue_gws, 0),
3219
3220	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SMI_EVENTS,
3221	kfd_ioctl_smi_events, 0),
3222
3223	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SVM, kfd_ioctl_svm, 0),
3224
3225	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_XNACK_MODE,
3226	kfd_ioctl_set_xnack_mode, 0),
3227
3228	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CRIU_OP,
3229	kfd_ioctl_criu, KFD_IOC_FLAG_CHECKPOINT_RESTORE),
3230
3231	AMDKFD_IOCTL_DEF(AMDKFD_IOC_AVAILABLE_MEMORY,
3232	kfd_ioctl_get_available_memory, 0),
3233
3234	AMDKFD_IOCTL_DEF(AMDKFD_IOC_EXPORT_DMABUF,
3235	kfd_ioctl_export_dmabuf, 0),
3236
3237	AMDKFD_IOCTL_DEF(AMDKFD_IOC_RUNTIME_ENABLE,
3238	kfd_ioctl_runtime_enable, 0),
3239
3240	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_TRAP,
3241	kfd_ioctl_set_debug_trap, 0),
3242	};
3243
3244	#define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)
3245
3246	static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
3247	{
3248	struct kfd_process *process;
3249	amdkfd_ioctl_t *func;
3250	const struct amdkfd_ioctl_desc *ioctl = NULL;
3251	unsigned int nr = _IOC_NR(cmd);
3252	char stack_kdata[128];
3253	char *kdata = NULL;
3254	unsigned int usize, asize;
3255	int retcode = -EINVAL;
3256	bool ptrace_attached = false;
3257
3258	if (nr >= AMDKFD_CORE_IOCTL_COUNT) {
3259	retcode = -ENOTTY;
3260	goto err_i1;
3261	}
3262
3263	if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) {
3264	u32 amdkfd_size;
3265
3266	ioctl = &amdkfd_ioctls[nr];
3267
3268	amdkfd_size = _IOC_SIZE(ioctl->cmd);
3269	usize = asize = _IOC_SIZE(cmd);
3270	if (amdkfd_size > asize)
3271	asize = amdkfd_size;
3272
3273	cmd = ioctl->cmd;
3274	} else {
3275	retcode = -ENOTTY;
3276	goto err_i1;
3277	}
3278
3279	dev_dbg(kfd_device, "ioctl cmd 0x%x (#0x%x), arg 0x%lx\n", cmd, nr, arg);
3280
3281	/* Get the process struct from the filep. Only the process
3282	* that opened /dev/kfd can use the file descriptor. Child
3283	* processes need to create their own KFD device context.
3284	*/
3285	process = filep->private_data;
3286
3287	rcu_read_lock();
3288	if ((ioctl->flags & KFD_IOC_FLAG_CHECKPOINT_RESTORE) &&
3289	ptrace_parent(task: process->lead_thread) == current)
3290	ptrace_attached = true;
3291	rcu_read_unlock();
3292
3293	if (process->lead_thread != current->group_leader
3294	&& !ptrace_attached) {
3295	dev_dbg(kfd_device, "Using KFD FD in wrong process\n");
3296	retcode = -EBADF;
3297	goto err_i1;
3298	}
3299
3300	/* Do not trust userspace, use our own definition */
3301	func = ioctl->func;
3302
3303	if (unlikely(!func)) {
3304	dev_dbg(kfd_device, "no function\n");
3305	retcode = -EINVAL;
3306	goto err_i1;
3307	}
3308
3309	/*
3310	* Versions of docker shipped in Ubuntu 18.xx and 20.xx do not support
3311	* CAP_CHECKPOINT_RESTORE, so we also allow access if CAP_SYS_ADMIN as CAP_SYS_ADMIN is a
3312	* more priviledged access.
3313	*/
3314	if (unlikely(ioctl->flags & KFD_IOC_FLAG_CHECKPOINT_RESTORE)) {
3315	if (!capable(CAP_CHECKPOINT_RESTORE) &&
3316	!capable(CAP_SYS_ADMIN)) {
3317	retcode = -EACCES;
3318	goto err_i1;
3319	}
3320	}
3321
3322	if (cmd & (IOC_IN | IOC_OUT)) {
3323	if (asize <= sizeof(stack_kdata)) {
3324	kdata = stack_kdata;
3325	} else {
3326	kdata = kmalloc(asize, GFP_KERNEL);
3327	if (!kdata) {
3328	retcode = -ENOMEM;
3329	goto err_i1;
3330	}
3331	}
3332	if (asize > usize)
3333	memset(kdata + usize, 0, asize - usize);
3334	}
3335
3336	if (cmd & IOC_IN) {
3337	if (copy_from_user(to: kdata, from: (void __user *)arg, n: usize) != 0) {
3338	retcode = -EFAULT;
3339	goto err_i1;
3340	}
3341	} else if (cmd & IOC_OUT) {
3342	memset(kdata, 0, usize);
3343	}
3344
3345	retcode = func(filep, process, kdata);
3346
3347	if (cmd & IOC_OUT)
3348	if (copy_to_user(to: (void __user *)arg, from: kdata, n: usize) != 0)
3349	retcode = -EFAULT;
3350
3351	err_i1:
3352	if (!ioctl)
3353	dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
3354	task_pid_nr(current), cmd, nr);
3355
3356	if (kdata != stack_kdata)
3357	kfree(objp: kdata);
3358
3359	if (retcode)
3360	dev_dbg(kfd_device, "ioctl cmd (#0x%x), arg 0x%lx, ret = %d\n",
3361	nr, arg, retcode);
3362
3363	return retcode;
3364	}
3365
3366	static int kfd_mmio_mmap(struct kfd_node *dev, struct kfd_process *process,
3367	struct vm_area_struct *vma)
3368	{
3369	phys_addr_t address;
3370
3371	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
3372	return -EINVAL;
3373
3374	if (PAGE_SIZE > 4096)
3375	return -EINVAL;
3376
3377	address = dev->adev->rmmio_remap.bus_addr;
3378
3379	vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
3380	VM_DONTDUMP | VM_PFNMAP);
3381
3382	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
3383
3384	pr_debug("process pid %d mapping mmio page\n"
3385	" target user address == 0x%08llX\n"
3386	" physical address == 0x%08llX\n"
3387	" vm_flags == 0x%04lX\n"
3388	" size == 0x%04lX\n",
3389	process->lead_thread->pid, (unsigned long long) vma->vm_start,
3390	address, vma->vm_flags, PAGE_SIZE);
3391
3392	return io_remap_pfn_range(vma,
3393	addr: vma->vm_start,
3394	orig_pfn: address >> PAGE_SHIFT,
3395	PAGE_SIZE,
3396	orig_prot: vma->vm_page_prot);
3397	}
3398
3399
3400	static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
3401	{
3402	struct kfd_process *process;
3403	struct kfd_node *dev = NULL;
3404	unsigned long mmap_offset;
3405	unsigned int gpu_id;
3406
3407	process = kfd_get_process(current);
3408	if (IS_ERR(ptr: process))
3409	return PTR_ERR(ptr: process);
3410
3411	mmap_offset = vma->vm_pgoff << PAGE_SHIFT;
3412	gpu_id = KFD_MMAP_GET_GPU_ID(mmap_offset);
3413	if (gpu_id)
3414	dev = kfd_device_by_id(gpu_id);
3415
3416	switch (mmap_offset & KFD_MMAP_TYPE_MASK) {
3417	case KFD_MMAP_TYPE_DOORBELL:
3418	if (!dev)
3419	return -ENODEV;
3420	return kfd_doorbell_mmap(dev, process, vma);
3421
3422	case KFD_MMAP_TYPE_EVENTS:
3423	return kfd_event_mmap(process, vma);
3424
3425	case KFD_MMAP_TYPE_RESERVED_MEM:
3426	if (!dev)
3427	return -ENODEV;
3428	return kfd_reserved_mem_mmap(dev, process, vma);
3429	case KFD_MMAP_TYPE_MMIO:
3430	if (!dev)
3431	return -ENODEV;
3432	return kfd_mmio_mmap(dev, process, vma);
3433	}
3434
3435	return -EFAULT;
3436	}
3437