Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0
2
3/*
4 * Copyright 2016-2019 HabanaLabs, Ltd.
5 * All Rights Reserved.
6 */
7
8#include <uapi/misc/habanalabs.h>
9#include "habanalabs.h"
10
11#include <linux/fs.h>
12#include <linux/uaccess.h>
13#include <linux/slab.h>
14
15static u32 hl_debug_struct_size[HL_DEBUG_OP_TIMESTAMP + 1] = {
16 [HL_DEBUG_OP_ETR] = sizeof(struct hl_debug_params_etr),
17 [HL_DEBUG_OP_ETF] = sizeof(struct hl_debug_params_etf),
18 [HL_DEBUG_OP_STM] = sizeof(struct hl_debug_params_stm),
19 [HL_DEBUG_OP_FUNNEL] = 0,
20 [HL_DEBUG_OP_BMON] = sizeof(struct hl_debug_params_bmon),
21 [HL_DEBUG_OP_SPMU] = sizeof(struct hl_debug_params_spmu),
22 [HL_DEBUG_OP_TIMESTAMP] = 0
23
24};
25
26static int device_status_info(struct hl_device *hdev, struct hl_info_args *args)
27{
28 struct hl_info_device_status dev_stat = {0};
29 u32 size = args->return_size;
30 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
31
32 if ((!size) || (!out))
33 return -EINVAL;
34
35 dev_stat.status = hl_device_status(hdev);
36
37 return copy_to_user(out, &dev_stat,
38 min((size_t)size, sizeof(dev_stat))) ? -EFAULT : 0;
39}
40
41static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args)
42{
43 struct hl_info_hw_ip_info hw_ip = {0};
44 u32 size = args->return_size;
45 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
46 struct asic_fixed_properties *prop = &hdev->asic_prop;
47 u64 sram_kmd_size, dram_kmd_size;
48
49 if ((!size) || (!out))
50 return -EINVAL;
51
52 sram_kmd_size = (prop->sram_user_base_address -
53 prop->sram_base_address);
54 dram_kmd_size = (prop->dram_user_base_address -
55 prop->dram_base_address);
56
57 hw_ip.device_id = hdev->asic_funcs->get_pci_id(hdev);
58 hw_ip.sram_base_address = prop->sram_user_base_address;
59 hw_ip.dram_base_address = prop->dram_user_base_address;
60 hw_ip.tpc_enabled_mask = prop->tpc_enabled_mask;
61 hw_ip.sram_size = prop->sram_size - sram_kmd_size;
62 hw_ip.dram_size = prop->dram_size - dram_kmd_size;
63 if (hw_ip.dram_size > 0)
64 hw_ip.dram_enabled = 1;
65 hw_ip.num_of_events = prop->num_of_events;
66 memcpy(hw_ip.armcp_version,
67 prop->armcp_info.armcp_version, VERSION_MAX_LEN);
68 hw_ip.armcp_cpld_version = le32_to_cpu(prop->armcp_info.cpld_version);
69 hw_ip.psoc_pci_pll_nr = prop->psoc_pci_pll_nr;
70 hw_ip.psoc_pci_pll_nf = prop->psoc_pci_pll_nf;
71 hw_ip.psoc_pci_pll_od = prop->psoc_pci_pll_od;
72 hw_ip.psoc_pci_pll_div_factor = prop->psoc_pci_pll_div_factor;
73
74 return copy_to_user(out, &hw_ip,
75 min((size_t)size, sizeof(hw_ip))) ? -EFAULT : 0;
76}
77
78static int hw_events_info(struct hl_device *hdev, bool aggregate,
79 struct hl_info_args *args)
80{
81 u32 size, max_size = args->return_size;
82 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
83 void *arr;
84
85 if ((!max_size) || (!out))
86 return -EINVAL;
87
88 arr = hdev->asic_funcs->get_events_stat(hdev, aggregate, &size);
89
90 return copy_to_user(out, arr, min(max_size, size)) ? -EFAULT : 0;
91}
92
93static int dram_usage_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
94{
95 struct hl_device *hdev = hpriv->hdev;
96 struct hl_info_dram_usage dram_usage = {0};
97 u32 max_size = args->return_size;
98 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
99 struct asic_fixed_properties *prop = &hdev->asic_prop;
100 u64 dram_kmd_size;
101
102 if ((!max_size) || (!out))
103 return -EINVAL;
104
105 dram_kmd_size = (prop->dram_user_base_address -
106 prop->dram_base_address);
107 dram_usage.dram_free_mem = (prop->dram_size - dram_kmd_size) -
108 atomic64_read(&hdev->dram_used_mem);
109 if (hpriv->ctx)
110 dram_usage.ctx_dram_mem =
111 atomic64_read(&hpriv->ctx->dram_phys_mem);
112
113 return copy_to_user(out, &dram_usage,
114 min((size_t) max_size, sizeof(dram_usage))) ? -EFAULT : 0;
115}
116
117static int hw_idle(struct hl_device *hdev, struct hl_info_args *args)
118{
119 struct hl_info_hw_idle hw_idle = {0};
120 u32 max_size = args->return_size;
121 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
122
123 if ((!max_size) || (!out))
124 return -EINVAL;
125
126 hw_idle.is_idle = hdev->asic_funcs->is_device_idle(hdev,
127 &hw_idle.busy_engines_mask, NULL);
128
129 return copy_to_user(out, &hw_idle,
130 min((size_t) max_size, sizeof(hw_idle))) ? -EFAULT : 0;
131}
132
133static int debug_coresight(struct hl_device *hdev, struct hl_debug_args *args)
134{
135 struct hl_debug_params *params;
136 void *input = NULL, *output = NULL;
137 int rc;
138
139 params = kzalloc(sizeof(*params), GFP_KERNEL);
140 if (!params)
141 return -ENOMEM;
142
143 params->reg_idx = args->reg_idx;
144 params->enable = args->enable;
145 params->op = args->op;
146
147 if (args->input_ptr && args->input_size) {
148 input = kzalloc(hl_debug_struct_size[args->op], GFP_KERNEL);
149 if (!input) {
150 rc = -ENOMEM;
151 goto out;
152 }
153
154 if (copy_from_user(input, u64_to_user_ptr(args->input_ptr),
155 args->input_size)) {
156 rc = -EFAULT;
157 dev_err(hdev->dev, "failed to copy input debug data\n");
158 goto out;
159 }
160
161 params->input = input;
162 }
163
164 if (args->output_ptr && args->output_size) {
165 output = kzalloc(args->output_size, GFP_KERNEL);
166 if (!output) {
167 rc = -ENOMEM;
168 goto out;
169 }
170
171 params->output = output;
172 params->output_size = args->output_size;
173 }
174
175 rc = hdev->asic_funcs->debug_coresight(hdev, params);
176 if (rc) {
177 dev_err(hdev->dev,
178 "debug coresight operation failed %d\n", rc);
179 goto out;
180 }
181
182 if (output) {
183 if (copy_to_user((void __user *) (uintptr_t) args->output_ptr,
184 output,
185 args->output_size)) {
186 dev_err(hdev->dev,
187 "copy to user failed in debug ioctl\n");
188 rc = -EFAULT;
189 goto out;
190 }
191 }
192
193out:
194 kfree(params);
195 kfree(output);
196 kfree(input);
197
198 return rc;
199}
200
201static int device_utilization(struct hl_device *hdev, struct hl_info_args *args)
202{
203 struct hl_info_device_utilization device_util = {0};
204 u32 max_size = args->return_size;
205 void __user *out = (void __user *) (uintptr_t) args->return_pointer;
206
207 if ((!max_size) || (!out))
208 return -EINVAL;
209
210 if ((args->period_ms < 100) || (args->period_ms > 1000) ||
211 (args->period_ms % 100)) {
212 dev_err(hdev->dev,
213 "period %u must be between 100 - 1000 and must be divisible by 100\n",
214 args->period_ms);
215 return -EINVAL;
216 }
217
218 device_util.utilization = hl_device_utilization(hdev, args->period_ms);
219
220 return copy_to_user(out, &device_util,
221 min((size_t) max_size, sizeof(device_util))) ? -EFAULT : 0;
222}
223
224static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data,
225 struct device *dev)
226{
227 struct hl_info_args *args = data;
228 struct hl_device *hdev = hpriv->hdev;
229 int rc;
230
231 /*
232 * Information is returned for the following opcodes even if the device
233 * is disabled or in reset.
234 */
235 switch (args->op) {
236 case HL_INFO_HW_IP_INFO:
237 return hw_ip_info(hdev, args);
238
239 case HL_INFO_DEVICE_STATUS:
240 return device_status_info(hdev, args);
241
242 default:
243 break;
244 }
245
246 if (hl_device_disabled_or_in_reset(hdev)) {
247 dev_warn_ratelimited(dev,
248 "Device is %s. Can't execute INFO IOCTL\n",
249 atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
250 return -EBUSY;
251 }
252
253 switch (args->op) {
254 case HL_INFO_HW_EVENTS:
255 rc = hw_events_info(hdev, false, args);
256 break;
257
258 case HL_INFO_DRAM_USAGE:
259 rc = dram_usage_info(hpriv, args);
260 break;
261
262 case HL_INFO_HW_IDLE:
263 rc = hw_idle(hdev, args);
264 break;
265
266 case HL_INFO_DEVICE_UTILIZATION:
267 rc = device_utilization(hdev, args);
268 break;
269
270 case HL_INFO_HW_EVENTS_AGGREGATE:
271 rc = hw_events_info(hdev, true, args);
272 break;
273
274 default:
275 dev_err(dev, "Invalid request %d\n", args->op);
276 rc = -ENOTTY;
277 break;
278 }
279
280 return rc;
281}
282
283static int hl_info_ioctl(struct hl_fpriv *hpriv, void *data)
284{
285 return _hl_info_ioctl(hpriv, data, hpriv->hdev->dev);
286}
287
288static int hl_info_ioctl_control(struct hl_fpriv *hpriv, void *data)
289{
290 return _hl_info_ioctl(hpriv, data, hpriv->hdev->dev_ctrl);
291}
292
293static int hl_debug_ioctl(struct hl_fpriv *hpriv, void *data)
294{
295 struct hl_debug_args *args = data;
296 struct hl_device *hdev = hpriv->hdev;
297 int rc = 0;
298
299 if (hl_device_disabled_or_in_reset(hdev)) {
300 dev_warn_ratelimited(hdev->dev,
301 "Device is %s. Can't execute DEBUG IOCTL\n",
302 atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
303 return -EBUSY;
304 }
305
306 switch (args->op) {
307 case HL_DEBUG_OP_ETR:
308 case HL_DEBUG_OP_ETF:
309 case HL_DEBUG_OP_STM:
310 case HL_DEBUG_OP_FUNNEL:
311 case HL_DEBUG_OP_BMON:
312 case HL_DEBUG_OP_SPMU:
313 case HL_DEBUG_OP_TIMESTAMP:
314 if (!hdev->in_debug) {
315 dev_err_ratelimited(hdev->dev,
316 "Rejecting debug configuration request because device not in debug mode\n");
317 return -EFAULT;
318 }
319 args->input_size =
320 min(args->input_size, hl_debug_struct_size[args->op]);
321 rc = debug_coresight(hdev, args);
322 break;
323 case HL_DEBUG_OP_SET_MODE:
324 rc = hl_device_set_debug_mode(hdev, (bool) args->enable);
325 break;
326 default:
327 dev_err(hdev->dev, "Invalid request %d\n", args->op);
328 rc = -ENOTTY;
329 break;
330 }
331
332 return rc;
333}
334
335#define HL_IOCTL_DEF(ioctl, _func) \
336 [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func}
337
338static const struct hl_ioctl_desc hl_ioctls[] = {
339 HL_IOCTL_DEF(HL_IOCTL_INFO, hl_info_ioctl),
340 HL_IOCTL_DEF(HL_IOCTL_CB, hl_cb_ioctl),
341 HL_IOCTL_DEF(HL_IOCTL_CS, hl_cs_ioctl),
342 HL_IOCTL_DEF(HL_IOCTL_WAIT_CS, hl_cs_wait_ioctl),
343 HL_IOCTL_DEF(HL_IOCTL_MEMORY, hl_mem_ioctl),
344 HL_IOCTL_DEF(HL_IOCTL_DEBUG, hl_debug_ioctl)
345};
346
347static const struct hl_ioctl_desc hl_ioctls_control[] = {
348 HL_IOCTL_DEF(HL_IOCTL_INFO, hl_info_ioctl_control)
349};
350
351static long _hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg,
352 const struct hl_ioctl_desc *ioctl, struct device *dev)
353{
354 struct hl_fpriv *hpriv = filep->private_data;
355 struct hl_device *hdev = hpriv->hdev;
356 unsigned int nr = _IOC_NR(cmd);
357 char stack_kdata[128] = {0};
358 char *kdata = NULL;
359 unsigned int usize, asize;
360 hl_ioctl_t *func;
361 u32 hl_size;
362 int retcode;
363
364 if (hdev->hard_reset_pending) {
365 dev_crit_ratelimited(hdev->dev_ctrl,
366 "Device HARD reset pending! Please close FD\n");
367 return -ENODEV;
368 }
369
370 /* Do not trust userspace, use our own definition */
371 func = ioctl->func;
372
373 if (unlikely(!func)) {
374 dev_dbg(dev, "no function\n");
375 retcode = -ENOTTY;
376 goto out_err;
377 }
378
379 hl_size = _IOC_SIZE(ioctl->cmd);
380 usize = asize = _IOC_SIZE(cmd);
381 if (hl_size > asize)
382 asize = hl_size;
383
384 cmd = ioctl->cmd;
385
386 if (cmd & (IOC_IN | IOC_OUT)) {
387 if (asize <= sizeof(stack_kdata)) {
388 kdata = stack_kdata;
389 } else {
390 kdata = kzalloc(asize, GFP_KERNEL);
391 if (!kdata) {
392 retcode = -ENOMEM;
393 goto out_err;
394 }
395 }
396 }
397
398 if (cmd & IOC_IN) {
399 if (copy_from_user(kdata, (void __user *)arg, usize)) {
400 retcode = -EFAULT;
401 goto out_err;
402 }
403 } else if (cmd & IOC_OUT) {
404 memset(kdata, 0, usize);
405 }
406
407 retcode = func(hpriv, kdata);
408
409 if (cmd & IOC_OUT)
410 if (copy_to_user((void __user *)arg, kdata, usize))
411 retcode = -EFAULT;
412
413out_err:
414 if (retcode)
415 dev_dbg(dev, "error in ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
416 task_pid_nr(current), cmd, nr);
417
418 if (kdata != stack_kdata)
419 kfree(kdata);
420
421 return retcode;
422}
423
424long hl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
425{
426 struct hl_fpriv *hpriv = filep->private_data;
427 struct hl_device *hdev = hpriv->hdev;
428 const struct hl_ioctl_desc *ioctl = NULL;
429 unsigned int nr = _IOC_NR(cmd);
430
431 if ((nr >= HL_COMMAND_START) && (nr < HL_COMMAND_END)) {
432 ioctl = &hl_ioctls[nr];
433 } else {
434 dev_err(hdev->dev, "invalid ioctl: pid=%d, nr=0x%02x\n",
435 task_pid_nr(current), nr);
436 return -ENOTTY;
437 }
438
439 return _hl_ioctl(filep, cmd, arg, ioctl, hdev->dev);
440}
441
442long hl_ioctl_control(struct file *filep, unsigned int cmd, unsigned long arg)
443{
444 struct hl_fpriv *hpriv = filep->private_data;
445 struct hl_device *hdev = hpriv->hdev;
446 const struct hl_ioctl_desc *ioctl = NULL;
447 unsigned int nr = _IOC_NR(cmd);
448
449 if (nr == _IOC_NR(HL_IOCTL_INFO)) {
450 ioctl = &hl_ioctls_control[nr];
451 } else {
452 dev_err(hdev->dev_ctrl, "invalid ioctl: pid=%d, nr=0x%02x\n",
453 task_pid_nr(current), nr);
454 return -ENOTTY;
455 }
456
457 return _hl_ioctl(filep, cmd, arg, ioctl, hdev->dev_ctrl);
458}