1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright(c) 2023 Intel Corporation */
  3
  4#include <linux/dev_printk.h>
  5#include <linux/dma-mapping.h>
  6#include <linux/export.h>
  7#include <linux/kernel.h>
  8#include <linux/kstrtox.h>
  9#include <linux/overflow.h>
 10#include <linux/string.h>
 11#include <linux/slab.h>
 12#include <linux/types.h>
 13#include <asm/errno.h>
 14#include "adf_accel_devices.h"
 15#include "adf_admin.h"
 16#include "adf_cfg.h"
 17#include "adf_cfg_strings.h"
 18#include "adf_clock.h"
 19#include "adf_common_drv.h"
 20#include "adf_heartbeat.h"
 21#include "adf_transport_internal.h"
 22#include "icp_qat_fw_init_admin.h"
 23
 24#define ADF_HB_EMPTY_SIG 0xA5A5A5A5
 25
 26/* Heartbeat counter pair */
 27struct hb_cnt_pair {
 28	__u16 resp_heartbeat_cnt;
 29	__u16 req_heartbeat_cnt;
 30};
 31
 32static int adf_hb_check_polling_freq(struct adf_accel_dev *accel_dev)
 33{
 34	u64 curr_time = adf_clock_get_current_time();
 35	u64 polling_time = curr_time - accel_dev->heartbeat->last_hb_check_time;
 36
 37	if (polling_time < accel_dev->heartbeat->hb_timer) {
 38		dev_warn(&GET_DEV(accel_dev),
 39			 "HB polling too frequent. Configured HB timer %d ms\n",
 40			 accel_dev->heartbeat->hb_timer);
 41		return -EINVAL;
 42	}
 43
 44	accel_dev->heartbeat->last_hb_check_time = curr_time;
 45	return 0;
 46}
 47
 48/**
 49 * validate_hb_ctrs_cnt() - checks if the number of heartbeat counters should
 50 * be updated by one to support the currently loaded firmware.
 51 * @accel_dev: Pointer to acceleration device.
 52 *
 53 * Return:
 54 * * true - hb_ctrs must increased by ADF_NUM_PKE_STRAND
 55 * * false - no changes needed
 56 */
 57static bool validate_hb_ctrs_cnt(struct adf_accel_dev *accel_dev)
 58{
 59	const size_t hb_ctrs = accel_dev->hw_device->num_hb_ctrs;
 60	const size_t max_aes = accel_dev->hw_device->num_engines;
 61	const size_t hb_struct_size = sizeof(struct hb_cnt_pair);
 62	const size_t exp_diff_size = array3_size(ADF_NUM_PKE_STRAND, max_aes,
 63						 hb_struct_size);
 64	const size_t dev_ctrs = size_mul(max_aes, hb_ctrs);
 65	const size_t stats_size = size_mul(dev_ctrs, hb_struct_size);
 66	const u32 exp_diff_cnt = exp_diff_size / sizeof(u32);
 67	const u32 stats_el_cnt = stats_size / sizeof(u32);
 68	struct hb_cnt_pair *hb_stats = accel_dev->heartbeat->dma.virt_addr;
 69	const u32 *mem_to_chk = (u32 *)(hb_stats + dev_ctrs);
 70	u32 el_diff_cnt = 0;
 71	int i;
 72
 73	/* count how many bytes are different from pattern */
 74	for (i = 0; i < stats_el_cnt; i++) {
 75		if (mem_to_chk[i] == ADF_HB_EMPTY_SIG)
 76			break;
 77
 78		el_diff_cnt++;
 79	}
 80
 81	return el_diff_cnt && el_diff_cnt == exp_diff_cnt;
 82}
 83
 84void adf_heartbeat_check_ctrs(struct adf_accel_dev *accel_dev)
 85{
 86	struct hb_cnt_pair *hb_stats = accel_dev->heartbeat->dma.virt_addr;
 87	const size_t hb_ctrs = accel_dev->hw_device->num_hb_ctrs;
 88	const size_t max_aes = accel_dev->hw_device->num_engines;
 89	const size_t dev_ctrs = size_mul(max_aes, hb_ctrs);
 90	const size_t stats_size = size_mul(dev_ctrs, sizeof(struct hb_cnt_pair));
 91	const size_t mem_items_to_fill = size_mul(stats_size, 2) / sizeof(u32);
 92
 93	/* fill hb stats memory with pattern */
 94	memset32((uint32_t *)hb_stats, ADF_HB_EMPTY_SIG, mem_items_to_fill);
 95	accel_dev->heartbeat->ctrs_cnt_checked = false;
 96}
 97EXPORT_SYMBOL_GPL(adf_heartbeat_check_ctrs);
 98
 99static int get_timer_ticks(struct adf_accel_dev *accel_dev, unsigned int *value)
100{
101	char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { };
102	u32 timer_ms = ADF_CFG_HB_TIMER_DEFAULT_MS;
103	int cfg_read_status;
104	u32 ticks;
105	int ret;
106
107	cfg_read_status = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC,
108						  ADF_HEARTBEAT_TIMER, timer_str);
109	if (cfg_read_status == 0) {
110		if (kstrtouint(timer_str, 10, &timer_ms))
111			dev_dbg(&GET_DEV(accel_dev),
112				"kstrtouint failed to parse the %s, param value",
113				ADF_HEARTBEAT_TIMER);
114	}
115
116	if (timer_ms < ADF_CFG_HB_TIMER_MIN_MS) {
117		dev_err(&GET_DEV(accel_dev), "Timer cannot be less than %u\n",
118			ADF_CFG_HB_TIMER_MIN_MS);
119		return -EINVAL;
120	}
121
122	/*
123	 * On 4xxx devices adf_timer is responsible for HB updates and
124	 * its period is fixed to 200ms
125	 */
126	if (accel_dev->timer)
127		timer_ms = ADF_CFG_HB_TIMER_MIN_MS;
128
129	ret = adf_heartbeat_ms_to_ticks(accel_dev, timer_ms, &ticks);
130	if (ret)
131		return ret;
132
133	adf_heartbeat_save_cfg_param(accel_dev, timer_ms);
134
135	accel_dev->heartbeat->hb_timer = timer_ms;
136	*value = ticks;
137
138	return 0;
139}
140
141static int check_ae(struct hb_cnt_pair *curr, struct hb_cnt_pair *prev,
142		    u16 *count, const size_t hb_ctrs)
143{
144	size_t thr;
145
146	/* loop through all threads in AE */
147	for (thr = 0; thr < hb_ctrs; thr++) {
148		u16 req = curr[thr].req_heartbeat_cnt;
149		u16 resp = curr[thr].resp_heartbeat_cnt;
150		u16 last = prev[thr].resp_heartbeat_cnt;
151
152		if ((thr == ADF_AE_ADMIN_THREAD || req != resp) && resp == last) {
153			u16 retry = ++count[thr];
154
155			if (retry >= ADF_CFG_HB_COUNT_THRESHOLD)
156				return -EIO;
157
158		} else {
159			count[thr] = 0;
160		}
161	}
162	return 0;
163}
164
165static int adf_hb_get_status(struct adf_accel_dev *accel_dev)
166{
167	struct adf_hw_device_data *hw_device = accel_dev->hw_device;
168	struct hb_cnt_pair *live_stats, *last_stats, *curr_stats;
169	const size_t hb_ctrs = hw_device->num_hb_ctrs;
170	const unsigned long ae_mask = hw_device->ae_mask;
171	const size_t max_aes = hw_device->num_engines;
172	const size_t dev_ctrs = size_mul(max_aes, hb_ctrs);
173	const size_t stats_size = size_mul(dev_ctrs, sizeof(*curr_stats));
174	struct hb_cnt_pair *ae_curr_p, *ae_prev_p;
175	u16 *count_fails, *ae_count_p;
176	size_t ae_offset;
177	size_t ae = 0;
178	int ret = 0;
179
180	if (!accel_dev->heartbeat->ctrs_cnt_checked) {
181		if (validate_hb_ctrs_cnt(accel_dev))
182			hw_device->num_hb_ctrs += ADF_NUM_PKE_STRAND;
183
184		accel_dev->heartbeat->ctrs_cnt_checked = true;
185	}
186
187	live_stats = accel_dev->heartbeat->dma.virt_addr;
188	last_stats = live_stats + dev_ctrs;
189	count_fails = (u16 *)(last_stats + dev_ctrs);
190
191	curr_stats = kmemdup(live_stats, stats_size, GFP_KERNEL);
192	if (!curr_stats)
193		return -ENOMEM;
194
195	/* loop through active AEs */
196	for_each_set_bit(ae, &ae_mask, max_aes) {
197		ae_offset = size_mul(ae, hb_ctrs);
198		ae_curr_p = curr_stats + ae_offset;
199		ae_prev_p = last_stats + ae_offset;
200		ae_count_p = count_fails + ae_offset;
201
202		ret = check_ae(ae_curr_p, ae_prev_p, ae_count_p, hb_ctrs);
203		if (ret)
204			break;
205	}
206
207	/* Copy current stats for the next iteration */
208	memcpy(last_stats, curr_stats, stats_size);
209	kfree(curr_stats);
210
211	return ret;
212}
213
214void adf_heartbeat_status(struct adf_accel_dev *accel_dev,
215			  enum adf_device_heartbeat_status *hb_status)
216{
217	struct adf_heartbeat *hb;
218
219	if (!adf_dev_started(accel_dev) ||
220	    test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) {
221		*hb_status = HB_DEV_UNRESPONSIVE;
222		return;
223	}
224
225	if (adf_hb_check_polling_freq(accel_dev) == -EINVAL) {
226		*hb_status = HB_DEV_UNSUPPORTED;
227		return;
228	}
229
230	hb = accel_dev->heartbeat;
231	hb->hb_sent_counter++;
232
233	if (adf_hb_get_status(accel_dev)) {
234		dev_err(&GET_DEV(accel_dev),
235			"Heartbeat ERROR: QAT is not responding.\n");
236		*hb_status = HB_DEV_UNRESPONSIVE;
237		hb->hb_failed_counter++;
238		return;
239	}
240
241	*hb_status = HB_DEV_ALIVE;
242}
243
244int adf_heartbeat_ms_to_ticks(struct adf_accel_dev *accel_dev, unsigned int time_ms,
245			      u32 *value)
246{
247	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
248	u32 clk_per_sec;
249
250	/* HB clock may be different than AE clock */
251	if (!hw_data->get_hb_clock)
252		return -EINVAL;
253
254	clk_per_sec = hw_data->get_hb_clock(hw_data);
255	*value = time_ms * (clk_per_sec / MSEC_PER_SEC);
256
257	return 0;
258}
259
260int adf_heartbeat_save_cfg_param(struct adf_accel_dev *accel_dev,
261				 unsigned int timer_ms)
262{
263	char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
264
265	snprintf(timer_str, sizeof(timer_str), "%u", timer_ms);
266	return adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC,
267					  ADF_HEARTBEAT_TIMER, timer_str,
268					  ADF_STR);
269}
270EXPORT_SYMBOL_GPL(adf_heartbeat_save_cfg_param);
271
272int adf_heartbeat_init(struct adf_accel_dev *accel_dev)
273{
274	struct adf_heartbeat *hb;
275
276	hb = kzalloc(sizeof(*hb), GFP_KERNEL);
277	if (!hb)
278		goto err_ret;
279
280	hb->dma.virt_addr = dma_alloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE,
281					       &hb->dma.phy_addr, GFP_KERNEL);
282	if (!hb->dma.virt_addr)
283		goto err_free;
284
285	/*
286	 * Default set this flag as true to avoid unnecessary checks,
287	 * it will be reset on platforms that need such a check
288	 */
289	hb->ctrs_cnt_checked = true;
290	accel_dev->heartbeat = hb;
291
292	return 0;
293
294err_free:
295	kfree(hb);
296err_ret:
297	return -ENOMEM;
298}
299
300int adf_heartbeat_start(struct adf_accel_dev *accel_dev)
301{
302	unsigned int timer_ticks;
303	int ret;
304
305	if (!accel_dev->heartbeat) {
306		dev_warn(&GET_DEV(accel_dev), "Heartbeat instance not found!");
307		return -EFAULT;
308	}
309
310	if (accel_dev->hw_device->check_hb_ctrs)
311		accel_dev->hw_device->check_hb_ctrs(accel_dev);
312
313	ret = get_timer_ticks(accel_dev, &timer_ticks);
314	if (ret)
315		return ret;
316
317	ret = adf_send_admin_hb_timer(accel_dev, timer_ticks);
318	if (ret)
319		dev_warn(&GET_DEV(accel_dev), "Heartbeat not supported!");
320
321	return ret;
322}
323
324void adf_heartbeat_shutdown(struct adf_accel_dev *accel_dev)
325{
326	struct adf_heartbeat *hb = accel_dev->heartbeat;
327
328	if (!hb)
329		return;
330
331	if (hb->dma.virt_addr)
332		dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE,
333				  hb->dma.virt_addr, hb->dma.phy_addr);
334
335	kfree(hb);
336	accel_dev->heartbeat = NULL;
337}