Loading...
Note: File does not exist in v6.2.
1/*
2 * Intel SOC Telemetry Platform Driver: Currently supports APL
3 * Copyright (c) 2015, Intel Corporation.
4 * All Rights Reserved.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
15 * This file provides the platform specific telemetry implementation for APL.
16 * It used the PUNIT and PMC IPC interfaces for configuring the counters.
17 * The accumulated results are fetched from SRAM.
18 */
19
20#include <linux/io.h>
21#include <linux/module.h>
22#include <linux/platform_device.h>
23
24#include <asm/cpu_device_id.h>
25#include <asm/intel-family.h>
26#include <asm/intel_pmc_ipc.h>
27#include <asm/intel_punit_ipc.h>
28#include <asm/intel_telemetry.h>
29
30#define DRIVER_NAME "intel_telemetry"
31#define DRIVER_VERSION "1.0.0"
32
33#define TELEM_TRC_VERBOSITY_MASK 0x3
34
35#define TELEM_MIN_PERIOD(x) ((x) & 0x7F0000)
36#define TELEM_MAX_PERIOD(x) ((x) & 0x7F000000)
37#define TELEM_SAMPLE_PERIOD_INVALID(x) ((x) & (BIT(7)))
38#define TELEM_CLEAR_SAMPLE_PERIOD(x) ((x) &= ~0x7F)
39
40#define TELEM_SAMPLING_DEFAULT_PERIOD 0xD
41
42#define TELEM_MAX_EVENTS_SRAM 28
43#define TELEM_SSRAM_STARTTIME_OFFSET 8
44#define TELEM_SSRAM_EVTLOG_OFFSET 16
45
46#define IOSS_TELEM_EVENT_READ 0x0
47#define IOSS_TELEM_EVENT_WRITE 0x1
48#define IOSS_TELEM_INFO_READ 0x2
49#define IOSS_TELEM_TRACE_CTL_READ 0x5
50#define IOSS_TELEM_TRACE_CTL_WRITE 0x6
51#define IOSS_TELEM_EVENT_CTL_READ 0x7
52#define IOSS_TELEM_EVENT_CTL_WRITE 0x8
53#define IOSS_TELEM_EVT_CTRL_WRITE_SIZE 0x4
54#define IOSS_TELEM_READ_WORD 0x1
55#define IOSS_TELEM_WRITE_FOURBYTES 0x4
56#define IOSS_TELEM_EVT_WRITE_SIZE 0x3
57
58#define TELEM_INFO_SRAMEVTS_MASK 0xFF00
59#define TELEM_INFO_SRAMEVTS_SHIFT 0x8
60#define TELEM_SSRAM_READ_TIMEOUT 10
61
62#define TELEM_INFO_NENABLES_MASK 0xFF
63#define TELEM_EVENT_ENABLE 0x8000
64
65#define TELEM_MASK_BIT 1
66#define TELEM_MASK_BYTE 0xFF
67#define BYTES_PER_LONG 8
68#define TELEM_MASK_PCS_STATE 0xF
69
70#define TELEM_DISABLE(x) ((x) &= ~(BIT(31)))
71#define TELEM_CLEAR_EVENTS(x) ((x) |= (BIT(30)))
72#define TELEM_ENABLE_SRAM_EVT_TRACE(x) ((x) &= ~(BIT(30) | BIT(24)))
73#define TELEM_ENABLE_PERIODIC(x) ((x) |= (BIT(23) | BIT(31) | BIT(7)))
74#define TELEM_EXTRACT_VERBOSITY(x, y) ((y) = (((x) >> 27) & 0x3))
75#define TELEM_CLEAR_VERBOSITY_BITS(x) ((x) &= ~(BIT(27) | BIT(28)))
76#define TELEM_SET_VERBOSITY_BITS(x, y) ((x) |= ((y) << 27))
77
78#define TELEM_CPU(model, data) \
79 { X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, (unsigned long)&data }
80
81enum telemetry_action {
82 TELEM_UPDATE = 0,
83 TELEM_ADD,
84 TELEM_RESET,
85 TELEM_ACTION_NONE
86};
87
88struct telem_ssram_region {
89 u64 timestamp;
90 u64 start_time;
91 u64 events[TELEM_MAX_EVENTS_SRAM];
92};
93
94static struct telemetry_plt_config *telm_conf;
95
96/*
97 * The following counters are programmed by default during setup.
98 * Only 20 allocated to kernel driver
99 */
100static struct telemetry_evtmap
101 telemetry_apl_ioss_default_events[TELEM_MAX_OS_ALLOCATED_EVENTS] = {
102 {"SOC_S0IX_TOTAL_RES", 0x4800},
103 {"SOC_S0IX_TOTAL_OCC", 0x4000},
104 {"SOC_S0IX_SHALLOW_RES", 0x4801},
105 {"SOC_S0IX_SHALLOW_OCC", 0x4001},
106 {"SOC_S0IX_DEEP_RES", 0x4802},
107 {"SOC_S0IX_DEEP_OCC", 0x4002},
108 {"PMC_POWER_GATE", 0x5818},
109 {"PMC_D3_STATES", 0x5819},
110 {"PMC_D0I3_STATES", 0x581A},
111 {"PMC_S0IX_WAKE_REASON_GPIO", 0x6000},
112 {"PMC_S0IX_WAKE_REASON_TIMER", 0x6001},
113 {"PMC_S0IX_WAKE_REASON_VNNREQ", 0x6002},
114 {"PMC_S0IX_WAKE_REASON_LOWPOWER", 0x6003},
115 {"PMC_S0IX_WAKE_REASON_EXTERNAL", 0x6004},
116 {"PMC_S0IX_WAKE_REASON_MISC", 0x6005},
117 {"PMC_S0IX_BLOCKING_IPS_D3_D0I3", 0x6006},
118 {"PMC_S0IX_BLOCKING_IPS_PG", 0x6007},
119 {"PMC_S0IX_BLOCKING_MISC_IPS_PG", 0x6008},
120 {"PMC_S0IX_BLOCK_IPS_VNN_REQ", 0x6009},
121 {"PMC_S0IX_BLOCK_IPS_CLOCKS", 0x600B},
122};
123
124
125static struct telemetry_evtmap
126 telemetry_apl_pss_default_events[TELEM_MAX_OS_ALLOCATED_EVENTS] = {
127 {"IA_CORE0_C6_RES", 0x0400},
128 {"IA_CORE0_C6_CTR", 0x0000},
129 {"IA_MODULE0_C7_RES", 0x0410},
130 {"IA_MODULE0_C7_CTR", 0x000E},
131 {"IA_C0_RES", 0x0805},
132 {"PCS_LTR", 0x2801},
133 {"PSTATES", 0x2802},
134 {"SOC_S0I3_RES", 0x0409},
135 {"SOC_S0I3_CTR", 0x000A},
136 {"PCS_S0I3_CTR", 0x0009},
137 {"PCS_C1E_RES", 0x041A},
138 {"PCS_IDLE_STATUS", 0x2806},
139 {"IA_PERF_LIMITS", 0x280B},
140 {"GT_PERF_LIMITS", 0x280C},
141 {"PCS_WAKEUP_S0IX_CTR", 0x0030},
142 {"PCS_IDLE_BLOCKED", 0x2C00},
143 {"PCS_S0IX_BLOCKED", 0x2C01},
144 {"PCS_S0IX_WAKE_REASONS", 0x2C02},
145 {"PCS_LTR_BLOCKING", 0x2C03},
146 {"PC2_AND_MEM_SHALLOW_IDLE_RES", 0x1D40},
147};
148
149static struct telemetry_evtmap
150 telemetry_glk_pss_default_events[TELEM_MAX_OS_ALLOCATED_EVENTS] = {
151 {"IA_CORE0_C6_RES", 0x0400},
152 {"IA_CORE0_C6_CTR", 0x0000},
153 {"IA_MODULE0_C7_RES", 0x0410},
154 {"IA_MODULE0_C7_CTR", 0x000C},
155 {"IA_C0_RES", 0x0805},
156 {"PCS_LTR", 0x2801},
157 {"PSTATES", 0x2802},
158 {"SOC_S0I3_RES", 0x0407},
159 {"SOC_S0I3_CTR", 0x0008},
160 {"PCS_S0I3_CTR", 0x0007},
161 {"PCS_C1E_RES", 0x0414},
162 {"PCS_IDLE_STATUS", 0x2806},
163 {"IA_PERF_LIMITS", 0x280B},
164 {"GT_PERF_LIMITS", 0x280C},
165 {"PCS_WAKEUP_S0IX_CTR", 0x0025},
166 {"PCS_IDLE_BLOCKED", 0x2C00},
167 {"PCS_S0IX_BLOCKED", 0x2C01},
168 {"PCS_S0IX_WAKE_REASONS", 0x2C02},
169 {"PCS_LTR_BLOCKING", 0x2C03},
170 {"PC2_AND_MEM_SHALLOW_IDLE_RES", 0x1D40},
171};
172
173/* APL specific Data */
174static struct telemetry_plt_config telem_apl_config = {
175 .pss_config = {
176 .telem_evts = telemetry_apl_pss_default_events,
177 },
178 .ioss_config = {
179 .telem_evts = telemetry_apl_ioss_default_events,
180 },
181};
182
183/* GLK specific Data */
184static struct telemetry_plt_config telem_glk_config = {
185 .pss_config = {
186 .telem_evts = telemetry_glk_pss_default_events,
187 },
188 .ioss_config = {
189 .telem_evts = telemetry_apl_ioss_default_events,
190 },
191};
192
193static const struct x86_cpu_id telemetry_cpu_ids[] = {
194 TELEM_CPU(INTEL_FAM6_ATOM_GOLDMONT, telem_apl_config),
195 TELEM_CPU(INTEL_FAM6_ATOM_GEMINI_LAKE, telem_glk_config),
196 {}
197};
198
199MODULE_DEVICE_TABLE(x86cpu, telemetry_cpu_ids);
200
201static inline int telem_get_unitconfig(enum telemetry_unit telem_unit,
202 struct telemetry_unit_config **unit_config)
203{
204 if (telem_unit == TELEM_PSS)
205 *unit_config = &(telm_conf->pss_config);
206 else if (telem_unit == TELEM_IOSS)
207 *unit_config = &(telm_conf->ioss_config);
208 else
209 return -EINVAL;
210
211 return 0;
212
213}
214
215static int telemetry_check_evtid(enum telemetry_unit telem_unit,
216 u32 *evtmap, u8 len,
217 enum telemetry_action action)
218{
219 struct telemetry_unit_config *unit_config;
220 int ret;
221
222 ret = telem_get_unitconfig(telem_unit, &unit_config);
223 if (ret < 0)
224 return ret;
225
226 switch (action) {
227 case TELEM_RESET:
228 if (len > TELEM_MAX_EVENTS_SRAM)
229 return -EINVAL;
230
231 break;
232
233 case TELEM_UPDATE:
234 if (len > TELEM_MAX_EVENTS_SRAM)
235 return -EINVAL;
236
237 if ((len > 0) && (evtmap == NULL))
238 return -EINVAL;
239
240 break;
241
242 case TELEM_ADD:
243 if ((len + unit_config->ssram_evts_used) >
244 TELEM_MAX_EVENTS_SRAM)
245 return -EINVAL;
246
247 if ((len > 0) && (evtmap == NULL))
248 return -EINVAL;
249
250 break;
251
252 default:
253 pr_err("Unknown Telemetry action specified %d\n", action);
254 return -EINVAL;
255 }
256
257 return 0;
258}
259
260
261static inline int telemetry_plt_config_ioss_event(u32 evt_id, int index)
262{
263 u32 write_buf;
264 int ret;
265
266 write_buf = evt_id | TELEM_EVENT_ENABLE;
267 write_buf <<= BITS_PER_BYTE;
268 write_buf |= index;
269
270 ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
271 IOSS_TELEM_EVENT_WRITE, (u8 *)&write_buf,
272 IOSS_TELEM_EVT_WRITE_SIZE, NULL, 0);
273
274 return ret;
275}
276
277static inline int telemetry_plt_config_pss_event(u32 evt_id, int index)
278{
279 u32 write_buf;
280 int ret;
281
282 write_buf = evt_id | TELEM_EVENT_ENABLE;
283 ret = intel_punit_ipc_command(IPC_PUNIT_BIOS_WRITE_TELE_EVENT,
284 index, 0, &write_buf, NULL);
285
286 return ret;
287}
288
289static int telemetry_setup_iossevtconfig(struct telemetry_evtconfig evtconfig,
290 enum telemetry_action action)
291{
292 u8 num_ioss_evts, ioss_period;
293 int ret, index, idx;
294 u32 *ioss_evtmap;
295 u32 telem_ctrl;
296
297 num_ioss_evts = evtconfig.num_evts;
298 ioss_period = evtconfig.period;
299 ioss_evtmap = evtconfig.evtmap;
300
301 /* Get telemetry EVENT CTL */
302 ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
303 IOSS_TELEM_EVENT_CTL_READ, NULL, 0,
304 &telem_ctrl, IOSS_TELEM_READ_WORD);
305 if (ret) {
306 pr_err("IOSS TELEM_CTRL Read Failed\n");
307 return ret;
308 }
309
310 /* Disable Telemetry */
311 TELEM_DISABLE(telem_ctrl);
312
313 ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
314 IOSS_TELEM_EVENT_CTL_WRITE,
315 (u8 *)&telem_ctrl,
316 IOSS_TELEM_EVT_CTRL_WRITE_SIZE,
317 NULL, 0);
318 if (ret) {
319 pr_err("IOSS TELEM_CTRL Event Disable Write Failed\n");
320 return ret;
321 }
322
323
324 /* Reset Everything */
325 if (action == TELEM_RESET) {
326 /* Clear All Events */
327 TELEM_CLEAR_EVENTS(telem_ctrl);
328
329 ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
330 IOSS_TELEM_EVENT_CTL_WRITE,
331 (u8 *)&telem_ctrl,
332 IOSS_TELEM_EVT_CTRL_WRITE_SIZE,
333 NULL, 0);
334 if (ret) {
335 pr_err("IOSS TELEM_CTRL Event Disable Write Failed\n");
336 return ret;
337 }
338 telm_conf->ioss_config.ssram_evts_used = 0;
339
340 /* Configure Events */
341 for (idx = 0; idx < num_ioss_evts; idx++) {
342 if (telemetry_plt_config_ioss_event(
343 telm_conf->ioss_config.telem_evts[idx].evt_id,
344 idx)) {
345 pr_err("IOSS TELEM_RESET Fail for data: %x\n",
346 telm_conf->ioss_config.telem_evts[idx].evt_id);
347 continue;
348 }
349 telm_conf->ioss_config.ssram_evts_used++;
350 }
351 }
352
353 /* Re-Configure Everything */
354 if (action == TELEM_UPDATE) {
355 /* Clear All Events */
356 TELEM_CLEAR_EVENTS(telem_ctrl);
357
358 ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
359 IOSS_TELEM_EVENT_CTL_WRITE,
360 (u8 *)&telem_ctrl,
361 IOSS_TELEM_EVT_CTRL_WRITE_SIZE,
362 NULL, 0);
363 if (ret) {
364 pr_err("IOSS TELEM_CTRL Event Disable Write Failed\n");
365 return ret;
366 }
367 telm_conf->ioss_config.ssram_evts_used = 0;
368
369 /* Configure Events */
370 for (index = 0; index < num_ioss_evts; index++) {
371 telm_conf->ioss_config.telem_evts[index].evt_id =
372 ioss_evtmap[index];
373
374 if (telemetry_plt_config_ioss_event(
375 telm_conf->ioss_config.telem_evts[index].evt_id,
376 index)) {
377 pr_err("IOSS TELEM_UPDATE Fail for Evt%x\n",
378 ioss_evtmap[index]);
379 continue;
380 }
381 telm_conf->ioss_config.ssram_evts_used++;
382 }
383 }
384
385 /* Add some Events */
386 if (action == TELEM_ADD) {
387 /* Configure Events */
388 for (index = telm_conf->ioss_config.ssram_evts_used, idx = 0;
389 idx < num_ioss_evts; index++, idx++) {
390 telm_conf->ioss_config.telem_evts[index].evt_id =
391 ioss_evtmap[idx];
392
393 if (telemetry_plt_config_ioss_event(
394 telm_conf->ioss_config.telem_evts[index].evt_id,
395 index)) {
396 pr_err("IOSS TELEM_ADD Fail for Event %x\n",
397 ioss_evtmap[idx]);
398 continue;
399 }
400 telm_conf->ioss_config.ssram_evts_used++;
401 }
402 }
403
404 /* Enable Periodic Telemetry Events and enable SRAM trace */
405 TELEM_CLEAR_SAMPLE_PERIOD(telem_ctrl);
406 TELEM_ENABLE_SRAM_EVT_TRACE(telem_ctrl);
407 TELEM_ENABLE_PERIODIC(telem_ctrl);
408 telem_ctrl |= ioss_period;
409
410 ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
411 IOSS_TELEM_EVENT_CTL_WRITE,
412 (u8 *)&telem_ctrl,
413 IOSS_TELEM_EVT_CTRL_WRITE_SIZE, NULL, 0);
414 if (ret) {
415 pr_err("IOSS TELEM_CTRL Event Enable Write Failed\n");
416 return ret;
417 }
418
419 telm_conf->ioss_config.curr_period = ioss_period;
420
421 return 0;
422}
423
424
425static int telemetry_setup_pssevtconfig(struct telemetry_evtconfig evtconfig,
426 enum telemetry_action action)
427{
428 u8 num_pss_evts, pss_period;
429 int ret, index, idx;
430 u32 *pss_evtmap;
431 u32 telem_ctrl;
432
433 num_pss_evts = evtconfig.num_evts;
434 pss_period = evtconfig.period;
435 pss_evtmap = evtconfig.evtmap;
436
437 /* PSS Config */
438 /* Get telemetry EVENT CTL */
439 ret = intel_punit_ipc_command(IPC_PUNIT_BIOS_READ_TELE_EVENT_CTRL,
440 0, 0, NULL, &telem_ctrl);
441 if (ret) {
442 pr_err("PSS TELEM_CTRL Read Failed\n");
443 return ret;
444 }
445
446 /* Disable Telemetry */
447 TELEM_DISABLE(telem_ctrl);
448 ret = intel_punit_ipc_command(IPC_PUNIT_BIOS_WRITE_TELE_EVENT_CTRL,
449 0, 0, &telem_ctrl, NULL);
450 if (ret) {
451 pr_err("PSS TELEM_CTRL Event Disable Write Failed\n");
452 return ret;
453 }
454
455 /* Reset Everything */
456 if (action == TELEM_RESET) {
457 /* Clear All Events */
458 TELEM_CLEAR_EVENTS(telem_ctrl);
459
460 ret = intel_punit_ipc_command(
461 IPC_PUNIT_BIOS_WRITE_TELE_EVENT_CTRL,
462 0, 0, &telem_ctrl, NULL);
463 if (ret) {
464 pr_err("PSS TELEM_CTRL Event Disable Write Failed\n");
465 return ret;
466 }
467 telm_conf->pss_config.ssram_evts_used = 0;
468 /* Configure Events */
469 for (idx = 0; idx < num_pss_evts; idx++) {
470 if (telemetry_plt_config_pss_event(
471 telm_conf->pss_config.telem_evts[idx].evt_id,
472 idx)) {
473 pr_err("PSS TELEM_RESET Fail for Event %x\n",
474 telm_conf->pss_config.telem_evts[idx].evt_id);
475 continue;
476 }
477 telm_conf->pss_config.ssram_evts_used++;
478 }
479 }
480
481 /* Re-Configure Everything */
482 if (action == TELEM_UPDATE) {
483 /* Clear All Events */
484 TELEM_CLEAR_EVENTS(telem_ctrl);
485
486 ret = intel_punit_ipc_command(
487 IPC_PUNIT_BIOS_WRITE_TELE_EVENT_CTRL,
488 0, 0, &telem_ctrl, NULL);
489 if (ret) {
490 pr_err("PSS TELEM_CTRL Event Disable Write Failed\n");
491 return ret;
492 }
493 telm_conf->pss_config.ssram_evts_used = 0;
494
495 /* Configure Events */
496 for (index = 0; index < num_pss_evts; index++) {
497 telm_conf->pss_config.telem_evts[index].evt_id =
498 pss_evtmap[index];
499
500 if (telemetry_plt_config_pss_event(
501 telm_conf->pss_config.telem_evts[index].evt_id,
502 index)) {
503 pr_err("PSS TELEM_UPDATE Fail for Event %x\n",
504 pss_evtmap[index]);
505 continue;
506 }
507 telm_conf->pss_config.ssram_evts_used++;
508 }
509 }
510
511 /* Add some Events */
512 if (action == TELEM_ADD) {
513 /* Configure Events */
514 for (index = telm_conf->pss_config.ssram_evts_used, idx = 0;
515 idx < num_pss_evts; index++, idx++) {
516
517 telm_conf->pss_config.telem_evts[index].evt_id =
518 pss_evtmap[idx];
519
520 if (telemetry_plt_config_pss_event(
521 telm_conf->pss_config.telem_evts[index].evt_id,
522 index)) {
523 pr_err("PSS TELEM_ADD Fail for Event %x\n",
524 pss_evtmap[idx]);
525 continue;
526 }
527 telm_conf->pss_config.ssram_evts_used++;
528 }
529 }
530
531 /* Enable Periodic Telemetry Events and enable SRAM trace */
532 TELEM_CLEAR_SAMPLE_PERIOD(telem_ctrl);
533 TELEM_ENABLE_SRAM_EVT_TRACE(telem_ctrl);
534 TELEM_ENABLE_PERIODIC(telem_ctrl);
535 telem_ctrl |= pss_period;
536
537 ret = intel_punit_ipc_command(IPC_PUNIT_BIOS_WRITE_TELE_EVENT_CTRL,
538 0, 0, &telem_ctrl, NULL);
539 if (ret) {
540 pr_err("PSS TELEM_CTRL Event Enable Write Failed\n");
541 return ret;
542 }
543
544 telm_conf->pss_config.curr_period = pss_period;
545
546 return 0;
547}
548
549static int telemetry_setup_evtconfig(struct telemetry_evtconfig pss_evtconfig,
550 struct telemetry_evtconfig ioss_evtconfig,
551 enum telemetry_action action)
552{
553 int ret;
554
555 mutex_lock(&(telm_conf->telem_lock));
556
557 if ((action == TELEM_UPDATE) && (telm_conf->telem_in_use)) {
558 ret = -EBUSY;
559 goto out;
560 }
561
562 ret = telemetry_check_evtid(TELEM_PSS, pss_evtconfig.evtmap,
563 pss_evtconfig.num_evts, action);
564 if (ret)
565 goto out;
566
567 ret = telemetry_check_evtid(TELEM_IOSS, ioss_evtconfig.evtmap,
568 ioss_evtconfig.num_evts, action);
569 if (ret)
570 goto out;
571
572 if (ioss_evtconfig.num_evts) {
573 ret = telemetry_setup_iossevtconfig(ioss_evtconfig, action);
574 if (ret)
575 goto out;
576 }
577
578 if (pss_evtconfig.num_evts) {
579 ret = telemetry_setup_pssevtconfig(pss_evtconfig, action);
580 if (ret)
581 goto out;
582 }
583
584 if ((action == TELEM_UPDATE) || (action == TELEM_ADD))
585 telm_conf->telem_in_use = true;
586 else
587 telm_conf->telem_in_use = false;
588
589out:
590 mutex_unlock(&(telm_conf->telem_lock));
591 return ret;
592}
593
594static int telemetry_setup(struct platform_device *pdev)
595{
596 struct telemetry_evtconfig pss_evtconfig, ioss_evtconfig;
597 u32 read_buf, events, event_regs;
598 int ret;
599
600 ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY, IOSS_TELEM_INFO_READ,
601 NULL, 0, &read_buf, IOSS_TELEM_READ_WORD);
602 if (ret) {
603 dev_err(&pdev->dev, "IOSS TELEM_INFO Read Failed\n");
604 return ret;
605 }
606
607 /* Get telemetry Info */
608 events = (read_buf & TELEM_INFO_SRAMEVTS_MASK) >>
609 TELEM_INFO_SRAMEVTS_SHIFT;
610 event_regs = read_buf & TELEM_INFO_NENABLES_MASK;
611 if ((events < TELEM_MAX_EVENTS_SRAM) ||
612 (event_regs < TELEM_MAX_EVENTS_SRAM)) {
613 dev_err(&pdev->dev, "IOSS:Insufficient Space for SRAM Trace\n");
614 dev_err(&pdev->dev, "SRAM Events %d; Event Regs %d\n",
615 events, event_regs);
616 return -ENOMEM;
617 }
618
619 telm_conf->ioss_config.min_period = TELEM_MIN_PERIOD(read_buf);
620 telm_conf->ioss_config.max_period = TELEM_MAX_PERIOD(read_buf);
621
622 /* PUNIT Mailbox Setup */
623 ret = intel_punit_ipc_command(IPC_PUNIT_BIOS_READ_TELE_INFO, 0, 0,
624 NULL, &read_buf);
625 if (ret) {
626 dev_err(&pdev->dev, "PSS TELEM_INFO Read Failed\n");
627 return ret;
628 }
629
630 /* Get telemetry Info */
631 events = (read_buf & TELEM_INFO_SRAMEVTS_MASK) >>
632 TELEM_INFO_SRAMEVTS_SHIFT;
633 event_regs = read_buf & TELEM_INFO_SRAMEVTS_MASK;
634 if ((events < TELEM_MAX_EVENTS_SRAM) ||
635 (event_regs < TELEM_MAX_EVENTS_SRAM)) {
636 dev_err(&pdev->dev, "PSS:Insufficient Space for SRAM Trace\n");
637 dev_err(&pdev->dev, "SRAM Events %d; Event Regs %d\n",
638 events, event_regs);
639 return -ENOMEM;
640 }
641
642 telm_conf->pss_config.min_period = TELEM_MIN_PERIOD(read_buf);
643 telm_conf->pss_config.max_period = TELEM_MAX_PERIOD(read_buf);
644
645 pss_evtconfig.evtmap = NULL;
646 pss_evtconfig.num_evts = TELEM_MAX_OS_ALLOCATED_EVENTS;
647 pss_evtconfig.period = TELEM_SAMPLING_DEFAULT_PERIOD;
648
649 ioss_evtconfig.evtmap = NULL;
650 ioss_evtconfig.num_evts = TELEM_MAX_OS_ALLOCATED_EVENTS;
651 ioss_evtconfig.period = TELEM_SAMPLING_DEFAULT_PERIOD;
652
653 ret = telemetry_setup_evtconfig(pss_evtconfig, ioss_evtconfig,
654 TELEM_RESET);
655 if (ret) {
656 dev_err(&pdev->dev, "TELEMETRY Setup Failed\n");
657 return ret;
658 }
659 return 0;
660}
661
662static int telemetry_plt_update_events(struct telemetry_evtconfig pss_evtconfig,
663 struct telemetry_evtconfig ioss_evtconfig)
664{
665 int ret;
666
667 if ((pss_evtconfig.num_evts > 0) &&
668 (TELEM_SAMPLE_PERIOD_INVALID(pss_evtconfig.period))) {
669 pr_err("PSS Sampling Period Out of Range\n");
670 return -EINVAL;
671 }
672
673 if ((ioss_evtconfig.num_evts > 0) &&
674 (TELEM_SAMPLE_PERIOD_INVALID(ioss_evtconfig.period))) {
675 pr_err("IOSS Sampling Period Out of Range\n");
676 return -EINVAL;
677 }
678
679 ret = telemetry_setup_evtconfig(pss_evtconfig, ioss_evtconfig,
680 TELEM_UPDATE);
681 if (ret)
682 pr_err("TELEMETRY Config Failed\n");
683
684 return ret;
685}
686
687
688static int telemetry_plt_set_sampling_period(u8 pss_period, u8 ioss_period)
689{
690 u32 telem_ctrl = 0;
691 int ret = 0;
692
693 mutex_lock(&(telm_conf->telem_lock));
694 if (ioss_period) {
695 if (TELEM_SAMPLE_PERIOD_INVALID(ioss_period)) {
696 pr_err("IOSS Sampling Period Out of Range\n");
697 ret = -EINVAL;
698 goto out;
699 }
700
701 /* Get telemetry EVENT CTL */
702 ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
703 IOSS_TELEM_EVENT_CTL_READ, NULL, 0,
704 &telem_ctrl, IOSS_TELEM_READ_WORD);
705 if (ret) {
706 pr_err("IOSS TELEM_CTRL Read Failed\n");
707 goto out;
708 }
709
710 /* Disable Telemetry */
711 TELEM_DISABLE(telem_ctrl);
712
713 ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
714 IOSS_TELEM_EVENT_CTL_WRITE,
715 (u8 *)&telem_ctrl,
716 IOSS_TELEM_EVT_CTRL_WRITE_SIZE,
717 NULL, 0);
718 if (ret) {
719 pr_err("IOSS TELEM_CTRL Event Disable Write Failed\n");
720 goto out;
721 }
722
723 /* Enable Periodic Telemetry Events and enable SRAM trace */
724 TELEM_CLEAR_SAMPLE_PERIOD(telem_ctrl);
725 TELEM_ENABLE_SRAM_EVT_TRACE(telem_ctrl);
726 TELEM_ENABLE_PERIODIC(telem_ctrl);
727 telem_ctrl |= ioss_period;
728
729 ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
730 IOSS_TELEM_EVENT_CTL_WRITE,
731 (u8 *)&telem_ctrl,
732 IOSS_TELEM_EVT_CTRL_WRITE_SIZE,
733 NULL, 0);
734 if (ret) {
735 pr_err("IOSS TELEM_CTRL Event Enable Write Failed\n");
736 goto out;
737 }
738 telm_conf->ioss_config.curr_period = ioss_period;
739 }
740
741 if (pss_period) {
742 if (TELEM_SAMPLE_PERIOD_INVALID(pss_period)) {
743 pr_err("PSS Sampling Period Out of Range\n");
744 ret = -EINVAL;
745 goto out;
746 }
747
748 /* Get telemetry EVENT CTL */
749 ret = intel_punit_ipc_command(
750 IPC_PUNIT_BIOS_READ_TELE_EVENT_CTRL,
751 0, 0, NULL, &telem_ctrl);
752 if (ret) {
753 pr_err("PSS TELEM_CTRL Read Failed\n");
754 goto out;
755 }
756
757 /* Disable Telemetry */
758 TELEM_DISABLE(telem_ctrl);
759 ret = intel_punit_ipc_command(
760 IPC_PUNIT_BIOS_WRITE_TELE_EVENT_CTRL,
761 0, 0, &telem_ctrl, NULL);
762 if (ret) {
763 pr_err("PSS TELEM_CTRL Event Disable Write Failed\n");
764 goto out;
765 }
766
767 /* Enable Periodic Telemetry Events and enable SRAM trace */
768 TELEM_CLEAR_SAMPLE_PERIOD(telem_ctrl);
769 TELEM_ENABLE_SRAM_EVT_TRACE(telem_ctrl);
770 TELEM_ENABLE_PERIODIC(telem_ctrl);
771 telem_ctrl |= pss_period;
772
773 ret = intel_punit_ipc_command(
774 IPC_PUNIT_BIOS_WRITE_TELE_EVENT_CTRL,
775 0, 0, &telem_ctrl, NULL);
776 if (ret) {
777 pr_err("PSS TELEM_CTRL Event Enable Write Failed\n");
778 goto out;
779 }
780 telm_conf->pss_config.curr_period = pss_period;
781 }
782
783out:
784 mutex_unlock(&(telm_conf->telem_lock));
785 return ret;
786}
787
788
789static int telemetry_plt_get_sampling_period(u8 *pss_min_period,
790 u8 *pss_max_period,
791 u8 *ioss_min_period,
792 u8 *ioss_max_period)
793{
794 *pss_min_period = telm_conf->pss_config.min_period;
795 *pss_max_period = telm_conf->pss_config.max_period;
796 *ioss_min_period = telm_conf->ioss_config.min_period;
797 *ioss_max_period = telm_conf->ioss_config.max_period;
798
799 return 0;
800}
801
802
803static int telemetry_plt_reset_events(void)
804{
805 struct telemetry_evtconfig pss_evtconfig, ioss_evtconfig;
806 int ret;
807
808 pss_evtconfig.evtmap = NULL;
809 pss_evtconfig.num_evts = TELEM_MAX_OS_ALLOCATED_EVENTS;
810 pss_evtconfig.period = TELEM_SAMPLING_DEFAULT_PERIOD;
811
812 ioss_evtconfig.evtmap = NULL;
813 ioss_evtconfig.num_evts = TELEM_MAX_OS_ALLOCATED_EVENTS;
814 ioss_evtconfig.period = TELEM_SAMPLING_DEFAULT_PERIOD;
815
816 ret = telemetry_setup_evtconfig(pss_evtconfig, ioss_evtconfig,
817 TELEM_RESET);
818 if (ret)
819 pr_err("TELEMETRY Reset Failed\n");
820
821 return ret;
822}
823
824
825static int telemetry_plt_get_eventconfig(struct telemetry_evtconfig *pss_config,
826 struct telemetry_evtconfig *ioss_config,
827 int pss_len, int ioss_len)
828{
829 u32 *pss_evtmap, *ioss_evtmap;
830 u32 index;
831
832 pss_evtmap = pss_config->evtmap;
833 ioss_evtmap = ioss_config->evtmap;
834
835 mutex_lock(&(telm_conf->telem_lock));
836 pss_config->num_evts = telm_conf->pss_config.ssram_evts_used;
837 ioss_config->num_evts = telm_conf->ioss_config.ssram_evts_used;
838
839 pss_config->period = telm_conf->pss_config.curr_period;
840 ioss_config->period = telm_conf->ioss_config.curr_period;
841
842 if ((pss_len < telm_conf->pss_config.ssram_evts_used) ||
843 (ioss_len < telm_conf->ioss_config.ssram_evts_used)) {
844 mutex_unlock(&(telm_conf->telem_lock));
845 return -EINVAL;
846 }
847
848 for (index = 0; index < telm_conf->pss_config.ssram_evts_used;
849 index++) {
850 pss_evtmap[index] =
851 telm_conf->pss_config.telem_evts[index].evt_id;
852 }
853
854 for (index = 0; index < telm_conf->ioss_config.ssram_evts_used;
855 index++) {
856 ioss_evtmap[index] =
857 telm_conf->ioss_config.telem_evts[index].evt_id;
858 }
859
860 mutex_unlock(&(telm_conf->telem_lock));
861 return 0;
862}
863
864
865static int telemetry_plt_add_events(u8 num_pss_evts, u8 num_ioss_evts,
866 u32 *pss_evtmap, u32 *ioss_evtmap)
867{
868 struct telemetry_evtconfig pss_evtconfig, ioss_evtconfig;
869 int ret;
870
871 pss_evtconfig.evtmap = pss_evtmap;
872 pss_evtconfig.num_evts = num_pss_evts;
873 pss_evtconfig.period = telm_conf->pss_config.curr_period;
874
875 ioss_evtconfig.evtmap = ioss_evtmap;
876 ioss_evtconfig.num_evts = num_ioss_evts;
877 ioss_evtconfig.period = telm_conf->ioss_config.curr_period;
878
879 ret = telemetry_setup_evtconfig(pss_evtconfig, ioss_evtconfig,
880 TELEM_ADD);
881 if (ret)
882 pr_err("TELEMETRY ADD Failed\n");
883
884 return ret;
885}
886
887static int telem_evtlog_read(enum telemetry_unit telem_unit,
888 struct telem_ssram_region *ssram_region, u8 len)
889{
890 struct telemetry_unit_config *unit_config;
891 u64 timestamp_prev, timestamp_next;
892 int ret, index, timeout = 0;
893
894 ret = telem_get_unitconfig(telem_unit, &unit_config);
895 if (ret < 0)
896 return ret;
897
898 if (len > unit_config->ssram_evts_used)
899 len = unit_config->ssram_evts_used;
900
901 do {
902 timestamp_prev = readq(unit_config->regmap);
903 if (!timestamp_prev) {
904 pr_err("Ssram under update. Please Try Later\n");
905 return -EBUSY;
906 }
907
908 ssram_region->start_time = readq(unit_config->regmap +
909 TELEM_SSRAM_STARTTIME_OFFSET);
910
911 for (index = 0; index < len; index++) {
912 ssram_region->events[index] =
913 readq(unit_config->regmap + TELEM_SSRAM_EVTLOG_OFFSET +
914 BYTES_PER_LONG*index);
915 }
916
917 timestamp_next = readq(unit_config->regmap);
918 if (!timestamp_next) {
919 pr_err("Ssram under update. Please Try Later\n");
920 return -EBUSY;
921 }
922
923 if (timeout++ > TELEM_SSRAM_READ_TIMEOUT) {
924 pr_err("Timeout while reading Events\n");
925 return -EBUSY;
926 }
927
928 } while (timestamp_prev != timestamp_next);
929
930 ssram_region->timestamp = timestamp_next;
931
932 return len;
933}
934
935static int telemetry_plt_raw_read_eventlog(enum telemetry_unit telem_unit,
936 struct telemetry_evtlog *evtlog,
937 int len, int log_all_evts)
938{
939 int index, idx1, ret, readlen = len;
940 struct telem_ssram_region ssram_region;
941 struct telemetry_evtmap *evtmap;
942
943 switch (telem_unit) {
944 case TELEM_PSS:
945 evtmap = telm_conf->pss_config.telem_evts;
946 break;
947
948 case TELEM_IOSS:
949 evtmap = telm_conf->ioss_config.telem_evts;
950 break;
951
952 default:
953 pr_err("Unknown Telemetry Unit Specified %d\n", telem_unit);
954 return -EINVAL;
955 }
956
957 if (!log_all_evts)
958 readlen = TELEM_MAX_EVENTS_SRAM;
959
960 ret = telem_evtlog_read(telem_unit, &ssram_region, readlen);
961 if (ret < 0)
962 return ret;
963
964 /* Invalid evt-id array specified via length mismatch */
965 if ((!log_all_evts) && (len > ret))
966 return -EINVAL;
967
968 if (log_all_evts)
969 for (index = 0; index < ret; index++) {
970 evtlog[index].telem_evtlog = ssram_region.events[index];
971 evtlog[index].telem_evtid = evtmap[index].evt_id;
972 }
973 else
974 for (index = 0, readlen = 0; (index < ret) && (readlen < len);
975 index++) {
976 for (idx1 = 0; idx1 < len; idx1++) {
977 /* Elements matched */
978 if (evtmap[index].evt_id ==
979 evtlog[idx1].telem_evtid) {
980 evtlog[idx1].telem_evtlog =
981 ssram_region.events[index];
982 readlen++;
983
984 break;
985 }
986 }
987 }
988
989 return readlen;
990}
991
992static int telemetry_plt_read_eventlog(enum telemetry_unit telem_unit,
993 struct telemetry_evtlog *evtlog, int len, int log_all_evts)
994{
995 int ret;
996
997 mutex_lock(&(telm_conf->telem_lock));
998 ret = telemetry_plt_raw_read_eventlog(telem_unit, evtlog,
999 len, log_all_evts);
1000 mutex_unlock(&(telm_conf->telem_lock));
1001
1002 return ret;
1003}
1004
1005static int telemetry_plt_get_trace_verbosity(enum telemetry_unit telem_unit,
1006 u32 *verbosity)
1007{
1008 u32 temp = 0;
1009 int ret;
1010
1011 if (verbosity == NULL)
1012 return -EINVAL;
1013
1014 mutex_lock(&(telm_conf->telem_trace_lock));
1015 switch (telem_unit) {
1016 case TELEM_PSS:
1017 ret = intel_punit_ipc_command(
1018 IPC_PUNIT_BIOS_READ_TELE_TRACE_CTRL,
1019 0, 0, NULL, &temp);
1020 if (ret) {
1021 pr_err("PSS TRACE_CTRL Read Failed\n");
1022 goto out;
1023 }
1024
1025 break;
1026
1027 case TELEM_IOSS:
1028 ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
1029 IOSS_TELEM_TRACE_CTL_READ, NULL, 0, &temp,
1030 IOSS_TELEM_READ_WORD);
1031 if (ret) {
1032 pr_err("IOSS TRACE_CTL Read Failed\n");
1033 goto out;
1034 }
1035
1036 break;
1037
1038 default:
1039 pr_err("Unknown Telemetry Unit Specified %d\n", telem_unit);
1040 ret = -EINVAL;
1041 break;
1042 }
1043 TELEM_EXTRACT_VERBOSITY(temp, *verbosity);
1044
1045out:
1046 mutex_unlock(&(telm_conf->telem_trace_lock));
1047 return ret;
1048}
1049
1050static int telemetry_plt_set_trace_verbosity(enum telemetry_unit telem_unit,
1051 u32 verbosity)
1052{
1053 u32 temp = 0;
1054 int ret;
1055
1056 verbosity &= TELEM_TRC_VERBOSITY_MASK;
1057
1058 mutex_lock(&(telm_conf->telem_trace_lock));
1059 switch (telem_unit) {
1060 case TELEM_PSS:
1061 ret = intel_punit_ipc_command(
1062 IPC_PUNIT_BIOS_READ_TELE_TRACE_CTRL,
1063 0, 0, NULL, &temp);
1064 if (ret) {
1065 pr_err("PSS TRACE_CTRL Read Failed\n");
1066 goto out;
1067 }
1068
1069 TELEM_CLEAR_VERBOSITY_BITS(temp);
1070 TELEM_SET_VERBOSITY_BITS(temp, verbosity);
1071
1072 ret = intel_punit_ipc_command(
1073 IPC_PUNIT_BIOS_WRITE_TELE_TRACE_CTRL,
1074 0, 0, &temp, NULL);
1075 if (ret) {
1076 pr_err("PSS TRACE_CTRL Verbosity Set Failed\n");
1077 goto out;
1078 }
1079 break;
1080
1081 case TELEM_IOSS:
1082 ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
1083 IOSS_TELEM_TRACE_CTL_READ, NULL, 0, &temp,
1084 IOSS_TELEM_READ_WORD);
1085 if (ret) {
1086 pr_err("IOSS TRACE_CTL Read Failed\n");
1087 goto out;
1088 }
1089
1090 TELEM_CLEAR_VERBOSITY_BITS(temp);
1091 TELEM_SET_VERBOSITY_BITS(temp, verbosity);
1092
1093 ret = intel_pmc_ipc_command(PMC_IPC_PMC_TELEMTRY,
1094 IOSS_TELEM_TRACE_CTL_WRITE, (u8 *)&temp,
1095 IOSS_TELEM_WRITE_FOURBYTES, NULL, 0);
1096 if (ret) {
1097 pr_err("IOSS TRACE_CTL Verbosity Set Failed\n");
1098 goto out;
1099 }
1100 break;
1101
1102 default:
1103 pr_err("Unknown Telemetry Unit Specified %d\n", telem_unit);
1104 ret = -EINVAL;
1105 break;
1106 }
1107
1108out:
1109 mutex_unlock(&(telm_conf->telem_trace_lock));
1110 return ret;
1111}
1112
1113static const struct telemetry_core_ops telm_pltops = {
1114 .get_trace_verbosity = telemetry_plt_get_trace_verbosity,
1115 .set_trace_verbosity = telemetry_plt_set_trace_verbosity,
1116 .set_sampling_period = telemetry_plt_set_sampling_period,
1117 .get_sampling_period = telemetry_plt_get_sampling_period,
1118 .raw_read_eventlog = telemetry_plt_raw_read_eventlog,
1119 .get_eventconfig = telemetry_plt_get_eventconfig,
1120 .update_events = telemetry_plt_update_events,
1121 .read_eventlog = telemetry_plt_read_eventlog,
1122 .reset_events = telemetry_plt_reset_events,
1123 .add_events = telemetry_plt_add_events,
1124};
1125
1126static int telemetry_pltdrv_probe(struct platform_device *pdev)
1127{
1128 struct resource *res0 = NULL, *res1 = NULL;
1129 const struct x86_cpu_id *id;
1130 int size, ret = -ENOMEM;
1131
1132 id = x86_match_cpu(telemetry_cpu_ids);
1133 if (!id)
1134 return -ENODEV;
1135
1136 telm_conf = (struct telemetry_plt_config *)id->driver_data;
1137
1138 res0 = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1139 if (!res0) {
1140 ret = -EINVAL;
1141 goto out;
1142 }
1143 size = resource_size(res0);
1144 if (!devm_request_mem_region(&pdev->dev, res0->start, size,
1145 pdev->name)) {
1146 ret = -EBUSY;
1147 goto out;
1148 }
1149 telm_conf->pss_config.ssram_base_addr = res0->start;
1150 telm_conf->pss_config.ssram_size = size;
1151
1152 res1 = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1153 if (!res1) {
1154 ret = -EINVAL;
1155 goto out;
1156 }
1157 size = resource_size(res1);
1158 if (!devm_request_mem_region(&pdev->dev, res1->start, size,
1159 pdev->name)) {
1160 ret = -EBUSY;
1161 goto out;
1162 }
1163
1164 telm_conf->ioss_config.ssram_base_addr = res1->start;
1165 telm_conf->ioss_config.ssram_size = size;
1166
1167 telm_conf->pss_config.regmap = ioremap_nocache(
1168 telm_conf->pss_config.ssram_base_addr,
1169 telm_conf->pss_config.ssram_size);
1170 if (!telm_conf->pss_config.regmap) {
1171 ret = -ENOMEM;
1172 goto out;
1173 }
1174
1175 telm_conf->ioss_config.regmap = ioremap_nocache(
1176 telm_conf->ioss_config.ssram_base_addr,
1177 telm_conf->ioss_config.ssram_size);
1178 if (!telm_conf->ioss_config.regmap) {
1179 ret = -ENOMEM;
1180 goto out;
1181 }
1182
1183 mutex_init(&telm_conf->telem_lock);
1184 mutex_init(&telm_conf->telem_trace_lock);
1185
1186 ret = telemetry_setup(pdev);
1187 if (ret)
1188 goto out;
1189
1190 ret = telemetry_set_pltdata(&telm_pltops, telm_conf);
1191 if (ret) {
1192 dev_err(&pdev->dev, "TELEMETRY Set Pltops Failed.\n");
1193 goto out;
1194 }
1195
1196 return 0;
1197
1198out:
1199 if (res0)
1200 release_mem_region(res0->start, resource_size(res0));
1201 if (res1)
1202 release_mem_region(res1->start, resource_size(res1));
1203 if (telm_conf->pss_config.regmap)
1204 iounmap(telm_conf->pss_config.regmap);
1205 if (telm_conf->ioss_config.regmap)
1206 iounmap(telm_conf->ioss_config.regmap);
1207 dev_err(&pdev->dev, "TELEMETRY Setup Failed.\n");
1208
1209 return ret;
1210}
1211
1212static int telemetry_pltdrv_remove(struct platform_device *pdev)
1213{
1214 telemetry_clear_pltdata();
1215 iounmap(telm_conf->pss_config.regmap);
1216 iounmap(telm_conf->ioss_config.regmap);
1217
1218 return 0;
1219}
1220
1221static struct platform_driver telemetry_soc_driver = {
1222 .probe = telemetry_pltdrv_probe,
1223 .remove = telemetry_pltdrv_remove,
1224 .driver = {
1225 .name = DRIVER_NAME,
1226 },
1227};
1228
1229static int __init telemetry_module_init(void)
1230{
1231 return platform_driver_register(&telemetry_soc_driver);
1232}
1233
1234static void __exit telemetry_module_exit(void)
1235{
1236 platform_driver_unregister(&telemetry_soc_driver);
1237}
1238
1239device_initcall(telemetry_module_init);
1240module_exit(telemetry_module_exit);
1241
1242MODULE_AUTHOR("Souvik Kumar Chakravarty <souvik.k.chakravarty@intel.com>");
1243MODULE_DESCRIPTION("Intel SoC Telemetry Platform Driver");
1244MODULE_VERSION(DRIVER_VERSION);
1245MODULE_LICENSE("GPL");