1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
   2/*
   3 * Copyright (C) 2012-2014, 2018-2023 Intel Corporation
   4 * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
   5 * Copyright (C) 2016-2017 Intel Deutschland GmbH
   6 */
   7#include <linux/module.h>
   8#include <linux/rtnetlink.h>
   9#include <linux/vmalloc.h>
  10#include <net/mac80211.h>
  11
  12#include "fw/notif-wait.h"
  13#include "iwl-trans.h"
  14#include "iwl-op-mode.h"
  15#include "fw/img.h"
  16#include "iwl-debug.h"
  17#include "iwl-drv.h"
  18#include "iwl-modparams.h"
  19#include "mvm.h"
  20#include "iwl-phy-db.h"
  21#include "iwl-eeprom-parse.h"
  22#include "iwl-csr.h"
  23#include "iwl-io.h"
  24#include "iwl-prph.h"
  25#include "rs.h"
  26#include "fw/api/scan.h"
  27#include "fw/api/rfi.h"
  28#include "time-event.h"
  29#include "fw-api.h"
  30#include "fw/acpi.h"
  31#include "fw/uefi.h"
  32#include "time-sync.h"
  33
  34#define DRV_DESCRIPTION	"The new Intel(R) wireless AGN driver for Linux"
  35MODULE_DESCRIPTION(DRV_DESCRIPTION);
  36MODULE_LICENSE("GPL");
  37MODULE_IMPORT_NS(IWLWIFI);
  38
  39static const struct iwl_op_mode_ops iwl_mvm_ops;
  40static const struct iwl_op_mode_ops iwl_mvm_ops_mq;
  41
  42struct iwl_mvm_mod_params iwlmvm_mod_params = {
  43	.power_scheme = IWL_POWER_SCHEME_BPS,
  44	/* rest of fields are 0 by default */
  45};
  46
  47module_param_named(init_dbg, iwlmvm_mod_params.init_dbg, bool, 0444);
  48MODULE_PARM_DESC(init_dbg,
  49		 "set to true to debug an ASSERT in INIT fw (default: false");
  50module_param_named(power_scheme, iwlmvm_mod_params.power_scheme, int, 0444);
  51MODULE_PARM_DESC(power_scheme,
  52		 "power management scheme: 1-active, 2-balanced, 3-low power, default: 2");
  53
  54/*
  55 * module init and exit functions
  56 */
  57static int __init iwl_mvm_init(void)
  58{
  59	int ret;
  60
  61	ret = iwl_mvm_rate_control_register();
  62	if (ret) {
  63		pr_err("Unable to register rate control algorithm: %d\n", ret);
  64		return ret;
  65	}
  66
  67	ret = iwl_opmode_register("iwlmvm", &iwl_mvm_ops);
  68	if (ret)
  69		pr_err("Unable to register MVM op_mode: %d\n", ret);
  70
  71	return ret;
  72}
  73module_init(iwl_mvm_init);
  74
  75static void __exit iwl_mvm_exit(void)
  76{
  77	iwl_opmode_deregister("iwlmvm");
  78	iwl_mvm_rate_control_unregister();
  79}
  80module_exit(iwl_mvm_exit);
  81
  82static void iwl_mvm_nic_config(struct iwl_op_mode *op_mode)
  83{
  84	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
  85	u8 radio_cfg_type, radio_cfg_step, radio_cfg_dash;
  86	u32 reg_val;
  87	u32 phy_config = iwl_mvm_get_phy_config(mvm);
  88
  89	radio_cfg_type = (phy_config & FW_PHY_CFG_RADIO_TYPE) >>
  90			 FW_PHY_CFG_RADIO_TYPE_POS;
  91	radio_cfg_step = (phy_config & FW_PHY_CFG_RADIO_STEP) >>
  92			 FW_PHY_CFG_RADIO_STEP_POS;
  93	radio_cfg_dash = (phy_config & FW_PHY_CFG_RADIO_DASH) >>
  94			 FW_PHY_CFG_RADIO_DASH_POS;
  95
  96	IWL_DEBUG_INFO(mvm, "Radio type=0x%x-0x%x-0x%x\n", radio_cfg_type,
  97		       radio_cfg_step, radio_cfg_dash);
  98
  99	if (mvm->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
 100		return;
 101
 102	/* SKU control */
 103	reg_val = CSR_HW_REV_STEP_DASH(mvm->trans->hw_rev);
 104
 105	/* radio configuration */
 106	reg_val |= radio_cfg_type << CSR_HW_IF_CONFIG_REG_POS_PHY_TYPE;
 107	reg_val |= radio_cfg_step << CSR_HW_IF_CONFIG_REG_POS_PHY_STEP;
 108	reg_val |= radio_cfg_dash << CSR_HW_IF_CONFIG_REG_POS_PHY_DASH;
 109
 110	WARN_ON((radio_cfg_type << CSR_HW_IF_CONFIG_REG_POS_PHY_TYPE) &
 111		 ~CSR_HW_IF_CONFIG_REG_MSK_PHY_TYPE);
 112
 113	/*
 114	 * TODO: Bits 7-8 of CSR in 8000 HW family and higher set the ADC
 115	 * sampling, and shouldn't be set to any non-zero value.
 116	 * The same is supposed to be true of the other HW, but unsetting
 117	 * them (such as the 7260) causes automatic tests to fail on seemingly
 118	 * unrelated errors. Need to further investigate this, but for now
 119	 * we'll separate cases.
 120	 */
 121	if (mvm->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_8000)
 122		reg_val |= CSR_HW_IF_CONFIG_REG_BIT_RADIO_SI;
 123
 124	if (iwl_fw_dbg_is_d3_debug_enabled(&mvm->fwrt))
 125		reg_val |= CSR_HW_IF_CONFIG_REG_D3_DEBUG;
 126
 127	iwl_trans_set_bits_mask(mvm->trans, CSR_HW_IF_CONFIG_REG,
 128				CSR_HW_IF_CONFIG_REG_MSK_MAC_STEP_DASH |
 129				CSR_HW_IF_CONFIG_REG_MSK_PHY_TYPE |
 130				CSR_HW_IF_CONFIG_REG_MSK_PHY_STEP |
 131				CSR_HW_IF_CONFIG_REG_MSK_PHY_DASH |
 132				CSR_HW_IF_CONFIG_REG_BIT_RADIO_SI |
 133				CSR_HW_IF_CONFIG_REG_BIT_MAC_SI   |
 134				CSR_HW_IF_CONFIG_REG_D3_DEBUG,
 135				reg_val);
 136
 137	/*
 138	 * W/A : NIC is stuck in a reset state after Early PCIe power off
 139	 * (PCIe power is lost before PERST# is asserted), causing ME FW
 140	 * to lose ownership and not being able to obtain it back.
 141	 */
 142	if (!mvm->trans->cfg->apmg_not_supported)
 143		iwl_set_bits_mask_prph(mvm->trans, APMG_PS_CTRL_REG,
 144				       APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS,
 145				       ~APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS);
 146}
 147
 148static void iwl_mvm_rx_monitor_notif(struct iwl_mvm *mvm,
 149				     struct iwl_rx_cmd_buffer *rxb)
 150{
 151	struct iwl_rx_packet *pkt = rxb_addr(rxb);
 152	struct iwl_datapath_monitor_notif *notif = (void *)pkt->data;
 153	struct ieee80211_supported_band *sband;
 154	const struct ieee80211_sta_he_cap *he_cap;
 155	struct ieee80211_vif *vif;
 156
 157	if (notif->type != cpu_to_le32(IWL_DP_MON_NOTIF_TYPE_EXT_CCA))
 158		return;
 159
 160	vif = iwl_mvm_get_vif_by_macid(mvm, notif->mac_id);
 161	if (!vif || vif->type != NL80211_IFTYPE_STATION)
 162		return;
 163
 164	if (!vif->bss_conf.chandef.chan ||
 165	    vif->bss_conf.chandef.chan->band != NL80211_BAND_2GHZ ||
 166	    vif->bss_conf.chandef.width < NL80211_CHAN_WIDTH_40)
 167		return;
 168
 169	if (!vif->cfg.assoc)
 170		return;
 171
 172	/* this shouldn't happen *again*, ignore it */
 173	if (mvm->cca_40mhz_workaround)
 174		return;
 175
 176	/*
 177	 * We'll decrement this on disconnect - so set to 2 since we'll
 178	 * still have to disconnect from the current AP first.
 179	 */
 180	mvm->cca_40mhz_workaround = 2;
 181
 182	/*
 183	 * This capability manipulation isn't really ideal, but it's the
 184	 * easiest choice - otherwise we'd have to do some major changes
 185	 * in mac80211 to support this, which isn't worth it. This does
 186	 * mean that userspace may have outdated information, but that's
 187	 * actually not an issue at all.
 188	 */
 189	sband = mvm->hw->wiphy->bands[NL80211_BAND_2GHZ];
 190
 191	WARN_ON(!sband->ht_cap.ht_supported);
 192	WARN_ON(!(sband->ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40));
 193	sband->ht_cap.cap &= ~IEEE80211_HT_CAP_SUP_WIDTH_20_40;
 194
 195	he_cap = ieee80211_get_he_iftype_cap_vif(sband, vif);
 
 196
 197	if (he_cap) {
 198		/* we know that ours is writable */
 199		struct ieee80211_sta_he_cap *he = (void *)(uintptr_t)he_cap;
 200
 201		WARN_ON(!he->has_he);
 202		WARN_ON(!(he->he_cap_elem.phy_cap_info[0] &
 203				IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G));
 204		he->he_cap_elem.phy_cap_info[0] &=
 205			~IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G;
 206	}
 207
 208	ieee80211_disconnect(vif, true);
 209}
 210
 211void iwl_mvm_update_link_smps(struct ieee80211_vif *vif,
 212			      struct ieee80211_bss_conf *link_conf)
 213{
 214	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 215	struct iwl_mvm *mvm = mvmvif->mvm;
 216	enum ieee80211_smps_mode mode = IEEE80211_SMPS_AUTOMATIC;
 217
 218	if (!link_conf)
 219		return;
 220
 221	if (mvm->fw_static_smps_request &&
 222	    link_conf->chandef.width == NL80211_CHAN_WIDTH_160 &&
 223	    link_conf->he_support)
 224		mode = IEEE80211_SMPS_STATIC;
 225
 226	iwl_mvm_update_smps(mvm, vif, IWL_MVM_SMPS_REQ_FW, mode,
 227			    link_conf->link_id);
 228}
 229
 230static void iwl_mvm_intf_dual_chain_req(void *data, u8 *mac,
 231					struct ieee80211_vif *vif)
 232{
 233	struct ieee80211_bss_conf *link_conf;
 234	unsigned int link_id;
 235
 236	rcu_read_lock();
 237
 238	for_each_vif_active_link(vif, link_conf, link_id)
 239		iwl_mvm_update_link_smps(vif, link_conf);
 240
 241	rcu_read_unlock();
 242}
 243
 244static void iwl_mvm_rx_thermal_dual_chain_req(struct iwl_mvm *mvm,
 245					      struct iwl_rx_cmd_buffer *rxb)
 246{
 247	struct iwl_rx_packet *pkt = rxb_addr(rxb);
 248	struct iwl_thermal_dual_chain_request *req = (void *)pkt->data;
 249
 250	/*
 251	 * We could pass it to the iterator data, but also need to remember
 252	 * it for new interfaces that are added while in this state.
 253	 */
 254	mvm->fw_static_smps_request =
 255		req->event == cpu_to_le32(THERMAL_DUAL_CHAIN_REQ_DISABLE);
 256	ieee80211_iterate_interfaces(mvm->hw,
 257				     IEEE80211_IFACE_SKIP_SDATA_NOT_IN_DRIVER,
 258				     iwl_mvm_intf_dual_chain_req, NULL);
 259}
 260
 261/**
 262 * enum iwl_rx_handler_context context for Rx handler
 263 * @RX_HANDLER_SYNC : this means that it will be called in the Rx path
 264 *	which can't acquire mvm->mutex.
 265 * @RX_HANDLER_ASYNC_LOCKED : If the handler needs to hold mvm->mutex
 266 *	(and only in this case!), it should be set as ASYNC. In that case,
 267 *	it will be called from a worker with mvm->mutex held.
 268 * @RX_HANDLER_ASYNC_UNLOCKED : in case the handler needs to lock the
 269 *	mutex itself, it will be called from a worker without mvm->mutex held.
 270 */
 271enum iwl_rx_handler_context {
 272	RX_HANDLER_SYNC,
 273	RX_HANDLER_ASYNC_LOCKED,
 274	RX_HANDLER_ASYNC_UNLOCKED,
 275};
 276
 277/**
 278 * struct iwl_rx_handlers handler for FW notification
 279 * @cmd_id: command id
 280 * @min_size: minimum size to expect for the notification
 281 * @context: see &iwl_rx_handler_context
 282 * @fn: the function is called when notification is received
 283 */
 284struct iwl_rx_handlers {
 285	u16 cmd_id, min_size;
 286	enum iwl_rx_handler_context context;
 287	void (*fn)(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb);
 288};
 289
 290#define RX_HANDLER_NO_SIZE(_cmd_id, _fn, _context)		\
 291	{ .cmd_id = _cmd_id, .fn = _fn, .context = _context, }
 292#define RX_HANDLER_GRP_NO_SIZE(_grp, _cmd, _fn, _context)	\
 293	{ .cmd_id = WIDE_ID(_grp, _cmd), .fn = _fn, .context = _context, }
 294#define RX_HANDLER(_cmd_id, _fn, _context, _struct)		\
 295	{ .cmd_id = _cmd_id, .fn = _fn,				\
 296	  .context = _context, .min_size = sizeof(_struct), }
 297#define RX_HANDLER_GRP(_grp, _cmd, _fn, _context, _struct)	\
 298	{ .cmd_id = WIDE_ID(_grp, _cmd), .fn = _fn,		\
 299	  .context = _context, .min_size = sizeof(_struct), }
 300
 301/*
 302 * Handlers for fw notifications
 303 * Convention: RX_HANDLER(CMD_NAME, iwl_mvm_rx_CMD_NAME
 304 * This list should be in order of frequency for performance purposes.
 305 *
 306 * The handler can be one from three contexts, see &iwl_rx_handler_context
 307 */
 308static const struct iwl_rx_handlers iwl_mvm_rx_handlers[] = {
 309	RX_HANDLER(TX_CMD, iwl_mvm_rx_tx_cmd, RX_HANDLER_SYNC,
 310		   struct iwl_mvm_tx_resp),
 311	RX_HANDLER(BA_NOTIF, iwl_mvm_rx_ba_notif, RX_HANDLER_SYNC,
 312		   struct iwl_mvm_ba_notif),
 313
 314	RX_HANDLER_GRP(DATA_PATH_GROUP, TLC_MNG_UPDATE_NOTIF,
 315		       iwl_mvm_tlc_update_notif, RX_HANDLER_SYNC,
 316		       struct iwl_tlc_update_notif),
 317
 318	RX_HANDLER(BT_PROFILE_NOTIFICATION, iwl_mvm_rx_bt_coex_notif,
 319		   RX_HANDLER_ASYNC_LOCKED, struct iwl_bt_coex_profile_notif),
 320	RX_HANDLER_NO_SIZE(BEACON_NOTIFICATION, iwl_mvm_rx_beacon_notif,
 321			   RX_HANDLER_ASYNC_LOCKED),
 322	RX_HANDLER_NO_SIZE(STATISTICS_NOTIFICATION, iwl_mvm_rx_statistics,
 323			   RX_HANDLER_ASYNC_LOCKED),
 324
 325	RX_HANDLER_GRP(STATISTICS_GROUP, STATISTICS_OPER_NOTIF,
 326		       iwl_mvm_handle_rx_system_oper_stats,
 327		       RX_HANDLER_ASYNC_LOCKED,
 328		       struct iwl_system_statistics_notif_oper),
 329	RX_HANDLER_GRP(STATISTICS_GROUP, STATISTICS_OPER_PART1_NOTIF,
 330		       iwl_mvm_handle_rx_system_oper_part1_stats,
 331		       RX_HANDLER_ASYNC_LOCKED,
 332		       struct iwl_system_statistics_part1_notif_oper),
 333	RX_HANDLER_GRP(SYSTEM_GROUP, SYSTEM_STATISTICS_END_NOTIF,
 334		       iwl_mvm_handle_rx_system_end_stats_notif,
 335		       RX_HANDLER_ASYNC_LOCKED,
 336		       struct iwl_system_statistics_end_notif),
 337
 338	RX_HANDLER(BA_WINDOW_STATUS_NOTIFICATION_ID,
 339		   iwl_mvm_window_status_notif, RX_HANDLER_SYNC,
 340		   struct iwl_ba_window_status_notif),
 341
 342	RX_HANDLER(TIME_EVENT_NOTIFICATION, iwl_mvm_rx_time_event_notif,
 343		   RX_HANDLER_SYNC, struct iwl_time_event_notif),
 344	RX_HANDLER_GRP(MAC_CONF_GROUP, SESSION_PROTECTION_NOTIF,
 345		       iwl_mvm_rx_session_protect_notif, RX_HANDLER_SYNC,
 346		       struct iwl_mvm_session_prot_notif),
 347	RX_HANDLER(MCC_CHUB_UPDATE_CMD, iwl_mvm_rx_chub_update_mcc,
 348		   RX_HANDLER_ASYNC_LOCKED, struct iwl_mcc_chub_notif),
 349
 350	RX_HANDLER(EOSP_NOTIFICATION, iwl_mvm_rx_eosp_notif, RX_HANDLER_SYNC,
 351		   struct iwl_mvm_eosp_notification),
 352
 353	RX_HANDLER(SCAN_ITERATION_COMPLETE,
 354		   iwl_mvm_rx_lmac_scan_iter_complete_notif, RX_HANDLER_SYNC,
 355		   struct iwl_lmac_scan_complete_notif),
 356	RX_HANDLER(SCAN_OFFLOAD_COMPLETE,
 357		   iwl_mvm_rx_lmac_scan_complete_notif,
 358		   RX_HANDLER_ASYNC_LOCKED, struct iwl_periodic_scan_complete),
 359	RX_HANDLER_NO_SIZE(MATCH_FOUND_NOTIFICATION,
 360			   iwl_mvm_rx_scan_match_found,
 361			   RX_HANDLER_SYNC),
 362	RX_HANDLER(SCAN_COMPLETE_UMAC, iwl_mvm_rx_umac_scan_complete_notif,
 363		   RX_HANDLER_ASYNC_LOCKED, struct iwl_umac_scan_complete),
 364	RX_HANDLER(SCAN_ITERATION_COMPLETE_UMAC,
 365		   iwl_mvm_rx_umac_scan_iter_complete_notif, RX_HANDLER_SYNC,
 366		   struct iwl_umac_scan_iter_complete_notif),
 367
 368	RX_HANDLER(MISSED_BEACONS_NOTIFICATION, iwl_mvm_rx_missed_beacons_notif,
 369		   RX_HANDLER_SYNC, struct iwl_missed_beacons_notif),
 370
 371	RX_HANDLER(REPLY_ERROR, iwl_mvm_rx_fw_error, RX_HANDLER_SYNC,
 372		   struct iwl_error_resp),
 373	RX_HANDLER(PSM_UAPSD_AP_MISBEHAVING_NOTIFICATION,
 374		   iwl_mvm_power_uapsd_misbehaving_ap_notif, RX_HANDLER_SYNC,
 375		   struct iwl_uapsd_misbehaving_ap_notif),
 376	RX_HANDLER_NO_SIZE(DTS_MEASUREMENT_NOTIFICATION, iwl_mvm_temp_notif,
 377			   RX_HANDLER_ASYNC_LOCKED),
 378	RX_HANDLER_GRP_NO_SIZE(PHY_OPS_GROUP, DTS_MEASUREMENT_NOTIF_WIDE,
 379			       iwl_mvm_temp_notif, RX_HANDLER_ASYNC_UNLOCKED),
 380	RX_HANDLER_GRP(PHY_OPS_GROUP, CT_KILL_NOTIFICATION,
 381		       iwl_mvm_ct_kill_notif, RX_HANDLER_SYNC,
 382		       struct ct_kill_notif),
 383
 384	RX_HANDLER(TDLS_CHANNEL_SWITCH_NOTIFICATION, iwl_mvm_rx_tdls_notif,
 385		   RX_HANDLER_ASYNC_LOCKED,
 386		   struct iwl_tdls_channel_switch_notif),
 387	RX_HANDLER(MFUART_LOAD_NOTIFICATION, iwl_mvm_rx_mfuart_notif,
 388		   RX_HANDLER_SYNC, struct iwl_mfuart_load_notif_v1),
 389	RX_HANDLER_GRP(LOCATION_GROUP, TOF_RESPONDER_STATS,
 390		       iwl_mvm_ftm_responder_stats, RX_HANDLER_ASYNC_LOCKED,
 391		       struct iwl_ftm_responder_stats),
 392
 393	RX_HANDLER_GRP_NO_SIZE(LOCATION_GROUP, TOF_RANGE_RESPONSE_NOTIF,
 394			       iwl_mvm_ftm_range_resp, RX_HANDLER_ASYNC_LOCKED),
 395	RX_HANDLER_GRP_NO_SIZE(LOCATION_GROUP, TOF_LC_NOTIF,
 396			       iwl_mvm_ftm_lc_notif, RX_HANDLER_ASYNC_LOCKED),
 397
 398	RX_HANDLER_GRP(DEBUG_GROUP, MFU_ASSERT_DUMP_NTF,
 399		       iwl_mvm_mfu_assert_dump_notif, RX_HANDLER_SYNC,
 400		       struct iwl_mfu_assert_dump_notif),
 401	RX_HANDLER_GRP(PROT_OFFLOAD_GROUP, STORED_BEACON_NTF,
 402		       iwl_mvm_rx_stored_beacon_notif, RX_HANDLER_SYNC,
 403		       struct iwl_stored_beacon_notif_v2),
 404	RX_HANDLER_GRP(DATA_PATH_GROUP, MU_GROUP_MGMT_NOTIF,
 405		       iwl_mvm_mu_mimo_grp_notif, RX_HANDLER_SYNC,
 406		       struct iwl_mu_group_mgmt_notif),
 407	RX_HANDLER_GRP(DATA_PATH_GROUP, STA_PM_NOTIF,
 408		       iwl_mvm_sta_pm_notif, RX_HANDLER_SYNC,
 409		       struct iwl_mvm_pm_state_notification),
 410	RX_HANDLER_GRP(MAC_CONF_GROUP, PROBE_RESPONSE_DATA_NOTIF,
 411		       iwl_mvm_probe_resp_data_notif,
 412		       RX_HANDLER_ASYNC_LOCKED,
 413		       struct iwl_probe_resp_data_notif),
 414	RX_HANDLER_GRP(MAC_CONF_GROUP, CHANNEL_SWITCH_START_NOTIF,
 415		       iwl_mvm_channel_switch_start_notif,
 416		       RX_HANDLER_SYNC, struct iwl_channel_switch_start_notif),
 417	RX_HANDLER_GRP(MAC_CONF_GROUP, CHANNEL_SWITCH_ERROR_NOTIF,
 418		       iwl_mvm_channel_switch_error_notif,
 419		       RX_HANDLER_ASYNC_UNLOCKED,
 420		       struct iwl_channel_switch_error_notif),
 421	RX_HANDLER_GRP(DATA_PATH_GROUP, MONITOR_NOTIF,
 422		       iwl_mvm_rx_monitor_notif, RX_HANDLER_ASYNC_LOCKED,
 423		       struct iwl_datapath_monitor_notif),
 424
 425	RX_HANDLER_GRP(DATA_PATH_GROUP, THERMAL_DUAL_CHAIN_REQUEST,
 426		       iwl_mvm_rx_thermal_dual_chain_req,
 427		       RX_HANDLER_ASYNC_LOCKED,
 428		       struct iwl_thermal_dual_chain_request),
 429
 430	RX_HANDLER_GRP(SYSTEM_GROUP, RFI_DEACTIVATE_NOTIF,
 431		       iwl_rfi_deactivate_notif_handler, RX_HANDLER_ASYNC_UNLOCKED,
 432		       struct iwl_rfi_deactivate_notif),
 433
 434	RX_HANDLER_GRP(LEGACY_GROUP,
 435		       WNM_80211V_TIMING_MEASUREMENT_NOTIFICATION,
 436		       iwl_mvm_time_sync_msmt_event, RX_HANDLER_SYNC,
 437		       struct iwl_time_msmt_notify),
 438	RX_HANDLER_GRP(LEGACY_GROUP,
 439		       WNM_80211V_TIMING_MEASUREMENT_CONFIRM_NOTIFICATION,
 440		       iwl_mvm_time_sync_msmt_confirm_event, RX_HANDLER_SYNC,
 441		       struct iwl_time_msmt_cfm_notify),
 442	RX_HANDLER_GRP(MAC_CONF_GROUP, ROC_NOTIF,
 443		       iwl_mvm_rx_roc_notif, RX_HANDLER_SYNC,
 444		       struct iwl_roc_notif),
 445};
 446#undef RX_HANDLER
 447#undef RX_HANDLER_GRP
 448
 449/* Please keep this array *SORTED* by hex value.
 450 * Access is done through binary search
 451 */
 452static const struct iwl_hcmd_names iwl_mvm_legacy_names[] = {
 453	HCMD_NAME(UCODE_ALIVE_NTFY),
 454	HCMD_NAME(REPLY_ERROR),
 455	HCMD_NAME(ECHO_CMD),
 456	HCMD_NAME(INIT_COMPLETE_NOTIF),
 457	HCMD_NAME(PHY_CONTEXT_CMD),
 458	HCMD_NAME(DBG_CFG),
 459	HCMD_NAME(SCAN_CFG_CMD),
 460	HCMD_NAME(SCAN_REQ_UMAC),
 461	HCMD_NAME(SCAN_ABORT_UMAC),
 462	HCMD_NAME(SCAN_COMPLETE_UMAC),
 463	HCMD_NAME(BA_WINDOW_STATUS_NOTIFICATION_ID),
 464	HCMD_NAME(ADD_STA_KEY),
 465	HCMD_NAME(ADD_STA),
 466	HCMD_NAME(REMOVE_STA),
 
 467	HCMD_NAME(TX_CMD),
 468	HCMD_NAME(SCD_QUEUE_CFG),
 469	HCMD_NAME(TXPATH_FLUSH),
 470	HCMD_NAME(MGMT_MCAST_KEY),
 471	HCMD_NAME(WEP_KEY),
 472	HCMD_NAME(SHARED_MEM_CFG),
 473	HCMD_NAME(TDLS_CHANNEL_SWITCH_CMD),
 474	HCMD_NAME(MAC_CONTEXT_CMD),
 475	HCMD_NAME(TIME_EVENT_CMD),
 476	HCMD_NAME(TIME_EVENT_NOTIFICATION),
 477	HCMD_NAME(BINDING_CONTEXT_CMD),
 478	HCMD_NAME(TIME_QUOTA_CMD),
 479	HCMD_NAME(NON_QOS_TX_COUNTER_CMD),
 480	HCMD_NAME(LEDS_CMD),
 481	HCMD_NAME(LQ_CMD),
 482	HCMD_NAME(FW_PAGING_BLOCK_CMD),
 483	HCMD_NAME(SCAN_OFFLOAD_REQUEST_CMD),
 484	HCMD_NAME(SCAN_OFFLOAD_ABORT_CMD),
 485	HCMD_NAME(HOT_SPOT_CMD),
 486	HCMD_NAME(SCAN_OFFLOAD_PROFILES_QUERY_CMD),
 487	HCMD_NAME(BT_COEX_UPDATE_REDUCED_TXP),
 488	HCMD_NAME(BT_COEX_CI),
 489	HCMD_NAME(WNM_80211V_TIMING_MEASUREMENT_NOTIFICATION),
 490	HCMD_NAME(WNM_80211V_TIMING_MEASUREMENT_CONFIRM_NOTIFICATION),
 491	HCMD_NAME(PHY_CONFIGURATION_CMD),
 492	HCMD_NAME(CALIB_RES_NOTIF_PHY_DB),
 493	HCMD_NAME(PHY_DB_CMD),
 494	HCMD_NAME(SCAN_OFFLOAD_COMPLETE),
 495	HCMD_NAME(SCAN_OFFLOAD_UPDATE_PROFILES_CMD),
 496	HCMD_NAME(POWER_TABLE_CMD),
 497	HCMD_NAME(PSM_UAPSD_AP_MISBEHAVING_NOTIFICATION),
 498	HCMD_NAME(REPLY_THERMAL_MNG_BACKOFF),
 499	HCMD_NAME(NVM_ACCESS_CMD),
 500	HCMD_NAME(BEACON_NOTIFICATION),
 501	HCMD_NAME(BEACON_TEMPLATE_CMD),
 502	HCMD_NAME(TX_ANT_CONFIGURATION_CMD),
 503	HCMD_NAME(BT_CONFIG),
 504	HCMD_NAME(STATISTICS_CMD),
 505	HCMD_NAME(STATISTICS_NOTIFICATION),
 506	HCMD_NAME(EOSP_NOTIFICATION),
 507	HCMD_NAME(REDUCE_TX_POWER_CMD),
 508	HCMD_NAME(MISSED_BEACONS_NOTIFICATION),
 509	HCMD_NAME(TDLS_CONFIG_CMD),
 510	HCMD_NAME(MAC_PM_POWER_TABLE),
 511	HCMD_NAME(TDLS_CHANNEL_SWITCH_NOTIFICATION),
 512	HCMD_NAME(MFUART_LOAD_NOTIFICATION),
 513	HCMD_NAME(RSS_CONFIG_CMD),
 514	HCMD_NAME(SCAN_ITERATION_COMPLETE_UMAC),
 515	HCMD_NAME(REPLY_RX_PHY_CMD),
 516	HCMD_NAME(REPLY_RX_MPDU_CMD),
 517	HCMD_NAME(BAR_FRAME_RELEASE),
 518	HCMD_NAME(FRAME_RELEASE),
 519	HCMD_NAME(BA_NOTIF),
 520	HCMD_NAME(MCC_UPDATE_CMD),
 521	HCMD_NAME(MCC_CHUB_UPDATE_CMD),
 522	HCMD_NAME(MARKER_CMD),
 523	HCMD_NAME(BT_PROFILE_NOTIFICATION),
 524	HCMD_NAME(MCAST_FILTER_CMD),
 525	HCMD_NAME(REPLY_SF_CFG_CMD),
 526	HCMD_NAME(REPLY_BEACON_FILTERING_CMD),
 527	HCMD_NAME(D3_CONFIG_CMD),
 528	HCMD_NAME(PROT_OFFLOAD_CONFIG_CMD),
 
 529	HCMD_NAME(MATCH_FOUND_NOTIFICATION),
 530	HCMD_NAME(DTS_MEASUREMENT_NOTIFICATION),
 531	HCMD_NAME(WOWLAN_PATTERNS),
 532	HCMD_NAME(WOWLAN_CONFIGURATION),
 533	HCMD_NAME(WOWLAN_TSC_RSC_PARAM),
 534	HCMD_NAME(WOWLAN_TKIP_PARAM),
 535	HCMD_NAME(WOWLAN_KEK_KCK_MATERIAL),
 536	HCMD_NAME(WOWLAN_GET_STATUSES),
 537	HCMD_NAME(SCAN_ITERATION_COMPLETE),
 538	HCMD_NAME(D0I3_END_CMD),
 539	HCMD_NAME(LTR_CONFIG),
 540	HCMD_NAME(LDBG_CONFIG_CMD),
 541};
 542
 543/* Please keep this array *SORTED* by hex value.
 544 * Access is done through binary search
 545 */
 546static const struct iwl_hcmd_names iwl_mvm_system_names[] = {
 547	HCMD_NAME(SHARED_MEM_CFG_CMD),
 548	HCMD_NAME(INIT_EXTENDED_CFG_CMD),
 549	HCMD_NAME(FW_ERROR_RECOVERY_CMD),
 550	HCMD_NAME(RFI_CONFIG_CMD),
 551	HCMD_NAME(RFI_GET_FREQ_TABLE_CMD),
 552	HCMD_NAME(SYSTEM_FEATURES_CONTROL_CMD),
 553	HCMD_NAME(SYSTEM_STATISTICS_CMD),
 554	HCMD_NAME(SYSTEM_STATISTICS_END_NOTIF),
 555	HCMD_NAME(RFI_DEACTIVATE_NOTIF),
 556};
 557
 558/* Please keep this array *SORTED* by hex value.
 559 * Access is done through binary search
 560 */
 561static const struct iwl_hcmd_names iwl_mvm_mac_conf_names[] = {
 562	HCMD_NAME(CHANNEL_SWITCH_TIME_EVENT_CMD),
 563	HCMD_NAME(SESSION_PROTECTION_CMD),
 564	HCMD_NAME(MAC_CONFIG_CMD),
 565	HCMD_NAME(LINK_CONFIG_CMD),
 566	HCMD_NAME(STA_CONFIG_CMD),
 567	HCMD_NAME(AUX_STA_CMD),
 568	HCMD_NAME(STA_REMOVE_CMD),
 569	HCMD_NAME(STA_DISABLE_TX_CMD),
 570	HCMD_NAME(ROC_CMD),
 571	HCMD_NAME(ROC_NOTIF),
 572	HCMD_NAME(SESSION_PROTECTION_NOTIF),
 573	HCMD_NAME(CHANNEL_SWITCH_START_NOTIF),
 574};
 575
 576/* Please keep this array *SORTED* by hex value.
 577 * Access is done through binary search
 578 */
 579static const struct iwl_hcmd_names iwl_mvm_phy_names[] = {
 580	HCMD_NAME(CMD_DTS_MEASUREMENT_TRIGGER_WIDE),
 581	HCMD_NAME(CTDP_CONFIG_CMD),
 582	HCMD_NAME(TEMP_REPORTING_THRESHOLDS_CMD),
 583	HCMD_NAME(PER_CHAIN_LIMIT_OFFSET_CMD),
 584	HCMD_NAME(CT_KILL_NOTIFICATION),
 585	HCMD_NAME(DTS_MEASUREMENT_NOTIF_WIDE),
 586};
 587
 588/* Please keep this array *SORTED* by hex value.
 589 * Access is done through binary search
 590 */
 591static const struct iwl_hcmd_names iwl_mvm_data_path_names[] = {
 592	HCMD_NAME(DQA_ENABLE_CMD),
 593	HCMD_NAME(UPDATE_MU_GROUPS_CMD),
 594	HCMD_NAME(TRIGGER_RX_QUEUES_NOTIF_CMD),
 595	HCMD_NAME(STA_HE_CTXT_CMD),
 596	HCMD_NAME(RLC_CONFIG_CMD),
 597	HCMD_NAME(RFH_QUEUE_CONFIG_CMD),
 598	HCMD_NAME(TLC_MNG_CONFIG_CMD),
 599	HCMD_NAME(CHEST_COLLECTOR_FILTER_CONFIG_CMD),
 600	HCMD_NAME(SCD_QUEUE_CONFIG_CMD),
 601	HCMD_NAME(SEC_KEY_CMD),
 602	HCMD_NAME(MONITOR_NOTIF),
 603	HCMD_NAME(THERMAL_DUAL_CHAIN_REQUEST),
 604	HCMD_NAME(STA_PM_NOTIF),
 605	HCMD_NAME(MU_GROUP_MGMT_NOTIF),
 606	HCMD_NAME(RX_QUEUES_NOTIFICATION),
 607};
 608
 609/* Please keep this array *SORTED* by hex value.
 610 * Access is done through binary search
 611 */
 612static const struct iwl_hcmd_names iwl_mvm_statistics_names[] = {
 613	HCMD_NAME(STATISTICS_OPER_NOTIF),
 614	HCMD_NAME(STATISTICS_OPER_PART1_NOTIF),
 615};
 616
 617/* Please keep this array *SORTED* by hex value.
 618 * Access is done through binary search
 619 */
 620static const struct iwl_hcmd_names iwl_mvm_scan_names[] = {
 621	HCMD_NAME(OFFLOAD_MATCH_INFO_NOTIF),
 622};
 623
 624/* Please keep this array *SORTED* by hex value.
 625 * Access is done through binary search
 626 */
 627static const struct iwl_hcmd_names iwl_mvm_location_names[] = {
 628	HCMD_NAME(TOF_RANGE_REQ_CMD),
 629	HCMD_NAME(TOF_CONFIG_CMD),
 630	HCMD_NAME(TOF_RANGE_ABORT_CMD),
 631	HCMD_NAME(TOF_RANGE_REQ_EXT_CMD),
 632	HCMD_NAME(TOF_RESPONDER_CONFIG_CMD),
 633	HCMD_NAME(TOF_RESPONDER_DYN_CONFIG_CMD),
 634	HCMD_NAME(TOF_LC_NOTIF),
 635	HCMD_NAME(TOF_RESPONDER_STATS),
 636	HCMD_NAME(TOF_MCSI_DEBUG_NOTIF),
 637	HCMD_NAME(TOF_RANGE_RESPONSE_NOTIF),
 638};
 639
 640/* Please keep this array *SORTED* by hex value.
 641 * Access is done through binary search
 642 */
 643static const struct iwl_hcmd_names iwl_mvm_prot_offload_names[] = {
 644	HCMD_NAME(WOWLAN_WAKE_PKT_NOTIFICATION),
 645	HCMD_NAME(WOWLAN_INFO_NOTIFICATION),
 646	HCMD_NAME(D3_END_NOTIFICATION),
 647	HCMD_NAME(STORED_BEACON_NTF),
 648};
 649
 650/* Please keep this array *SORTED* by hex value.
 651 * Access is done through binary search
 652 */
 653static const struct iwl_hcmd_names iwl_mvm_regulatory_and_nvm_names[] = {
 654	HCMD_NAME(NVM_ACCESS_COMPLETE),
 655	HCMD_NAME(NVM_GET_INFO),
 656	HCMD_NAME(TAS_CONFIG),
 657};
 658
 659static const struct iwl_hcmd_arr iwl_mvm_groups[] = {
 660	[LEGACY_GROUP] = HCMD_ARR(iwl_mvm_legacy_names),
 661	[LONG_GROUP] = HCMD_ARR(iwl_mvm_legacy_names),
 662	[SYSTEM_GROUP] = HCMD_ARR(iwl_mvm_system_names),
 663	[MAC_CONF_GROUP] = HCMD_ARR(iwl_mvm_mac_conf_names),
 664	[PHY_OPS_GROUP] = HCMD_ARR(iwl_mvm_phy_names),
 665	[DATA_PATH_GROUP] = HCMD_ARR(iwl_mvm_data_path_names),
 666	[SCAN_GROUP] = HCMD_ARR(iwl_mvm_scan_names),
 667	[LOCATION_GROUP] = HCMD_ARR(iwl_mvm_location_names),
 668	[PROT_OFFLOAD_GROUP] = HCMD_ARR(iwl_mvm_prot_offload_names),
 669	[REGULATORY_AND_NVM_GROUP] =
 670		HCMD_ARR(iwl_mvm_regulatory_and_nvm_names),
 671	[STATISTICS_GROUP] = HCMD_ARR(iwl_mvm_statistics_names),
 672};
 673
 674/* this forward declaration can avoid to export the function */
 675static void iwl_mvm_async_handlers_wk(struct work_struct *wk);
 676
 677static u32 iwl_mvm_min_backoff(struct iwl_mvm *mvm)
 678{
 679	const struct iwl_pwr_tx_backoff *backoff = mvm->cfg->pwr_tx_backoffs;
 680	u64 dflt_pwr_limit;
 681
 682	if (!backoff)
 683		return 0;
 684
 685	dflt_pwr_limit = iwl_acpi_get_pwr_limit(mvm->dev);
 686
 687	while (backoff->pwr) {
 688		if (dflt_pwr_limit >= backoff->pwr)
 689			return backoff->backoff;
 690
 691		backoff++;
 692	}
 693
 694	return 0;
 695}
 696
 697static void iwl_mvm_tx_unblock_dwork(struct work_struct *work)
 698{
 699	struct iwl_mvm *mvm =
 700		container_of(work, struct iwl_mvm, cs_tx_unblock_dwork.work);
 701	struct ieee80211_vif *tx_blocked_vif;
 702	struct iwl_mvm_vif *mvmvif;
 703
 704	mutex_lock(&mvm->mutex);
 705
 706	tx_blocked_vif =
 707		rcu_dereference_protected(mvm->csa_tx_blocked_vif,
 708					  lockdep_is_held(&mvm->mutex));
 709
 710	if (!tx_blocked_vif)
 711		goto unlock;
 712
 713	mvmvif = iwl_mvm_vif_from_mac80211(tx_blocked_vif);
 714	iwl_mvm_modify_all_sta_disable_tx(mvm, mvmvif, false);
 715	RCU_INIT_POINTER(mvm->csa_tx_blocked_vif, NULL);
 716unlock:
 717	mutex_unlock(&mvm->mutex);
 718}
 719
 720static void iwl_mvm_fwrt_dump_start(void *ctx)
 721{
 722	struct iwl_mvm *mvm = ctx;
 723
 724	mutex_lock(&mvm->mutex);
 725}
 726
 727static void iwl_mvm_fwrt_dump_end(void *ctx)
 728{
 729	struct iwl_mvm *mvm = ctx;
 730
 731	mutex_unlock(&mvm->mutex);
 732}
 733
 734static bool iwl_mvm_fwrt_fw_running(void *ctx)
 735{
 736	return iwl_mvm_firmware_running(ctx);
 737}
 738
 739static int iwl_mvm_fwrt_send_hcmd(void *ctx, struct iwl_host_cmd *host_cmd)
 740{
 741	struct iwl_mvm *mvm = (struct iwl_mvm *)ctx;
 742	int ret;
 743
 744	mutex_lock(&mvm->mutex);
 745	ret = iwl_mvm_send_cmd(mvm, host_cmd);
 746	mutex_unlock(&mvm->mutex);
 747
 748	return ret;
 749}
 750
 751static bool iwl_mvm_d3_debug_enable(void *ctx)
 752{
 753	return IWL_MVM_D3_DEBUG;
 754}
 755
 756static const struct iwl_fw_runtime_ops iwl_mvm_fwrt_ops = {
 757	.dump_start = iwl_mvm_fwrt_dump_start,
 758	.dump_end = iwl_mvm_fwrt_dump_end,
 759	.fw_running = iwl_mvm_fwrt_fw_running,
 760	.send_hcmd = iwl_mvm_fwrt_send_hcmd,
 761	.d3_debug_enable = iwl_mvm_d3_debug_enable,
 762};
 763
 764static int iwl_mvm_start_get_nvm(struct iwl_mvm *mvm)
 765{
 766	struct iwl_trans *trans = mvm->trans;
 767	int ret;
 768
 769	if (trans->csme_own) {
 770		if (WARN(!mvm->mei_registered,
 771			 "csme is owner, but we aren't registered to iwlmei\n"))
 772			goto get_nvm_from_fw;
 773
 774		mvm->mei_nvm_data = iwl_mei_get_nvm();
 775		if (mvm->mei_nvm_data) {
 776			/*
 777			 * mvm->mei_nvm_data is set and because of that,
 778			 * we'll load the NVM from the FW when we'll get
 779			 * ownership.
 780			 */
 781			mvm->nvm_data =
 782				iwl_parse_mei_nvm_data(trans, trans->cfg,
 783						       mvm->mei_nvm_data,
 784						       mvm->fw,
 785						       mvm->set_tx_ant,
 786						       mvm->set_rx_ant);
 787			return 0;
 788		}
 789
 790		IWL_ERR(mvm,
 791			"Got a NULL NVM from CSME, trying to get it from the device\n");
 792	}
 793
 794get_nvm_from_fw:
 795	rtnl_lock();
 796	wiphy_lock(mvm->hw->wiphy);
 797	mutex_lock(&mvm->mutex);
 798
 799	ret = iwl_trans_start_hw(mvm->trans);
 800	if (ret) {
 801		mutex_unlock(&mvm->mutex);
 802		wiphy_unlock(mvm->hw->wiphy);
 803		rtnl_unlock();
 804		return ret;
 805	}
 806
 807	ret = iwl_run_init_mvm_ucode(mvm);
 808	if (ret && ret != -ERFKILL)
 809		iwl_fw_dbg_error_collect(&mvm->fwrt, FW_DBG_TRIGGER_DRIVER);
 810	if (!ret && iwl_mvm_is_lar_supported(mvm)) {
 811		mvm->hw->wiphy->regulatory_flags |= REGULATORY_WIPHY_SELF_MANAGED;
 812		ret = iwl_mvm_init_mcc(mvm);
 813	}
 814
 815	if (!iwlmvm_mod_params.init_dbg || !ret)
 816		iwl_mvm_stop_device(mvm);
 817
 818	mutex_unlock(&mvm->mutex);
 819	wiphy_unlock(mvm->hw->wiphy);
 820	rtnl_unlock();
 821
 822	if (ret)
 823		IWL_ERR(mvm, "Failed to run INIT ucode: %d\n", ret);
 824
 825	/* no longer need this regardless of failure or not */
 826	mvm->pldr_sync = false;
 827
 828	return ret;
 829}
 830
 831static int iwl_mvm_start_post_nvm(struct iwl_mvm *mvm)
 832{
 833	struct iwl_mvm_csme_conn_info *csme_conn_info __maybe_unused;
 834	int ret;
 835
 836	iwl_mvm_toggle_tx_ant(mvm, &mvm->mgmt_last_antenna_idx);
 837
 838	ret = iwl_mvm_mac_setup_register(mvm);
 839	if (ret)
 840		return ret;
 841
 842	mvm->hw_registered = true;
 843
 844	iwl_mvm_dbgfs_register(mvm);
 845
 846	wiphy_rfkill_set_hw_state_reason(mvm->hw->wiphy,
 847					 mvm->mei_rfkill_blocked,
 848					 RFKILL_HARD_BLOCK_NOT_OWNER);
 849
 850	iwl_mvm_mei_set_sw_rfkill_state(mvm);
 851
 852	return 0;
 853}
 854
 855struct iwl_mvm_frob_txf_data {
 856	u8 *buf;
 857	size_t buflen;
 858};
 859
 860static void iwl_mvm_frob_txf_key_iter(struct ieee80211_hw *hw,
 861				      struct ieee80211_vif *vif,
 862				      struct ieee80211_sta *sta,
 863				      struct ieee80211_key_conf *key,
 864				      void *data)
 865{
 866	struct iwl_mvm_frob_txf_data *txf = data;
 867	u8 keylen, match, matchend;
 868	u8 *keydata;
 869	size_t i;
 870
 871	switch (key->cipher) {
 872	case WLAN_CIPHER_SUITE_CCMP:
 873		keydata = key->key;
 874		keylen = key->keylen;
 875		break;
 876	case WLAN_CIPHER_SUITE_WEP40:
 877	case WLAN_CIPHER_SUITE_WEP104:
 878	case WLAN_CIPHER_SUITE_TKIP:
 879		/*
 880		 * WEP has short keys which might show up in the payload,
 881		 * and then you can deduce the key, so in this case just
 882		 * remove all FIFO data.
 883		 * For TKIP, we don't know the phase 2 keys here, so same.
 884		 */
 885		memset(txf->buf, 0xBB, txf->buflen);
 886		return;
 887	default:
 888		return;
 889	}
 890
 891	/* scan for key material and clear it out */
 892	match = 0;
 893	for (i = 0; i < txf->buflen; i++) {
 894		if (txf->buf[i] != keydata[match]) {
 895			match = 0;
 896			continue;
 897		}
 898		match++;
 899		if (match == keylen) {
 900			memset(txf->buf + i - keylen, 0xAA, keylen);
 901			match = 0;
 902		}
 903	}
 904
 905	/* we're dealing with a FIFO, so check wrapped around data */
 906	matchend = match;
 907	for (i = 0; match && i < keylen - match; i++) {
 908		if (txf->buf[i] != keydata[match])
 909			break;
 910		match++;
 911		if (match == keylen) {
 912			memset(txf->buf, 0xAA, i + 1);
 913			memset(txf->buf + txf->buflen - matchend, 0xAA,
 914			       matchend);
 915			break;
 916		}
 917	}
 918}
 919
 920static void iwl_mvm_frob_txf(void *ctx, void *buf, size_t buflen)
 921{
 922	struct iwl_mvm_frob_txf_data txf = {
 923		.buf = buf,
 924		.buflen = buflen,
 925	};
 926	struct iwl_mvm *mvm = ctx;
 927
 928	/* embedded key material exists only on old API */
 929	if (iwl_mvm_has_new_tx_api(mvm))
 930		return;
 931
 932	rcu_read_lock();
 933	ieee80211_iter_keys_rcu(mvm->hw, NULL, iwl_mvm_frob_txf_key_iter, &txf);
 934	rcu_read_unlock();
 935}
 936
 937static void iwl_mvm_frob_hcmd(void *ctx, void *hcmd, size_t len)
 938{
 939	/* we only use wide headers for commands */
 940	struct iwl_cmd_header_wide *hdr = hcmd;
 941	unsigned int frob_start = sizeof(*hdr), frob_end = 0;
 942
 943	if (len < sizeof(hdr))
 944		return;
 945
 946	/* all the commands we care about are in LONG_GROUP */
 947	if (hdr->group_id != LONG_GROUP)
 948		return;
 949
 950	switch (hdr->cmd) {
 951	case WEP_KEY:
 952	case WOWLAN_TKIP_PARAM:
 953	case WOWLAN_KEK_KCK_MATERIAL:
 954	case ADD_STA_KEY:
 955		/*
 956		 * blank out everything here, easier than dealing
 957		 * with the various versions of the command
 958		 */
 959		frob_end = INT_MAX;
 960		break;
 961	case MGMT_MCAST_KEY:
 962		frob_start = offsetof(struct iwl_mvm_mgmt_mcast_key_cmd, igtk);
 963		BUILD_BUG_ON(offsetof(struct iwl_mvm_mgmt_mcast_key_cmd, igtk) !=
 964			     offsetof(struct iwl_mvm_mgmt_mcast_key_cmd_v1, igtk));
 965
 966		frob_end = offsetofend(struct iwl_mvm_mgmt_mcast_key_cmd, igtk);
 967		BUILD_BUG_ON(offsetof(struct iwl_mvm_mgmt_mcast_key_cmd, igtk) <
 968			     offsetof(struct iwl_mvm_mgmt_mcast_key_cmd_v1, igtk));
 969		break;
 970	}
 971
 972	if (frob_start >= frob_end)
 973		return;
 974
 975	if (frob_end > len)
 976		frob_end = len;
 977
 978	memset((u8 *)hcmd + frob_start, 0xAA, frob_end - frob_start);
 979}
 980
 981static void iwl_mvm_frob_mem(void *ctx, u32 mem_addr, void *mem, size_t buflen)
 982{
 983	const struct iwl_dump_exclude *excl;
 984	struct iwl_mvm *mvm = ctx;
 985	int i;
 986
 987	switch (mvm->fwrt.cur_fw_img) {
 988	case IWL_UCODE_INIT:
 989	default:
 990		/* not relevant */
 991		return;
 992	case IWL_UCODE_REGULAR:
 993	case IWL_UCODE_REGULAR_USNIFFER:
 994		excl = mvm->fw->dump_excl;
 995		break;
 996	case IWL_UCODE_WOWLAN:
 997		excl = mvm->fw->dump_excl_wowlan;
 998		break;
 999	}
1000
1001	BUILD_BUG_ON(sizeof(mvm->fw->dump_excl) !=
1002		     sizeof(mvm->fw->dump_excl_wowlan));
1003
1004	for (i = 0; i < ARRAY_SIZE(mvm->fw->dump_excl); i++) {
1005		u32 start, end;
1006
1007		if (!excl[i].addr || !excl[i].size)
1008			continue;
1009
1010		start = excl[i].addr;
1011		end = start + excl[i].size;
1012
1013		if (end <= mem_addr || start >= mem_addr + buflen)
1014			continue;
1015
1016		if (start < mem_addr)
1017			start = mem_addr;
1018
1019		if (end > mem_addr + buflen)
1020			end = mem_addr + buflen;
1021
1022		memset((u8 *)mem + start - mem_addr, 0xAA, end - start);
1023	}
1024}
1025
1026static const struct iwl_dump_sanitize_ops iwl_mvm_sanitize_ops = {
1027	.frob_txf = iwl_mvm_frob_txf,
1028	.frob_hcmd = iwl_mvm_frob_hcmd,
1029	.frob_mem = iwl_mvm_frob_mem,
1030};
1031
1032static void iwl_mvm_me_conn_status(void *priv, const struct iwl_mei_conn_info *conn_info)
1033{
1034	struct iwl_mvm *mvm = priv;
1035	struct iwl_mvm_csme_conn_info *prev_conn_info, *curr_conn_info;
1036
1037	/*
1038	 * This is protected by the guarantee that this function will not be
1039	 * called twice on two different threads
1040	 */
1041	prev_conn_info = rcu_dereference_protected(mvm->csme_conn_info, true);
1042
1043	curr_conn_info = kzalloc(sizeof(*curr_conn_info), GFP_KERNEL);
1044	if (!curr_conn_info)
1045		return;
1046
1047	curr_conn_info->conn_info = *conn_info;
1048
1049	rcu_assign_pointer(mvm->csme_conn_info, curr_conn_info);
1050
1051	if (prev_conn_info)
1052		kfree_rcu(prev_conn_info, rcu_head);
1053}
1054
1055static void iwl_mvm_mei_rfkill(void *priv, bool blocked,
1056			       bool csme_taking_ownership)
1057{
1058	struct iwl_mvm *mvm = priv;
1059
1060	if (blocked && !csme_taking_ownership)
1061		return;
1062
1063	mvm->mei_rfkill_blocked = blocked;
1064	if (!mvm->hw_registered)
1065		return;
1066
1067	wiphy_rfkill_set_hw_state_reason(mvm->hw->wiphy,
1068					 mvm->mei_rfkill_blocked,
1069					 RFKILL_HARD_BLOCK_NOT_OWNER);
1070}
1071
1072static void iwl_mvm_mei_roaming_forbidden(void *priv, bool forbidden)
1073{
1074	struct iwl_mvm *mvm = priv;
1075
1076	if (!mvm->hw_registered || !mvm->csme_vif)
1077		return;
1078
1079	iwl_mvm_send_roaming_forbidden_event(mvm, mvm->csme_vif, forbidden);
1080}
1081
1082static void iwl_mvm_sap_connected_wk(struct work_struct *wk)
1083{
1084	struct iwl_mvm *mvm =
1085		container_of(wk, struct iwl_mvm, sap_connected_wk);
1086	int ret;
1087
1088	ret = iwl_mvm_start_get_nvm(mvm);
1089	if (ret)
1090		goto out_free;
1091
1092	ret = iwl_mvm_start_post_nvm(mvm);
1093	if (ret)
1094		goto out_free;
1095
1096	return;
1097
1098out_free:
1099	IWL_ERR(mvm, "Couldn't get started...\n");
1100	iwl_mei_start_unregister();
1101	iwl_mei_unregister_complete();
1102	iwl_fw_flush_dumps(&mvm->fwrt);
1103	iwl_mvm_thermal_exit(mvm);
1104	iwl_fw_runtime_free(&mvm->fwrt);
1105	iwl_phy_db_free(mvm->phy_db);
1106	kfree(mvm->scan_cmd);
1107	iwl_trans_op_mode_leave(mvm->trans);
1108	kfree(mvm->nvm_data);
1109	kfree(mvm->mei_nvm_data);
1110
1111	ieee80211_free_hw(mvm->hw);
1112}
1113
1114static void iwl_mvm_mei_sap_connected(void *priv)
1115{
1116	struct iwl_mvm *mvm = priv;
1117
1118	if (!mvm->hw_registered)
1119		schedule_work(&mvm->sap_connected_wk);
1120}
1121
1122static void iwl_mvm_mei_nic_stolen(void *priv)
1123{
1124	struct iwl_mvm *mvm = priv;
1125
1126	rtnl_lock();
1127	cfg80211_shutdown_all_interfaces(mvm->hw->wiphy);
1128	rtnl_unlock();
1129}
1130
1131static const struct iwl_mei_ops mei_ops = {
1132	.me_conn_status = iwl_mvm_me_conn_status,
1133	.rfkill = iwl_mvm_mei_rfkill,
1134	.roaming_forbidden = iwl_mvm_mei_roaming_forbidden,
1135	.sap_connected = iwl_mvm_mei_sap_connected,
1136	.nic_stolen = iwl_mvm_mei_nic_stolen,
1137};
1138
1139static struct iwl_op_mode *
1140iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
1141		      const struct iwl_fw *fw, struct dentry *dbgfs_dir)
1142{
1143	struct ieee80211_hw *hw;
1144	struct iwl_op_mode *op_mode;
1145	struct iwl_mvm *mvm;
1146	struct iwl_trans_config trans_cfg = {};
1147	static const u8 no_reclaim_cmds[] = {
1148		TX_CMD,
1149	};
1150	u32 max_agg;
1151	size_t scan_size;
1152	u32 min_backoff;
1153	struct iwl_mvm_csme_conn_info *csme_conn_info __maybe_unused;
1154
1155	/*
1156	 * We use IWL_MVM_STATION_COUNT_MAX to check the validity of the station
1157	 * index all over the driver - check that its value corresponds to the
1158	 * array size.
1159	 */
1160	BUILD_BUG_ON(ARRAY_SIZE(mvm->fw_id_to_mac_id) !=
1161		     IWL_MVM_STATION_COUNT_MAX);
1162
1163	/********************************
1164	 * 1. Allocating and configuring HW data
1165	 ********************************/
1166	hw = ieee80211_alloc_hw(sizeof(struct iwl_op_mode) +
1167				sizeof(struct iwl_mvm),
1168				iwl_mvm_has_mld_api(fw) ? &iwl_mvm_mld_hw_ops :
1169				&iwl_mvm_hw_ops);
1170	if (!hw)
1171		return NULL;
1172
1173	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
1174		max_agg = 512;
1175	else
1176		max_agg = IEEE80211_MAX_AMPDU_BUF_HE;
1177
1178	hw->max_rx_aggregation_subframes = max_agg;
1179
1180	if (cfg->max_tx_agg_size)
1181		hw->max_tx_aggregation_subframes = cfg->max_tx_agg_size;
1182	else
1183		hw->max_tx_aggregation_subframes = max_agg;
1184
1185	op_mode = hw->priv;
1186
1187	mvm = IWL_OP_MODE_GET_MVM(op_mode);
1188	mvm->dev = trans->dev;
1189	mvm->trans = trans;
1190	mvm->cfg = cfg;
1191	mvm->fw = fw;
1192	mvm->hw = hw;
1193
1194	iwl_fw_runtime_init(&mvm->fwrt, trans, fw, &iwl_mvm_fwrt_ops, mvm,
1195			    &iwl_mvm_sanitize_ops, mvm, dbgfs_dir);
1196
1197	iwl_mvm_get_acpi_tables(mvm);
1198	iwl_uefi_get_sgom_table(trans, &mvm->fwrt);
1199	iwl_uefi_get_step_table(trans);
1200
1201	mvm->init_status = 0;
1202
1203	if (iwl_mvm_has_new_rx_api(mvm)) {
1204		op_mode->ops = &iwl_mvm_ops_mq;
1205		trans->rx_mpdu_cmd_hdr_size =
1206			(trans->trans_cfg->device_family >=
1207			 IWL_DEVICE_FAMILY_AX210) ?
1208			sizeof(struct iwl_rx_mpdu_desc) :
1209			IWL_RX_DESC_SIZE_V1;
1210	} else {
1211		op_mode->ops = &iwl_mvm_ops;
1212		trans->rx_mpdu_cmd_hdr_size =
1213			sizeof(struct iwl_rx_mpdu_res_start);
1214
1215		if (WARN_ON(trans->num_rx_queues > 1))
1216			goto out_free;
1217	}
1218
1219	mvm->fw_restart = iwlwifi_mod_params.fw_restart ? -1 : 0;
1220
1221	if (iwl_mvm_has_new_tx_api(mvm)) {
1222		/*
1223		 * If we have the new TX/queue allocation API initialize them
1224		 * all to invalid numbers. We'll rewrite the ones that we need
1225		 * later, but that doesn't happen for all of them all of the
1226		 * time (e.g. P2P Device is optional), and if a dynamic queue
1227		 * ends up getting number 2 (IWL_MVM_DQA_P2P_DEVICE_QUEUE) then
1228		 * iwl_mvm_is_static_queue() erroneously returns true, and we
1229		 * might have things getting stuck.
1230		 */
1231		mvm->aux_queue = IWL_MVM_INVALID_QUEUE;
1232		mvm->snif_queue = IWL_MVM_INVALID_QUEUE;
1233		mvm->probe_queue = IWL_MVM_INVALID_QUEUE;
1234		mvm->p2p_dev_queue = IWL_MVM_INVALID_QUEUE;
1235	} else {
1236		mvm->aux_queue = IWL_MVM_DQA_AUX_QUEUE;
1237		mvm->snif_queue = IWL_MVM_DQA_INJECT_MONITOR_QUEUE;
1238		mvm->probe_queue = IWL_MVM_DQA_AP_PROBE_RESP_QUEUE;
1239		mvm->p2p_dev_queue = IWL_MVM_DQA_P2P_DEVICE_QUEUE;
1240	}
1241
1242	mvm->sf_state = SF_UNINIT;
1243	if (iwl_mvm_has_unified_ucode(mvm))
1244		iwl_fw_set_current_image(&mvm->fwrt, IWL_UCODE_REGULAR);
1245	else
1246		iwl_fw_set_current_image(&mvm->fwrt, IWL_UCODE_INIT);
1247	mvm->drop_bcn_ap_mode = true;
1248
1249	mutex_init(&mvm->mutex);
1250	spin_lock_init(&mvm->async_handlers_lock);
1251	INIT_LIST_HEAD(&mvm->time_event_list);
1252	INIT_LIST_HEAD(&mvm->aux_roc_te_list);
1253	INIT_LIST_HEAD(&mvm->async_handlers_list);
1254	spin_lock_init(&mvm->time_event_lock);
1255	INIT_LIST_HEAD(&mvm->ftm_initiator.loc_list);
1256	INIT_LIST_HEAD(&mvm->ftm_initiator.pasn_list);
1257	INIT_LIST_HEAD(&mvm->resp_pasn_list);
1258
1259	INIT_WORK(&mvm->async_handlers_wk, iwl_mvm_async_handlers_wk);
1260	INIT_WORK(&mvm->roc_done_wk, iwl_mvm_roc_done_wk);
1261	INIT_WORK(&mvm->sap_connected_wk, iwl_mvm_sap_connected_wk);
1262	INIT_DELAYED_WORK(&mvm->tdls_cs.dwork, iwl_mvm_tdls_ch_switch_work);
1263	INIT_DELAYED_WORK(&mvm->scan_timeout_dwork, iwl_mvm_scan_timeout_wk);
1264	INIT_WORK(&mvm->add_stream_wk, iwl_mvm_add_new_dqa_stream_wk);
1265	INIT_LIST_HEAD(&mvm->add_stream_txqs);
1266	spin_lock_init(&mvm->add_stream_lock);
1267
1268	init_waitqueue_head(&mvm->rx_sync_waitq);
1269
1270	mvm->queue_sync_state = 0;
1271
1272	SET_IEEE80211_DEV(mvm->hw, mvm->trans->dev);
1273
1274	spin_lock_init(&mvm->tcm.lock);
1275	INIT_DELAYED_WORK(&mvm->tcm.work, iwl_mvm_tcm_work);
1276	mvm->tcm.ts = jiffies;
1277	mvm->tcm.ll_ts = jiffies;
1278	mvm->tcm.uapsd_nonagg_ts = jiffies;
1279
1280	INIT_DELAYED_WORK(&mvm->cs_tx_unblock_dwork, iwl_mvm_tx_unblock_dwork);
1281
1282	mvm->cmd_ver.range_resp =
1283		iwl_fw_lookup_notif_ver(mvm->fw, LOCATION_GROUP,
1284					TOF_RANGE_RESPONSE_NOTIF, 5);
1285	/* we only support up to version 9 */
1286	if (WARN_ON_ONCE(mvm->cmd_ver.range_resp > 9))
1287		goto out_free;
1288
1289	/*
1290	 * Populate the state variables that the transport layer needs
1291	 * to know about.
1292	 */
1293	trans_cfg.op_mode = op_mode;
1294	trans_cfg.no_reclaim_cmds = no_reclaim_cmds;
1295	trans_cfg.n_no_reclaim_cmds = ARRAY_SIZE(no_reclaim_cmds);
1296
1297	switch (iwlwifi_mod_params.amsdu_size) {
1298	case IWL_AMSDU_DEF:
1299		trans_cfg.rx_buf_size = IWL_AMSDU_4K;
1300		break;
1301	case IWL_AMSDU_4K:
1302		trans_cfg.rx_buf_size = IWL_AMSDU_4K;
1303		break;
1304	case IWL_AMSDU_8K:
1305		trans_cfg.rx_buf_size = IWL_AMSDU_8K;
1306		break;
1307	case IWL_AMSDU_12K:
1308		trans_cfg.rx_buf_size = IWL_AMSDU_12K;
1309		break;
1310	default:
1311		pr_err("%s: Unsupported amsdu_size: %d\n", KBUILD_MODNAME,
1312		       iwlwifi_mod_params.amsdu_size);
1313		trans_cfg.rx_buf_size = IWL_AMSDU_4K;
1314	}
1315
1316	trans->wide_cmd_header = true;
1317	trans_cfg.bc_table_dword =
1318		mvm->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210;
1319
1320	trans_cfg.command_groups = iwl_mvm_groups;
1321	trans_cfg.command_groups_size = ARRAY_SIZE(iwl_mvm_groups);
1322
1323	trans_cfg.cmd_queue = IWL_MVM_DQA_CMD_QUEUE;
1324	trans_cfg.cmd_fifo = IWL_MVM_TX_FIFO_CMD;
1325	trans_cfg.scd_set_active = true;
1326
1327	trans_cfg.cb_data_offs = offsetof(struct ieee80211_tx_info,
1328					  driver_data[2]);
1329
1330	/* Set a short watchdog for the command queue */
1331	trans_cfg.cmd_q_wdg_timeout =
1332		iwl_mvm_get_wd_timeout(mvm, NULL, false, true);
1333
1334	snprintf(mvm->hw->wiphy->fw_version,
1335		 sizeof(mvm->hw->wiphy->fw_version),
1336		 "%.31s", fw->fw_version);
1337
1338	trans_cfg.fw_reset_handshake = fw_has_capa(&mvm->fw->ucode_capa,
1339						   IWL_UCODE_TLV_CAPA_FW_RESET_HANDSHAKE);
1340
1341	trans_cfg.queue_alloc_cmd_ver =
1342		iwl_fw_lookup_cmd_ver(mvm->fw,
1343				      WIDE_ID(DATA_PATH_GROUP,
1344					      SCD_QUEUE_CONFIG_CMD),
1345				      0);
1346	mvm->sta_remove_requires_queue_remove =
1347		trans_cfg.queue_alloc_cmd_ver > 0;
1348
1349	mvm->mld_api_is_used = iwl_mvm_has_mld_api(mvm->fw);
1350
1351	/* Configure transport layer */
1352	iwl_trans_configure(mvm->trans, &trans_cfg);
1353
1354	trans->rx_mpdu_cmd = REPLY_RX_MPDU_CMD;
1355	trans->dbg.dest_tlv = mvm->fw->dbg.dest_tlv;
1356	trans->dbg.n_dest_reg = mvm->fw->dbg.n_dest_reg;
1357	memcpy(trans->dbg.conf_tlv, mvm->fw->dbg.conf_tlv,
1358	       sizeof(trans->dbg.conf_tlv));
1359	trans->dbg.trigger_tlv = mvm->fw->dbg.trigger_tlv;
1360
1361	trans->iml = mvm->fw->iml;
1362	trans->iml_len = mvm->fw->iml_len;
1363
1364	/* set up notification wait support */
1365	iwl_notification_wait_init(&mvm->notif_wait);
1366
1367	/* Init phy db */
1368	mvm->phy_db = iwl_phy_db_init(trans);
1369	if (!mvm->phy_db) {
1370		IWL_ERR(mvm, "Cannot init phy_db\n");
1371		goto out_free;
1372	}
1373
1374	IWL_INFO(mvm, "Detected %s, REV=0x%X\n",
1375		 mvm->trans->name, mvm->trans->hw_rev);
1376
1377	if (iwlwifi_mod_params.nvm_file)
1378		mvm->nvm_file_name = iwlwifi_mod_params.nvm_file;
1379	else
1380		IWL_DEBUG_EEPROM(mvm->trans->dev,
1381				 "working without external nvm file\n");
1382
1383	scan_size = iwl_mvm_scan_size(mvm);
1384
1385	mvm->scan_cmd = kmalloc(scan_size, GFP_KERNEL);
1386	if (!mvm->scan_cmd)
1387		goto out_free;
1388	mvm->scan_cmd_size = scan_size;
1389
1390	/* invalidate ids to prevent accidental removal of sta_id 0 */
1391	mvm->aux_sta.sta_id = IWL_MVM_INVALID_STA;
1392	mvm->snif_sta.sta_id = IWL_MVM_INVALID_STA;
1393
1394	/* Set EBS as successful as long as not stated otherwise by the FW. */
1395	mvm->last_ebs_successful = true;
1396
1397	min_backoff = iwl_mvm_min_backoff(mvm);
1398	iwl_mvm_thermal_initialize(mvm, min_backoff);
1399
1400	if (!iwl_mvm_has_new_rx_stats_api(mvm))
1401		memset(&mvm->rx_stats_v3, 0,
1402		       sizeof(struct mvm_statistics_rx_v3));
1403	else
1404		memset(&mvm->rx_stats, 0, sizeof(struct mvm_statistics_rx));
1405
1406	iwl_mvm_ftm_initiator_smooth_config(mvm);
1407
1408	iwl_mvm_init_time_sync(&mvm->time_sync);
1409
1410	mvm->debugfs_dir = dbgfs_dir;
1411
1412	mvm->mei_registered = !iwl_mei_register(mvm, &mei_ops);
1413
1414	iwl_mvm_mei_scan_filter_init(&mvm->mei_scan_filter);
1415
1416	if (iwl_mvm_start_get_nvm(mvm)) {
1417		/*
1418		 * Getting NVM failed while CSME is the owner, but we are
1419		 * registered to MEI, we'll get the NVM later when it'll be
1420		 * possible to get it from CSME.
1421		 */
1422		if (trans->csme_own && mvm->mei_registered)
1423			return op_mode;
1424
1425		goto out_thermal_exit;
1426	}
1427
1428
1429	if (iwl_mvm_start_post_nvm(mvm))
1430		goto out_thermal_exit;
1431
1432	return op_mode;
1433
1434 out_thermal_exit:
1435	iwl_mvm_thermal_exit(mvm);
1436	if (mvm->mei_registered) {
1437		iwl_mei_start_unregister();
1438		iwl_mei_unregister_complete();
1439	}
1440 out_free:
1441	iwl_fw_flush_dumps(&mvm->fwrt);
1442	iwl_fw_runtime_free(&mvm->fwrt);
1443
1444	if (iwlmvm_mod_params.init_dbg)
1445		return op_mode;
1446	iwl_phy_db_free(mvm->phy_db);
1447	kfree(mvm->scan_cmd);
1448	iwl_trans_op_mode_leave(trans);
1449
1450	ieee80211_free_hw(mvm->hw);
1451	return NULL;
1452}
1453
1454void iwl_mvm_stop_device(struct iwl_mvm *mvm)
1455{
1456	lockdep_assert_held(&mvm->mutex);
1457
1458	iwl_fw_cancel_timestamp(&mvm->fwrt);
1459
1460	clear_bit(IWL_MVM_STATUS_FIRMWARE_RUNNING, &mvm->status);
1461
1462	iwl_fw_dbg_stop_sync(&mvm->fwrt);
1463	iwl_trans_stop_device(mvm->trans);
1464	iwl_free_fw_paging(&mvm->fwrt);
1465	iwl_fw_dump_conf_clear(&mvm->fwrt);
1466	iwl_mvm_mei_device_state(mvm, false);
1467}
1468
1469static void iwl_op_mode_mvm_stop(struct iwl_op_mode *op_mode)
1470{
1471	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
1472	int i;
1473
1474	if (mvm->mei_registered) {
1475		rtnl_lock();
1476		iwl_mei_set_netdev(NULL);
1477		rtnl_unlock();
1478		iwl_mei_start_unregister();
1479	}
1480
1481	/*
1482	 * After we unregister from mei, the worker can't be scheduled
1483	 * anymore.
1484	 */
1485	cancel_work_sync(&mvm->sap_connected_wk);
1486
1487	iwl_mvm_leds_exit(mvm);
1488
1489	iwl_mvm_thermal_exit(mvm);
1490
1491	/*
1492	 * If we couldn't get ownership on the device and we couldn't
1493	 * get the NVM from CSME, we haven't registered to mac80211.
1494	 * In that case, we didn't fail op_mode_start, because we are
1495	 * waiting for CSME to allow us to get the NVM to register to
1496	 * mac80211. If that didn't happen, we haven't registered to
1497	 * mac80211, hence the if below.
1498	 */
1499	if (mvm->hw_registered)
1500		ieee80211_unregister_hw(mvm->hw);
1501
1502	kfree(mvm->scan_cmd);
1503	kfree(mvm->mcast_filter_cmd);
1504	mvm->mcast_filter_cmd = NULL;
1505
1506	kfree(mvm->error_recovery_buf);
1507	mvm->error_recovery_buf = NULL;
1508
1509	iwl_mvm_ptp_remove(mvm);
1510
1511	iwl_trans_op_mode_leave(mvm->trans);
1512
1513	iwl_phy_db_free(mvm->phy_db);
1514	mvm->phy_db = NULL;
1515
1516	kfree(mvm->nvm_data);
1517	kfree(mvm->mei_nvm_data);
1518	kfree(rcu_access_pointer(mvm->csme_conn_info));
1519	kfree(mvm->temp_nvm_data);
1520	for (i = 0; i < NVM_MAX_NUM_SECTIONS; i++)
1521		kfree(mvm->nvm_sections[i].data);
1522
1523	cancel_delayed_work_sync(&mvm->tcm.work);
1524
1525	iwl_fw_runtime_free(&mvm->fwrt);
1526	mutex_destroy(&mvm->mutex);
1527
1528	if (mvm->mei_registered)
1529		iwl_mei_unregister_complete();
1530
1531	ieee80211_free_hw(mvm->hw);
1532}
1533
1534struct iwl_async_handler_entry {
1535	struct list_head list;
1536	struct iwl_rx_cmd_buffer rxb;
1537	enum iwl_rx_handler_context context;
1538	void (*fn)(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb);
1539};
1540
1541void iwl_mvm_async_handlers_purge(struct iwl_mvm *mvm)
1542{
1543	struct iwl_async_handler_entry *entry, *tmp;
1544
1545	spin_lock_bh(&mvm->async_handlers_lock);
1546	list_for_each_entry_safe(entry, tmp, &mvm->async_handlers_list, list) {
1547		iwl_free_rxb(&entry->rxb);
1548		list_del(&entry->list);
1549		kfree(entry);
1550	}
1551	spin_unlock_bh(&mvm->async_handlers_lock);
1552}
1553
1554static void iwl_mvm_async_handlers_wk(struct work_struct *wk)
1555{
1556	struct iwl_mvm *mvm =
1557		container_of(wk, struct iwl_mvm, async_handlers_wk);
1558	struct iwl_async_handler_entry *entry, *tmp;
1559	LIST_HEAD(local_list);
1560
1561	/* Ensure that we are not in stop flow (check iwl_mvm_mac_stop) */
1562
1563	/*
1564	 * Sync with Rx path with a lock. Remove all the entries from this list,
1565	 * add them to a local one (lock free), and then handle them.
1566	 */
1567	spin_lock_bh(&mvm->async_handlers_lock);
1568	list_splice_init(&mvm->async_handlers_list, &local_list);
1569	spin_unlock_bh(&mvm->async_handlers_lock);
1570
1571	list_for_each_entry_safe(entry, tmp, &local_list, list) {
1572		if (entry->context == RX_HANDLER_ASYNC_LOCKED)
1573			mutex_lock(&mvm->mutex);
1574		entry->fn(mvm, &entry->rxb);
1575		iwl_free_rxb(&entry->rxb);
1576		list_del(&entry->list);
1577		if (entry->context == RX_HANDLER_ASYNC_LOCKED)
1578			mutex_unlock(&mvm->mutex);
1579		kfree(entry);
1580	}
1581}
1582
1583static inline void iwl_mvm_rx_check_trigger(struct iwl_mvm *mvm,
1584					    struct iwl_rx_packet *pkt)
1585{
1586	struct iwl_fw_dbg_trigger_tlv *trig;
1587	struct iwl_fw_dbg_trigger_cmd *cmds_trig;
1588	int i;
1589
1590	trig = iwl_fw_dbg_trigger_on(&mvm->fwrt, NULL,
1591				     FW_DBG_TRIGGER_FW_NOTIF);
1592	if (!trig)
1593		return;
1594
1595	cmds_trig = (void *)trig->data;
1596
1597	for (i = 0; i < ARRAY_SIZE(cmds_trig->cmds); i++) {
1598		/* don't collect on CMD 0 */
1599		if (!cmds_trig->cmds[i].cmd_id)
1600			break;
1601
1602		if (cmds_trig->cmds[i].cmd_id != pkt->hdr.cmd ||
1603		    cmds_trig->cmds[i].group_id != pkt->hdr.group_id)
1604			continue;
1605
1606		iwl_fw_dbg_collect_trig(&mvm->fwrt, trig,
1607					"CMD 0x%02x.%02x received",
1608					pkt->hdr.group_id, pkt->hdr.cmd);
1609		break;
1610	}
1611}
1612
1613static void iwl_mvm_rx_common(struct iwl_mvm *mvm,
1614			      struct iwl_rx_cmd_buffer *rxb,
1615			      struct iwl_rx_packet *pkt)
1616{
1617	unsigned int pkt_len = iwl_rx_packet_payload_len(pkt);
1618	int i;
1619	union iwl_dbg_tlv_tp_data tp_data = { .fw_pkt = pkt };
1620
1621	iwl_dbg_tlv_time_point(&mvm->fwrt,
1622			       IWL_FW_INI_TIME_POINT_FW_RSP_OR_NOTIF, &tp_data);
1623	iwl_mvm_rx_check_trigger(mvm, pkt);
1624
1625	/*
1626	 * Do the notification wait before RX handlers so
1627	 * even if the RX handler consumes the RXB we have
1628	 * access to it in the notification wait entry.
1629	 */
1630	iwl_notification_wait_notify(&mvm->notif_wait, pkt);
1631
1632	for (i = 0; i < ARRAY_SIZE(iwl_mvm_rx_handlers); i++) {
1633		const struct iwl_rx_handlers *rx_h = &iwl_mvm_rx_handlers[i];
1634		struct iwl_async_handler_entry *entry;
1635
1636		if (rx_h->cmd_id != WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd))
1637			continue;
1638
1639		if (IWL_FW_CHECK(mvm, pkt_len < rx_h->min_size,
1640				 "unexpected notification 0x%04x size %d, need %d\n",
1641				 rx_h->cmd_id, pkt_len, rx_h->min_size))
1642			return;
1643
1644		if (rx_h->context == RX_HANDLER_SYNC) {
1645			rx_h->fn(mvm, rxb);
1646			return;
1647		}
1648
1649		entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
1650		/* we can't do much... */
1651		if (!entry)
1652			return;
1653
1654		entry->rxb._page = rxb_steal_page(rxb);
1655		entry->rxb._offset = rxb->_offset;
1656		entry->rxb._rx_page_order = rxb->_rx_page_order;
1657		entry->fn = rx_h->fn;
1658		entry->context = rx_h->context;
1659		spin_lock(&mvm->async_handlers_lock);
1660		list_add_tail(&entry->list, &mvm->async_handlers_list);
1661		spin_unlock(&mvm->async_handlers_lock);
1662		schedule_work(&mvm->async_handlers_wk);
1663		break;
1664	}
1665}
1666
1667static void iwl_mvm_rx(struct iwl_op_mode *op_mode,
1668		       struct napi_struct *napi,
1669		       struct iwl_rx_cmd_buffer *rxb)
1670{
1671	struct iwl_rx_packet *pkt = rxb_addr(rxb);
1672	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
1673	u16 cmd = WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd);
1674
1675	if (likely(cmd == WIDE_ID(LEGACY_GROUP, REPLY_RX_MPDU_CMD)))
1676		iwl_mvm_rx_rx_mpdu(mvm, napi, rxb);
1677	else if (cmd == WIDE_ID(LEGACY_GROUP, REPLY_RX_PHY_CMD))
1678		iwl_mvm_rx_rx_phy_cmd(mvm, rxb);
1679	else
1680		iwl_mvm_rx_common(mvm, rxb, pkt);
1681}
1682
1683void iwl_mvm_rx_mq(struct iwl_op_mode *op_mode,
1684		   struct napi_struct *napi,
1685		   struct iwl_rx_cmd_buffer *rxb)
1686{
1687	struct iwl_rx_packet *pkt = rxb_addr(rxb);
1688	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
1689	u16 cmd = WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd);
1690
1691	if (likely(cmd == WIDE_ID(LEGACY_GROUP, REPLY_RX_MPDU_CMD)))
1692		iwl_mvm_rx_mpdu_mq(mvm, napi, rxb, 0);
1693	else if (unlikely(cmd == WIDE_ID(DATA_PATH_GROUP,
1694					 RX_QUEUES_NOTIFICATION)))
1695		iwl_mvm_rx_queue_notif(mvm, napi, rxb, 0);
1696	else if (cmd == WIDE_ID(LEGACY_GROUP, FRAME_RELEASE))
1697		iwl_mvm_rx_frame_release(mvm, napi, rxb, 0);
1698	else if (cmd == WIDE_ID(LEGACY_GROUP, BAR_FRAME_RELEASE))
1699		iwl_mvm_rx_bar_frame_release(mvm, napi, rxb, 0);
1700	else if (cmd == WIDE_ID(DATA_PATH_GROUP, RX_NO_DATA_NOTIF))
1701		iwl_mvm_rx_monitor_no_data(mvm, napi, rxb, 0);
1702	else
1703		iwl_mvm_rx_common(mvm, rxb, pkt);
1704}
1705
 
 
 
 
 
 
 
 
 
 
 
 
1706static int iwl_mvm_is_static_queue(struct iwl_mvm *mvm, int queue)
1707{
1708	return queue == mvm->aux_queue || queue == mvm->probe_queue ||
1709		queue == mvm->p2p_dev_queue || queue == mvm->snif_queue;
1710}
1711
1712static void iwl_mvm_queue_state_change(struct iwl_op_mode *op_mode,
1713				       int hw_queue, bool start)
1714{
1715	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
1716	struct ieee80211_sta *sta;
1717	struct ieee80211_txq *txq;
1718	struct iwl_mvm_txq *mvmtxq;
1719	int i;
1720	unsigned long tid_bitmap;
1721	struct iwl_mvm_sta *mvmsta;
1722	u8 sta_id;
1723
1724	sta_id = iwl_mvm_has_new_tx_api(mvm) ?
1725		mvm->tvqm_info[hw_queue].sta_id :
1726		mvm->queue_info[hw_queue].ra_sta_id;
1727
1728	if (WARN_ON_ONCE(sta_id >= mvm->fw->ucode_capa.num_stations))
1729		return;
1730
1731	rcu_read_lock();
1732
1733	sta = rcu_dereference(mvm->fw_id_to_mac_id[sta_id]);
1734	if (IS_ERR_OR_NULL(sta))
1735		goto out;
1736	mvmsta = iwl_mvm_sta_from_mac80211(sta);
1737
1738	if (iwl_mvm_is_static_queue(mvm, hw_queue)) {
1739		if (!start)
1740			ieee80211_stop_queues(mvm->hw);
1741		else if (mvmsta->sta_state != IEEE80211_STA_NOTEXIST)
1742			ieee80211_wake_queues(mvm->hw);
1743
1744		goto out;
1745	}
1746
1747	if (iwl_mvm_has_new_tx_api(mvm)) {
1748		int tid = mvm->tvqm_info[hw_queue].txq_tid;
1749
1750		tid_bitmap = BIT(tid);
1751	} else {
1752		tid_bitmap = mvm->queue_info[hw_queue].tid_bitmap;
1753	}
1754
1755	for_each_set_bit(i, &tid_bitmap, IWL_MAX_TID_COUNT + 1) {
1756		int tid = i;
1757
1758		if (tid == IWL_MAX_TID_COUNT)
1759			tid = IEEE80211_NUM_TIDS;
1760
1761		txq = sta->txq[tid];
1762		mvmtxq = iwl_mvm_txq_from_mac80211(txq);
1763		if (start)
1764			clear_bit(IWL_MVM_TXQ_STATE_STOP_FULL, &mvmtxq->state);
1765		else
1766			set_bit(IWL_MVM_TXQ_STATE_STOP_FULL, &mvmtxq->state);
1767
1768		if (start && mvmsta->sta_state != IEEE80211_STA_NOTEXIST) {
1769			local_bh_disable();
1770			iwl_mvm_mac_itxq_xmit(mvm->hw, txq);
1771			local_bh_enable();
1772		}
1773	}
1774
1775out:
1776	rcu_read_unlock();
1777}
1778
1779static void iwl_mvm_stop_sw_queue(struct iwl_op_mode *op_mode, int hw_queue)
1780{
1781	iwl_mvm_queue_state_change(op_mode, hw_queue, false);
1782}
1783
1784static void iwl_mvm_wake_sw_queue(struct iwl_op_mode *op_mode, int hw_queue)
1785{
1786	iwl_mvm_queue_state_change(op_mode, hw_queue, true);
1787}
1788
1789static void iwl_mvm_set_rfkill_state(struct iwl_mvm *mvm)
1790{
1791	bool state = iwl_mvm_is_radio_killed(mvm);
1792
1793	if (state)
1794		wake_up(&mvm->rx_sync_waitq);
1795
1796	wiphy_rfkill_set_hw_state(mvm->hw->wiphy, state);
1797}
1798
1799void iwl_mvm_set_hw_ctkill_state(struct iwl_mvm *mvm, bool state)
1800{
1801	if (state)
1802		set_bit(IWL_MVM_STATUS_HW_CTKILL, &mvm->status);
1803	else
1804		clear_bit(IWL_MVM_STATUS_HW_CTKILL, &mvm->status);
1805
1806	iwl_mvm_set_rfkill_state(mvm);
1807}
1808
1809struct iwl_mvm_csme_conn_info *iwl_mvm_get_csme_conn_info(struct iwl_mvm *mvm)
1810{
1811	return rcu_dereference_protected(mvm->csme_conn_info,
1812					 lockdep_is_held(&mvm->mutex));
1813}
1814
1815static bool iwl_mvm_set_hw_rfkill_state(struct iwl_op_mode *op_mode, bool state)
1816{
1817	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
1818	bool rfkill_safe_init_done = READ_ONCE(mvm->rfkill_safe_init_done);
1819	bool unified = iwl_mvm_has_unified_ucode(mvm);
1820
1821	if (state)
1822		set_bit(IWL_MVM_STATUS_HW_RFKILL, &mvm->status);
1823	else
1824		clear_bit(IWL_MVM_STATUS_HW_RFKILL, &mvm->status);
1825
1826	iwl_mvm_set_rfkill_state(mvm);
1827
1828	 /* iwl_run_init_mvm_ucode is waiting for results, abort it. */
1829	if (rfkill_safe_init_done)
1830		iwl_abort_notification_waits(&mvm->notif_wait);
1831
1832	/*
1833	 * Don't ask the transport to stop the firmware. We'll do it
1834	 * after cfg80211 takes us down.
1835	 */
1836	if (unified)
1837		return false;
1838
1839	/*
1840	 * Stop the device if we run OPERATIONAL firmware or if we are in the
1841	 * middle of the calibrations.
1842	 */
1843	return state && rfkill_safe_init_done;
1844}
1845
1846static void iwl_mvm_free_skb(struct iwl_op_mode *op_mode, struct sk_buff *skb)
1847{
1848	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
1849	struct ieee80211_tx_info *info;
1850
1851	info = IEEE80211_SKB_CB(skb);
1852	iwl_trans_free_tx_cmd(mvm->trans, info->driver_data[1]);
1853	ieee80211_free_txskb(mvm->hw, skb);
1854}
1855
1856struct iwl_mvm_reprobe {
1857	struct device *dev;
1858	struct work_struct work;
1859};
1860
1861static void iwl_mvm_reprobe_wk(struct work_struct *wk)
1862{
1863	struct iwl_mvm_reprobe *reprobe;
1864
1865	reprobe = container_of(wk, struct iwl_mvm_reprobe, work);
1866	if (device_reprobe(reprobe->dev))
1867		dev_err(reprobe->dev, "reprobe failed!\n");
1868	put_device(reprobe->dev);
1869	kfree(reprobe);
1870	module_put(THIS_MODULE);
1871}
1872
1873void iwl_mvm_nic_restart(struct iwl_mvm *mvm, bool fw_error)
1874{
1875	iwl_abort_notification_waits(&mvm->notif_wait);
1876	iwl_dbg_tlv_del_timers(mvm->trans);
1877
1878	/*
1879	 * This is a bit racy, but worst case we tell mac80211 about
1880	 * a stopped/aborted scan when that was already done which
1881	 * is not a problem. It is necessary to abort any os scan
1882	 * here because mac80211 requires having the scan cleared
1883	 * before restarting.
1884	 * We'll reset the scan_status to NONE in restart cleanup in
1885	 * the next start() call from mac80211. If restart isn't called
1886	 * (no fw restart) scan status will stay busy.
1887	 */
1888	iwl_mvm_report_scan_aborted(mvm);
1889
1890	/*
1891	 * If we're restarting already, don't cycle restarts.
1892	 * If INIT fw asserted, it will likely fail again.
1893	 * If WoWLAN fw asserted, don't restart either, mac80211
1894	 * can't recover this since we're already half suspended.
1895	 */
1896	if (!mvm->fw_restart && fw_error) {
1897		iwl_fw_error_collect(&mvm->fwrt, false);
1898	} else if (test_bit(IWL_MVM_STATUS_STARTING,
1899			    &mvm->status)) {
1900		IWL_ERR(mvm, "Starting mac, retry will be triggered anyway\n");
1901	} else if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status)) {
1902		struct iwl_mvm_reprobe *reprobe;
1903
1904		IWL_ERR(mvm,
1905			"Firmware error during reconfiguration - reprobe!\n");
1906
1907		/*
1908		 * get a module reference to avoid doing this while unloading
1909		 * anyway and to avoid scheduling a work with code that's
1910		 * being removed.
1911		 */
1912		if (!try_module_get(THIS_MODULE)) {
1913			IWL_ERR(mvm, "Module is being unloaded - abort\n");
1914			return;
1915		}
1916
1917		reprobe = kzalloc(sizeof(*reprobe), GFP_ATOMIC);
1918		if (!reprobe) {
1919			module_put(THIS_MODULE);
1920			return;
1921		}
1922		reprobe->dev = get_device(mvm->trans->dev);
1923		INIT_WORK(&reprobe->work, iwl_mvm_reprobe_wk);
1924		schedule_work(&reprobe->work);
1925	} else if (test_bit(IWL_MVM_STATUS_HW_RESTART_REQUESTED,
1926			    &mvm->status)) {
1927		IWL_ERR(mvm, "HW restart already requested, but not started\n");
1928	} else if (mvm->fwrt.cur_fw_img == IWL_UCODE_REGULAR &&
1929		   mvm->hw_registered &&
1930		   !test_bit(STATUS_TRANS_DEAD, &mvm->trans->status)) {
1931		/* This should be first thing before trying to collect any
1932		 * data to avoid endless loops if any HW error happens while
1933		 * collecting debug data.
1934		 */
1935		set_bit(IWL_MVM_STATUS_HW_RESTART_REQUESTED, &mvm->status);
1936
1937		if (mvm->fw->ucode_capa.error_log_size) {
1938			u32 src_size = mvm->fw->ucode_capa.error_log_size;
1939			u32 src_addr = mvm->fw->ucode_capa.error_log_addr;
1940			u8 *recover_buf = kzalloc(src_size, GFP_ATOMIC);
1941
1942			if (recover_buf) {
1943				mvm->error_recovery_buf = recover_buf;
1944				iwl_trans_read_mem_bytes(mvm->trans,
1945							 src_addr,
1946							 recover_buf,
1947							 src_size);
1948			}
1949		}
1950
1951		iwl_fw_error_collect(&mvm->fwrt, false);
1952
1953		if (fw_error && mvm->fw_restart > 0) {
1954			mvm->fw_restart--;
1955			ieee80211_restart_hw(mvm->hw);
1956		} else if (mvm->fwrt.trans->dbg.restart_required) {
1957			IWL_DEBUG_INFO(mvm, "FW restart requested after debug collection\n");
1958			mvm->fwrt.trans->dbg.restart_required = FALSE;
1959			ieee80211_restart_hw(mvm->hw);
1960		} else if (mvm->trans->trans_cfg->device_family <= IWL_DEVICE_FAMILY_8000) {
1961			ieee80211_restart_hw(mvm->hw);
1962		}
1963	}
1964}
1965
1966static void iwl_mvm_nic_error(struct iwl_op_mode *op_mode, bool sync)
1967{
1968	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
1969
 
 
 
1970	if (!test_bit(STATUS_TRANS_DEAD, &mvm->trans->status) &&
1971	    !test_and_clear_bit(IWL_MVM_STATUS_SUPPRESS_ERROR_LOG_ONCE,
1972				&mvm->status))
1973		iwl_mvm_dump_nic_error_log(mvm);
1974
1975	if (sync) {
1976		iwl_fw_error_collect(&mvm->fwrt, true);
1977		/*
1978		 * Currently, the only case for sync=true is during
1979		 * shutdown, so just stop in this case. If/when that
1980		 * changes, we need to be a bit smarter here.
1981		 */
1982		return;
1983	}
1984
1985	/*
1986	 * If the firmware crashes while we're already considering it
1987	 * to be dead then don't ask for a restart, that cannot do
1988	 * anything useful anyway.
1989	 */
1990	if (!test_bit(IWL_MVM_STATUS_FIRMWARE_RUNNING, &mvm->status))
1991		return;
1992
1993	iwl_mvm_nic_restart(mvm, false);
1994}
1995
1996static void iwl_mvm_cmd_queue_full(struct iwl_op_mode *op_mode)
1997{
1998	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
1999
2000	WARN_ON(1);
2001	iwl_mvm_nic_restart(mvm, true);
2002}
2003
2004static void iwl_op_mode_mvm_time_point(struct iwl_op_mode *op_mode,
2005				       enum iwl_fw_ini_time_point tp_id,
2006				       union iwl_dbg_tlv_tp_data *tp_data)
2007{
2008	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
2009
2010	iwl_dbg_tlv_time_point(&mvm->fwrt, tp_id, tp_data);
2011}
2012
2013#define IWL_MVM_COMMON_OPS					\
2014	/* these could be differentiated */			\
 
2015	.queue_full = iwl_mvm_stop_sw_queue,			\
2016	.queue_not_full = iwl_mvm_wake_sw_queue,		\
2017	.hw_rf_kill = iwl_mvm_set_hw_rfkill_state,		\
2018	.free_skb = iwl_mvm_free_skb,				\
2019	.nic_error = iwl_mvm_nic_error,				\
2020	.cmd_queue_full = iwl_mvm_cmd_queue_full,		\
2021	.nic_config = iwl_mvm_nic_config,			\
2022	/* as we only register one, these MUST be common! */	\
2023	.start = iwl_op_mode_mvm_start,				\
2024	.stop = iwl_op_mode_mvm_stop,				\
2025	.time_point = iwl_op_mode_mvm_time_point
2026
2027static const struct iwl_op_mode_ops iwl_mvm_ops = {
2028	IWL_MVM_COMMON_OPS,
2029	.rx = iwl_mvm_rx,
2030};
2031
2032static void iwl_mvm_rx_mq_rss(struct iwl_op_mode *op_mode,
2033			      struct napi_struct *napi,
2034			      struct iwl_rx_cmd_buffer *rxb,
2035			      unsigned int queue)
2036{
2037	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
2038	struct iwl_rx_packet *pkt = rxb_addr(rxb);
2039	u16 cmd = WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd);
2040
2041	if (unlikely(queue >= mvm->trans->num_rx_queues))
2042		return;
2043
2044	if (unlikely(cmd == WIDE_ID(LEGACY_GROUP, FRAME_RELEASE)))
2045		iwl_mvm_rx_frame_release(mvm, napi, rxb, queue);
2046	else if (unlikely(cmd == WIDE_ID(DATA_PATH_GROUP,
2047					 RX_QUEUES_NOTIFICATION)))
2048		iwl_mvm_rx_queue_notif(mvm, napi, rxb, queue);
2049	else if (likely(cmd == WIDE_ID(LEGACY_GROUP, REPLY_RX_MPDU_CMD)))
2050		iwl_mvm_rx_mpdu_mq(mvm, napi, rxb, queue);
2051}
2052
2053static const struct iwl_op_mode_ops iwl_mvm_ops_mq = {
2054	IWL_MVM_COMMON_OPS,
2055	.rx = iwl_mvm_rx_mq,
2056	.rx_rss = iwl_mvm_rx_mq_rss,
2057};