1/*
   2 * This file is subject to the terms and conditions of the GNU General Public
   3 * License.  See the file "COPYING" in the main directory of this archive
   4 * for more details.
   5 *
   6 * Copyright (c) 2008-2009 Silicon Graphics, Inc.  All Rights Reserved.
   7 */
   8
   9/*
  10 * Cross Partition Communication (XPC) sn2-based functions.
  11 *
  12 *     Architecture specific implementation of common functions.
  13 *
  14 */
  15
  16#include <linux/delay.h>
  17#include <linux/slab.h>
  18#include <asm/uncached.h>
  19#include <asm/sn/mspec.h>
  20#include <asm/sn/sn_sal.h>
  21#include "xpc.h"
  22
  23/*
  24 * Define the number of u64s required to represent all the C-brick nasids
  25 * as a bitmap.  The cross-partition kernel modules deal only with
  26 * C-brick nasids, thus the need for bitmaps which don't account for
  27 * odd-numbered (non C-brick) nasids.
  28 */
  29#define XPC_MAX_PHYSNODES_SN2	(MAX_NUMALINK_NODES / 2)
  30#define XP_NASID_MASK_BYTES_SN2	((XPC_MAX_PHYSNODES_SN2 + 7) / 8)
  31#define XP_NASID_MASK_WORDS_SN2	((XPC_MAX_PHYSNODES_SN2 + 63) / 64)
  32
  33/*
  34 * Memory for XPC's amo variables is allocated by the MSPEC driver. These
  35 * pages are located in the lowest granule. The lowest granule uses 4k pages
  36 * for cached references and an alternate TLB handler to never provide a
  37 * cacheable mapping for the entire region. This will prevent speculative
  38 * reading of cached copies of our lines from being issued which will cause
  39 * a PI FSB Protocol error to be generated by the SHUB. For XPC, we need 64
  40 * amo variables (based on XP_MAX_NPARTITIONS_SN2) to identify the senders of
  41 * NOTIFY IRQs, 128 amo variables (based on XP_NASID_MASK_WORDS_SN2) to identify
  42 * the senders of ACTIVATE IRQs, 1 amo variable to identify which remote
  43 * partitions (i.e., XPCs) consider themselves currently engaged with the
  44 * local XPC and 1 amo variable to request partition deactivation.
  45 */
  46#define XPC_NOTIFY_IRQ_AMOS_SN2		0
  47#define XPC_ACTIVATE_IRQ_AMOS_SN2	(XPC_NOTIFY_IRQ_AMOS_SN2 + \
  48					 XP_MAX_NPARTITIONS_SN2)
  49#define XPC_ENGAGED_PARTITIONS_AMO_SN2	(XPC_ACTIVATE_IRQ_AMOS_SN2 + \
  50					 XP_NASID_MASK_WORDS_SN2)
  51#define XPC_DEACTIVATE_REQUEST_AMO_SN2	(XPC_ENGAGED_PARTITIONS_AMO_SN2 + 1)
  52
  53/*
  54 * Buffer used to store a local copy of portions of a remote partition's
  55 * reserved page (either its header and part_nasids mask, or its vars).
  56 */
  57static void *xpc_remote_copy_buffer_base_sn2;
  58static char *xpc_remote_copy_buffer_sn2;
  59
  60static struct xpc_vars_sn2 *xpc_vars_sn2;
  61static struct xpc_vars_part_sn2 *xpc_vars_part_sn2;
  62
  63static int
  64xpc_setup_partitions_sn2(void)
  65{
  66	/* nothing needs to be done */
  67	return 0;
  68}
  69
  70static void
  71xpc_teardown_partitions_sn2(void)
  72{
  73	/* nothing needs to be done */
  74}
  75
  76/* SH_IPI_ACCESS shub register value on startup */
  77static u64 xpc_sh1_IPI_access_sn2;
  78static u64 xpc_sh2_IPI_access0_sn2;
  79static u64 xpc_sh2_IPI_access1_sn2;
  80static u64 xpc_sh2_IPI_access2_sn2;
  81static u64 xpc_sh2_IPI_access3_sn2;
  82
  83/*
  84 * Change protections to allow IPI operations.
  85 */
  86static void
  87xpc_allow_IPI_ops_sn2(void)
  88{
  89	int node;
  90	int nasid;
  91
  92	/* !!! The following should get moved into SAL. */
  93	if (is_shub2()) {
  94		xpc_sh2_IPI_access0_sn2 =
  95		    (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS0));
  96		xpc_sh2_IPI_access1_sn2 =
  97		    (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS1));
  98		xpc_sh2_IPI_access2_sn2 =
  99		    (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS2));
 100		xpc_sh2_IPI_access3_sn2 =
 101		    (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS3));
 102
 103		for_each_online_node(node) {
 104			nasid = cnodeid_to_nasid(node);
 105			HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0),
 106			      -1UL);
 107			HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1),
 108			      -1UL);
 109			HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2),
 110			      -1UL);
 111			HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3),
 112			      -1UL);
 113		}
 114	} else {
 115		xpc_sh1_IPI_access_sn2 =
 116		    (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH1_IPI_ACCESS));
 117
 118		for_each_online_node(node) {
 119			nasid = cnodeid_to_nasid(node);
 120			HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS),
 121			      -1UL);
 122		}
 123	}
 124}
 125
 126/*
 127 * Restrict protections to disallow IPI operations.
 128 */
 129static void
 130xpc_disallow_IPI_ops_sn2(void)
 131{
 132	int node;
 133	int nasid;
 134
 135	/* !!! The following should get moved into SAL. */
 136	if (is_shub2()) {
 137		for_each_online_node(node) {
 138			nasid = cnodeid_to_nasid(node);
 139			HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0),
 140			      xpc_sh2_IPI_access0_sn2);
 141			HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1),
 142			      xpc_sh2_IPI_access1_sn2);
 143			HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2),
 144			      xpc_sh2_IPI_access2_sn2);
 145			HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3),
 146			      xpc_sh2_IPI_access3_sn2);
 147		}
 148	} else {
 149		for_each_online_node(node) {
 150			nasid = cnodeid_to_nasid(node);
 151			HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS),
 152			      xpc_sh1_IPI_access_sn2);
 153		}
 154	}
 155}
 156
 157/*
 158 * The following set of functions are used for the sending and receiving of
 159 * IRQs (also known as IPIs). There are two flavors of IRQs, one that is
 160 * associated with partition activity (SGI_XPC_ACTIVATE) and the other that
 161 * is associated with channel activity (SGI_XPC_NOTIFY).
 162 */
 163
 164static u64
 165xpc_receive_IRQ_amo_sn2(struct amo *amo)
 166{
 167	return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_CLEAR);
 168}
 169
 170static enum xp_retval
 171xpc_send_IRQ_sn2(struct amo *amo, u64 flag, int nasid, int phys_cpuid,
 172		 int vector)
 173{
 174	int ret = 0;
 175	unsigned long irq_flags;
 176
 177	local_irq_save(irq_flags);
 178
 179	FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, flag);
 180	sn_send_IPI_phys(nasid, phys_cpuid, vector, 0);
 181
 182	/*
 183	 * We must always use the nofault function regardless of whether we
 184	 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
 185	 * didn't, we'd never know that the other partition is down and would
 186	 * keep sending IRQs and amos to it until the heartbeat times out.
 187	 */
 188	ret = xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->variable),
 189						     xp_nofault_PIOR_target));
 190
 191	local_irq_restore(irq_flags);
 192
 193	return (ret == 0) ? xpSuccess : xpPioReadError;
 194}
 195
 196static struct amo *
 197xpc_init_IRQ_amo_sn2(int index)
 198{
 199	struct amo *amo = xpc_vars_sn2->amos_page + index;
 200
 201	(void)xpc_receive_IRQ_amo_sn2(amo);	/* clear amo variable */
 202	return amo;
 203}
 204
 205/*
 206 * Functions associated with SGI_XPC_ACTIVATE IRQ.
 207 */
 208
 209/*
 210 * Notify the heartbeat check thread that an activate IRQ has been received.
 211 */
 212static irqreturn_t
 213xpc_handle_activate_IRQ_sn2(int irq, void *dev_id)
 214{
 215	unsigned long irq_flags;
 216
 217	spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
 218	xpc_activate_IRQ_rcvd++;
 219	spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
 220
 221	wake_up_interruptible(&xpc_activate_IRQ_wq);
 222	return IRQ_HANDLED;
 223}
 224
 225/*
 226 * Flag the appropriate amo variable and send an IRQ to the specified node.
 227 */
 228static void
 229xpc_send_activate_IRQ_sn2(unsigned long amos_page_pa, int from_nasid,
 230			  int to_nasid, int to_phys_cpuid)
 231{
 232	struct amo *amos = (struct amo *)__va(amos_page_pa +
 233					      (XPC_ACTIVATE_IRQ_AMOS_SN2 *
 234					      sizeof(struct amo)));
 235
 236	(void)xpc_send_IRQ_sn2(&amos[BIT_WORD(from_nasid / 2)],
 237			       BIT_MASK(from_nasid / 2), to_nasid,
 238			       to_phys_cpuid, SGI_XPC_ACTIVATE);
 239}
 240
 241static void
 242xpc_send_local_activate_IRQ_sn2(int from_nasid)
 243{
 244	unsigned long irq_flags;
 245	struct amo *amos = (struct amo *)__va(xpc_vars_sn2->amos_page_pa +
 246					      (XPC_ACTIVATE_IRQ_AMOS_SN2 *
 247					      sizeof(struct amo)));
 248
 249	/* fake the sending and receipt of an activate IRQ from remote nasid */
 250	FETCHOP_STORE_OP(TO_AMO((u64)&amos[BIT_WORD(from_nasid / 2)].variable),
 251			 FETCHOP_OR, BIT_MASK(from_nasid / 2));
 252
 253	spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
 254	xpc_activate_IRQ_rcvd++;
 255	spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
 256
 257	wake_up_interruptible(&xpc_activate_IRQ_wq);
 258}
 259
 260/*
 261 * Functions associated with SGI_XPC_NOTIFY IRQ.
 262 */
 263
 264/*
 265 * Check to see if any chctl flags were sent from the specified partition.
 266 */
 267static void
 268xpc_check_for_sent_chctl_flags_sn2(struct xpc_partition *part)
 269{
 270	union xpc_channel_ctl_flags chctl;
 271	unsigned long irq_flags;
 272
 273	chctl.all_flags = xpc_receive_IRQ_amo_sn2(part->sn.sn2.
 274						  local_chctl_amo_va);
 275	if (chctl.all_flags == 0)
 276		return;
 277
 278	spin_lock_irqsave(&part->chctl_lock, irq_flags);
 279	part->chctl.all_flags |= chctl.all_flags;
 280	spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
 281
 282	dev_dbg(xpc_chan, "received notify IRQ from partid=%d, chctl.all_flags="
 283		"0x%llx\n", XPC_PARTID(part), chctl.all_flags);
 284
 285	xpc_wakeup_channel_mgr(part);
 286}
 287
 288/*
 289 * Handle the receipt of a SGI_XPC_NOTIFY IRQ by seeing whether the specified
 290 * partition actually sent it. Since SGI_XPC_NOTIFY IRQs may be shared by more
 291 * than one partition, we use an amo structure per partition to indicate
 292 * whether a partition has sent an IRQ or not.  If it has, then wake up the
 293 * associated kthread to handle it.
 294 *
 295 * All SGI_XPC_NOTIFY IRQs received by XPC are the result of IRQs sent by XPC
 296 * running on other partitions.
 297 *
 298 * Noteworthy Arguments:
 299 *
 300 *	irq - Interrupt ReQuest number. NOT USED.
 301 *
 302 *	dev_id - partid of IRQ's potential sender.
 303 */
 304static irqreturn_t
 305xpc_handle_notify_IRQ_sn2(int irq, void *dev_id)
 306{
 307	short partid = (short)(u64)dev_id;
 308	struct xpc_partition *part = &xpc_partitions[partid];
 309
 310	DBUG_ON(partid < 0 || partid >= XP_MAX_NPARTITIONS_SN2);
 311
 312	if (xpc_part_ref(part)) {
 313		xpc_check_for_sent_chctl_flags_sn2(part);
 314
 315		xpc_part_deref(part);
 316	}
 317	return IRQ_HANDLED;
 318}
 319
 320/*
 321 * Check to see if xpc_handle_notify_IRQ_sn2() dropped any IRQs on the floor
 322 * because the write to their associated amo variable completed after the IRQ
 323 * was received.
 324 */
 325static void
 326xpc_check_for_dropped_notify_IRQ_sn2(struct xpc_partition *part)
 327{
 328	struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
 329
 330	if (xpc_part_ref(part)) {
 331		xpc_check_for_sent_chctl_flags_sn2(part);
 332
 333		part_sn2->dropped_notify_IRQ_timer.expires = jiffies +
 334		    XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL;
 335		add_timer(&part_sn2->dropped_notify_IRQ_timer);
 336		xpc_part_deref(part);
 337	}
 338}
 339
 340/*
 341 * Send a notify IRQ to the remote partition that is associated with the
 342 * specified channel.
 343 */
 344static void
 345xpc_send_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag,
 346			char *chctl_flag_string, unsigned long *irq_flags)
 347{
 348	struct xpc_partition *part = &xpc_partitions[ch->partid];
 349	struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
 350	union xpc_channel_ctl_flags chctl = { 0 };
 351	enum xp_retval ret;
 352
 353	if (likely(part->act_state != XPC_P_AS_DEACTIVATING)) {
 354		chctl.flags[ch->number] = chctl_flag;
 355		ret = xpc_send_IRQ_sn2(part_sn2->remote_chctl_amo_va,
 356				       chctl.all_flags,
 357				       part_sn2->notify_IRQ_nasid,
 358				       part_sn2->notify_IRQ_phys_cpuid,
 359				       SGI_XPC_NOTIFY);
 360		dev_dbg(xpc_chan, "%s sent to partid=%d, channel=%d, ret=%d\n",
 361			chctl_flag_string, ch->partid, ch->number, ret);
 362		if (unlikely(ret != xpSuccess)) {
 363			if (irq_flags != NULL)
 364				spin_unlock_irqrestore(&ch->lock, *irq_flags);
 365			XPC_DEACTIVATE_PARTITION(part, ret);
 366			if (irq_flags != NULL)
 367				spin_lock_irqsave(&ch->lock, *irq_flags);
 368		}
 369	}
 370}
 371
 372#define XPC_SEND_NOTIFY_IRQ_SN2(_ch, _ipi_f, _irq_f) \
 373		xpc_send_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f, _irq_f)
 374
 375/*
 376 * Make it look like the remote partition, which is associated with the
 377 * specified channel, sent us a notify IRQ. This faked IRQ will be handled
 378 * by xpc_check_for_dropped_notify_IRQ_sn2().
 379 */
 380static void
 381xpc_send_local_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag,
 382			      char *chctl_flag_string)
 383{
 384	struct xpc_partition *part = &xpc_partitions[ch->partid];
 385	union xpc_channel_ctl_flags chctl = { 0 };
 386
 387	chctl.flags[ch->number] = chctl_flag;
 388	FETCHOP_STORE_OP(TO_AMO((u64)&part->sn.sn2.local_chctl_amo_va->
 389				variable), FETCHOP_OR, chctl.all_flags);
 390	dev_dbg(xpc_chan, "%s sent local from partid=%d, channel=%d\n",
 391		chctl_flag_string, ch->partid, ch->number);
 392}
 393
 394#define XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(_ch, _ipi_f) \
 395		xpc_send_local_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f)
 396
 397static void
 398xpc_send_chctl_closerequest_sn2(struct xpc_channel *ch,
 399				unsigned long *irq_flags)
 400{
 401	struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args;
 402
 403	args->reason = ch->reason;
 404	XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREQUEST, irq_flags);
 405}
 406
 407static void
 408xpc_send_chctl_closereply_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
 409{
 410	XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREPLY, irq_flags);
 411}
 412
 413static void
 414xpc_send_chctl_openrequest_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
 415{
 416	struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args;
 417
 418	args->entry_size = ch->entry_size;
 419	args->local_nentries = ch->local_nentries;
 420	XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREQUEST, irq_flags);
 421}
 422
 423static void
 424xpc_send_chctl_openreply_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
 425{
 426	struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args;
 427
 428	args->remote_nentries = ch->remote_nentries;
 429	args->local_nentries = ch->local_nentries;
 430	args->local_msgqueue_pa = xp_pa(ch->sn.sn2.local_msgqueue);
 431	XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREPLY, irq_flags);
 432}
 433
 434static void
 435xpc_send_chctl_opencomplete_sn2(struct xpc_channel *ch,
 436				unsigned long *irq_flags)
 437{
 438	XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENCOMPLETE, irq_flags);
 439}
 440
 441static void
 442xpc_send_chctl_msgrequest_sn2(struct xpc_channel *ch)
 443{
 444	XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST, NULL);
 445}
 446
 447static void
 448xpc_send_chctl_local_msgrequest_sn2(struct xpc_channel *ch)
 449{
 450	XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST);
 451}
 452
 453static enum xp_retval
 454xpc_save_remote_msgqueue_pa_sn2(struct xpc_channel *ch,
 455				unsigned long msgqueue_pa)
 456{
 457	ch->sn.sn2.remote_msgqueue_pa = msgqueue_pa;
 458	return xpSuccess;
 459}
 460
 461/*
 462 * This next set of functions are used to keep track of when a partition is
 463 * potentially engaged in accessing memory belonging to another partition.
 464 */
 465
 466static void
 467xpc_indicate_partition_engaged_sn2(struct xpc_partition *part)
 468{
 469	unsigned long irq_flags;
 470	struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa +
 471					     (XPC_ENGAGED_PARTITIONS_AMO_SN2 *
 472					     sizeof(struct amo)));
 473
 474	local_irq_save(irq_flags);
 475
 476	/* set bit corresponding to our partid in remote partition's amo */
 477	FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR,
 478			 BIT(sn_partition_id));
 479
 480	/*
 481	 * We must always use the nofault function regardless of whether we
 482	 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
 483	 * didn't, we'd never know that the other partition is down and would
 484	 * keep sending IRQs and amos to it until the heartbeat times out.
 485	 */
 486	(void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
 487							       variable),
 488						     xp_nofault_PIOR_target));
 489
 490	local_irq_restore(irq_flags);
 491}
 492
 493static void
 494xpc_indicate_partition_disengaged_sn2(struct xpc_partition *part)
 495{
 496	struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
 497	unsigned long irq_flags;
 498	struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa +
 499					     (XPC_ENGAGED_PARTITIONS_AMO_SN2 *
 500					     sizeof(struct amo)));
 501
 502	local_irq_save(irq_flags);
 503
 504	/* clear bit corresponding to our partid in remote partition's amo */
 505	FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
 506			 ~BIT(sn_partition_id));
 507
 508	/*
 509	 * We must always use the nofault function regardless of whether we
 510	 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
 511	 * didn't, we'd never know that the other partition is down and would
 512	 * keep sending IRQs and amos to it until the heartbeat times out.
 513	 */
 514	(void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
 515							       variable),
 516						     xp_nofault_PIOR_target));
 517
 518	local_irq_restore(irq_flags);
 519
 520	/*
 521	 * Send activate IRQ to get other side to see that we've cleared our
 522	 * bit in their engaged partitions amo.
 523	 */
 524	xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa,
 525				  cnodeid_to_nasid(0),
 526				  part_sn2->activate_IRQ_nasid,
 527				  part_sn2->activate_IRQ_phys_cpuid);
 528}
 529
 530static void
 531xpc_assume_partition_disengaged_sn2(short partid)
 532{
 533	struct amo *amo = xpc_vars_sn2->amos_page +
 534			  XPC_ENGAGED_PARTITIONS_AMO_SN2;
 535
 536	/* clear bit(s) based on partid mask in our partition's amo */
 537	FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
 538			 ~BIT(partid));
 539}
 540
 541static int
 542xpc_partition_engaged_sn2(short partid)
 543{
 544	struct amo *amo = xpc_vars_sn2->amos_page +
 545			  XPC_ENGAGED_PARTITIONS_AMO_SN2;
 546
 547	/* our partition's amo variable ANDed with partid mask */
 548	return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) &
 549		BIT(partid)) != 0;
 550}
 551
 552static int
 553xpc_any_partition_engaged_sn2(void)
 554{
 555	struct amo *amo = xpc_vars_sn2->amos_page +
 556			  XPC_ENGAGED_PARTITIONS_AMO_SN2;
 557
 558	/* our partition's amo variable */
 559	return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) != 0;
 560}
 561
 562/* original protection values for each node */
 563static u64 xpc_prot_vec_sn2[MAX_NUMNODES];
 564
 565/*
 566 * Change protections to allow amo operations on non-Shub 1.1 systems.
 567 */
 568static enum xp_retval
 569xpc_allow_amo_ops_sn2(struct amo *amos_page)
 570{
 571	enum xp_retval ret = xpSuccess;
 572
 573	/*
 574	 * On SHUB 1.1, we cannot call sn_change_memprotect() since the BIST
 575	 * collides with memory operations. On those systems we call
 576	 * xpc_allow_amo_ops_shub_wars_1_1_sn2() instead.
 577	 */
 578	if (!enable_shub_wars_1_1())
 579		ret = xp_expand_memprotect(ia64_tpa((u64)amos_page), PAGE_SIZE);
 580
 581	return ret;
 582}
 583
 584/*
 585 * Change protections to allow amo operations on Shub 1.1 systems.
 586 */
 587static void
 588xpc_allow_amo_ops_shub_wars_1_1_sn2(void)
 589{
 590	int node;
 591	int nasid;
 592
 593	if (!enable_shub_wars_1_1())
 594		return;
 595
 596	for_each_online_node(node) {
 597		nasid = cnodeid_to_nasid(node);
 598		/* save current protection values */
 599		xpc_prot_vec_sn2[node] =
 600		    (u64)HUB_L((u64 *)GLOBAL_MMR_ADDR(nasid,
 601						  SH1_MD_DQLP_MMR_DIR_PRIVEC0));
 602		/* open up everything */
 603		HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid,
 604					     SH1_MD_DQLP_MMR_DIR_PRIVEC0),
 605		      -1UL);
 606		HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid,
 607					     SH1_MD_DQRP_MMR_DIR_PRIVEC0),
 608		      -1UL);
 609	}
 610}
 611
 612static enum xp_retval
 613xpc_get_partition_rsvd_page_pa_sn2(void *buf, u64 *cookie, unsigned long *rp_pa,
 614				   size_t *len)
 615{
 616	s64 status;
 617	enum xp_retval ret;
 618
 619	status = sn_partition_reserved_page_pa((u64)buf, cookie,
 620			(u64 *)rp_pa, (u64 *)len);
 621	if (status == SALRET_OK)
 622		ret = xpSuccess;
 623	else if (status == SALRET_MORE_PASSES)
 624		ret = xpNeedMoreInfo;
 625	else
 626		ret = xpSalError;
 627
 628	return ret;
 629}
 630
 631
 632static int
 633xpc_setup_rsvd_page_sn2(struct xpc_rsvd_page *rp)
 634{
 635	struct amo *amos_page;
 636	int i;
 637	int ret;
 638
 639	xpc_vars_sn2 = XPC_RP_VARS(rp);
 640
 641	rp->sn.sn2.vars_pa = xp_pa(xpc_vars_sn2);
 642
 643	/* vars_part array follows immediately after vars */
 644	xpc_vars_part_sn2 = (struct xpc_vars_part_sn2 *)((u8 *)XPC_RP_VARS(rp) +
 645							 XPC_RP_VARS_SIZE);
 646
 647	/*
 648	 * Before clearing xpc_vars_sn2, see if a page of amos had been
 649	 * previously allocated. If not we'll need to allocate one and set
 650	 * permissions so that cross-partition amos are allowed.
 651	 *
 652	 * The allocated amo page needs MCA reporting to remain disabled after
 653	 * XPC has unloaded.  To make this work, we keep a copy of the pointer
 654	 * to this page (i.e., amos_page) in the struct xpc_vars_sn2 structure,
 655	 * which is pointed to by the reserved page, and re-use that saved copy
 656	 * on subsequent loads of XPC. This amo page is never freed, and its
 657	 * memory protections are never restricted.
 658	 */
 659	amos_page = xpc_vars_sn2->amos_page;
 660	if (amos_page == NULL) {
 661		amos_page = (struct amo *)TO_AMO(uncached_alloc_page(0, 1));
 662		if (amos_page == NULL) {
 663			dev_err(xpc_part, "can't allocate page of amos\n");
 664			return -ENOMEM;
 665		}
 666
 667		/*
 668		 * Open up amo-R/W to cpu.  This is done on Shub 1.1 systems
 669		 * when xpc_allow_amo_ops_shub_wars_1_1_sn2() is called.
 670		 */
 671		ret = xpc_allow_amo_ops_sn2(amos_page);
 672		if (ret != xpSuccess) {
 673			dev_err(xpc_part, "can't allow amo operations\n");
 674			uncached_free_page(__IA64_UNCACHED_OFFSET |
 675					   TO_PHYS((u64)amos_page), 1);
 676			return -EPERM;
 677		}
 678	}
 679
 680	/* clear xpc_vars_sn2 */
 681	memset(xpc_vars_sn2, 0, sizeof(struct xpc_vars_sn2));
 682
 683	xpc_vars_sn2->version = XPC_V_VERSION;
 684	xpc_vars_sn2->activate_IRQ_nasid = cpuid_to_nasid(0);
 685	xpc_vars_sn2->activate_IRQ_phys_cpuid = cpu_physical_id(0);
 686	xpc_vars_sn2->vars_part_pa = xp_pa(xpc_vars_part_sn2);
 687	xpc_vars_sn2->amos_page_pa = ia64_tpa((u64)amos_page);
 688	xpc_vars_sn2->amos_page = amos_page;	/* save for next load of XPC */
 689
 690	/* clear xpc_vars_part_sn2 */
 691	memset((u64 *)xpc_vars_part_sn2, 0, sizeof(struct xpc_vars_part_sn2) *
 692	       XP_MAX_NPARTITIONS_SN2);
 693
 694	/* initialize the activate IRQ related amo variables */
 695	for (i = 0; i < xpc_nasid_mask_nlongs; i++)
 696		(void)xpc_init_IRQ_amo_sn2(XPC_ACTIVATE_IRQ_AMOS_SN2 + i);
 697
 698	/* initialize the engaged remote partitions related amo variables */
 699	(void)xpc_init_IRQ_amo_sn2(XPC_ENGAGED_PARTITIONS_AMO_SN2);
 700	(void)xpc_init_IRQ_amo_sn2(XPC_DEACTIVATE_REQUEST_AMO_SN2);
 701
 702	return 0;
 703}
 704
 705static int
 706xpc_hb_allowed_sn2(short partid, void *heartbeating_to_mask)
 707{
 708	return test_bit(partid, heartbeating_to_mask);
 709}
 710
 711static void
 712xpc_allow_hb_sn2(short partid)
 713{
 714	DBUG_ON(xpc_vars_sn2 == NULL);
 715	set_bit(partid, xpc_vars_sn2->heartbeating_to_mask);
 716}
 717
 718static void
 719xpc_disallow_hb_sn2(short partid)
 720{
 721	DBUG_ON(xpc_vars_sn2 == NULL);
 722	clear_bit(partid, xpc_vars_sn2->heartbeating_to_mask);
 723}
 724
 725static void
 726xpc_disallow_all_hbs_sn2(void)
 727{
 728	DBUG_ON(xpc_vars_sn2 == NULL);
 729	bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, xp_max_npartitions);
 730}
 731
 732static void
 733xpc_increment_heartbeat_sn2(void)
 734{
 735	xpc_vars_sn2->heartbeat++;
 736}
 737
 738static void
 739xpc_offline_heartbeat_sn2(void)
 740{
 741	xpc_increment_heartbeat_sn2();
 742	xpc_vars_sn2->heartbeat_offline = 1;
 743}
 744
 745static void
 746xpc_online_heartbeat_sn2(void)
 747{
 748	xpc_increment_heartbeat_sn2();
 749	xpc_vars_sn2->heartbeat_offline = 0;
 750}
 751
 752static void
 753xpc_heartbeat_init_sn2(void)
 754{
 755	DBUG_ON(xpc_vars_sn2 == NULL);
 756
 757	bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, XP_MAX_NPARTITIONS_SN2);
 758	xpc_online_heartbeat_sn2();
 759}
 760
 761static void
 762xpc_heartbeat_exit_sn2(void)
 763{
 764	xpc_offline_heartbeat_sn2();
 765}
 766
 767static enum xp_retval
 768xpc_get_remote_heartbeat_sn2(struct xpc_partition *part)
 769{
 770	struct xpc_vars_sn2 *remote_vars;
 771	enum xp_retval ret;
 772
 773	remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2;
 774
 775	/* pull the remote vars structure that contains the heartbeat */
 776	ret = xp_remote_memcpy(xp_pa(remote_vars),
 777			       part->sn.sn2.remote_vars_pa,
 778			       XPC_RP_VARS_SIZE);
 779	if (ret != xpSuccess)
 780		return ret;
 781
 782	dev_dbg(xpc_part, "partid=%d, heartbeat=%lld, last_heartbeat=%lld, "
 783		"heartbeat_offline=%lld, HB_mask[0]=0x%lx\n", XPC_PARTID(part),
 784		remote_vars->heartbeat, part->last_heartbeat,
 785		remote_vars->heartbeat_offline,
 786		remote_vars->heartbeating_to_mask[0]);
 787
 788	if ((remote_vars->heartbeat == part->last_heartbeat &&
 789	    !remote_vars->heartbeat_offline) ||
 790	    !xpc_hb_allowed_sn2(sn_partition_id,
 791				remote_vars->heartbeating_to_mask)) {
 792		ret = xpNoHeartbeat;
 793	} else {
 794		part->last_heartbeat = remote_vars->heartbeat;
 795	}
 796
 797	return ret;
 798}
 799
 800/*
 801 * Get a copy of the remote partition's XPC variables from the reserved page.
 802 *
 803 * remote_vars points to a buffer that is cacheline aligned for BTE copies and
 804 * assumed to be of size XPC_RP_VARS_SIZE.
 805 */
 806static enum xp_retval
 807xpc_get_remote_vars_sn2(unsigned long remote_vars_pa,
 808			struct xpc_vars_sn2 *remote_vars)
 809{
 810	enum xp_retval ret;
 811
 812	if (remote_vars_pa == 0)
 813		return xpVarsNotSet;
 814
 815	/* pull over the cross partition variables */
 816	ret = xp_remote_memcpy(xp_pa(remote_vars), remote_vars_pa,
 817			       XPC_RP_VARS_SIZE);
 818	if (ret != xpSuccess)
 819		return ret;
 820
 821	if (XPC_VERSION_MAJOR(remote_vars->version) !=
 822	    XPC_VERSION_MAJOR(XPC_V_VERSION)) {
 823		return xpBadVersion;
 824	}
 825
 826	return xpSuccess;
 827}
 828
 829static void
 830xpc_request_partition_activation_sn2(struct xpc_rsvd_page *remote_rp,
 831				     unsigned long remote_rp_pa, int nasid)
 832{
 833	xpc_send_local_activate_IRQ_sn2(nasid);
 834}
 835
 836static void
 837xpc_request_partition_reactivation_sn2(struct xpc_partition *part)
 838{
 839	xpc_send_local_activate_IRQ_sn2(part->sn.sn2.activate_IRQ_nasid);
 840}
 841
 842static void
 843xpc_request_partition_deactivation_sn2(struct xpc_partition *part)
 844{
 845	struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
 846	unsigned long irq_flags;
 847	struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa +
 848					     (XPC_DEACTIVATE_REQUEST_AMO_SN2 *
 849					     sizeof(struct amo)));
 850
 851	local_irq_save(irq_flags);
 852
 853	/* set bit corresponding to our partid in remote partition's amo */
 854	FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR,
 855			 BIT(sn_partition_id));
 856
 857	/*
 858	 * We must always use the nofault function regardless of whether we
 859	 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
 860	 * didn't, we'd never know that the other partition is down and would
 861	 * keep sending IRQs and amos to it until the heartbeat times out.
 862	 */
 863	(void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
 864							       variable),
 865						     xp_nofault_PIOR_target));
 866
 867	local_irq_restore(irq_flags);
 868
 869	/*
 870	 * Send activate IRQ to get other side to see that we've set our
 871	 * bit in their deactivate request amo.
 872	 */
 873	xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa,
 874				  cnodeid_to_nasid(0),
 875				  part_sn2->activate_IRQ_nasid,
 876				  part_sn2->activate_IRQ_phys_cpuid);
 877}
 878
 879static void
 880xpc_cancel_partition_deactivation_request_sn2(struct xpc_partition *part)
 881{
 882	unsigned long irq_flags;
 883	struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa +
 884					     (XPC_DEACTIVATE_REQUEST_AMO_SN2 *
 885					     sizeof(struct amo)));
 886
 887	local_irq_save(irq_flags);
 888
 889	/* clear bit corresponding to our partid in remote partition's amo */
 890	FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
 891			 ~BIT(sn_partition_id));
 892
 893	/*
 894	 * We must always use the nofault function regardless of whether we
 895	 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
 896	 * didn't, we'd never know that the other partition is down and would
 897	 * keep sending IRQs and amos to it until the heartbeat times out.
 898	 */
 899	(void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
 900							       variable),
 901						     xp_nofault_PIOR_target));
 902
 903	local_irq_restore(irq_flags);
 904}
 905
 906static int
 907xpc_partition_deactivation_requested_sn2(short partid)
 908{
 909	struct amo *amo = xpc_vars_sn2->amos_page +
 910			  XPC_DEACTIVATE_REQUEST_AMO_SN2;
 911
 912	/* our partition's amo variable ANDed with partid mask */
 913	return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) &
 914		BIT(partid)) != 0;
 915}
 916
 917/*
 918 * Update the remote partition's info.
 919 */
 920static void
 921xpc_update_partition_info_sn2(struct xpc_partition *part, u8 remote_rp_version,
 922			      unsigned long *remote_rp_ts_jiffies,
 923			      unsigned long remote_rp_pa,
 924			      unsigned long remote_vars_pa,
 925			      struct xpc_vars_sn2 *remote_vars)
 926{
 927	struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
 928
 929	part->remote_rp_version = remote_rp_version;
 930	dev_dbg(xpc_part, "  remote_rp_version = 0x%016x\n",
 931		part->remote_rp_version);
 932
 933	part->remote_rp_ts_jiffies = *remote_rp_ts_jiffies;
 934	dev_dbg(xpc_part, "  remote_rp_ts_jiffies = 0x%016lx\n",
 935		part->remote_rp_ts_jiffies);
 936
 937	part->remote_rp_pa = remote_rp_pa;
 938	dev_dbg(xpc_part, "  remote_rp_pa = 0x%016lx\n", part->remote_rp_pa);
 939
 940	part_sn2->remote_vars_pa = remote_vars_pa;
 941	dev_dbg(xpc_part, "  remote_vars_pa = 0x%016lx\n",
 942		part_sn2->remote_vars_pa);
 943
 944	part->last_heartbeat = remote_vars->heartbeat - 1;
 945	dev_dbg(xpc_part, "  last_heartbeat = 0x%016llx\n",
 946		part->last_heartbeat);
 947
 948	part_sn2->remote_vars_part_pa = remote_vars->vars_part_pa;
 949	dev_dbg(xpc_part, "  remote_vars_part_pa = 0x%016lx\n",
 950		part_sn2->remote_vars_part_pa);
 951
 952	part_sn2->activate_IRQ_nasid = remote_vars->activate_IRQ_nasid;
 953	dev_dbg(xpc_part, "  activate_IRQ_nasid = 0x%x\n",
 954		part_sn2->activate_IRQ_nasid);
 955
 956	part_sn2->activate_IRQ_phys_cpuid =
 957	    remote_vars->activate_IRQ_phys_cpuid;
 958	dev_dbg(xpc_part, "  activate_IRQ_phys_cpuid = 0x%x\n",
 959		part_sn2->activate_IRQ_phys_cpuid);
 960
 961	part_sn2->remote_amos_page_pa = remote_vars->amos_page_pa;
 962	dev_dbg(xpc_part, "  remote_amos_page_pa = 0x%lx\n",
 963		part_sn2->remote_amos_page_pa);
 964
 965	part_sn2->remote_vars_version = remote_vars->version;
 966	dev_dbg(xpc_part, "  remote_vars_version = 0x%x\n",
 967		part_sn2->remote_vars_version);
 968}
 969
 970/*
 971 * Prior code has determined the nasid which generated a activate IRQ.
 972 * Inspect that nasid to determine if its partition needs to be activated
 973 * or deactivated.
 974 *
 975 * A partition is considered "awaiting activation" if our partition
 976 * flags indicate it is not active and it has a heartbeat.  A
 977 * partition is considered "awaiting deactivation" if our partition
 978 * flags indicate it is active but it has no heartbeat or it is not
 979 * sending its heartbeat to us.
 980 *
 981 * To determine the heartbeat, the remote nasid must have a properly
 982 * initialized reserved page.
 983 */
 984static void
 985xpc_identify_activate_IRQ_req_sn2(int nasid)
 986{
 987	struct xpc_rsvd_page *remote_rp;
 988	struct xpc_vars_sn2 *remote_vars;
 989	unsigned long remote_rp_pa;
 990	unsigned long remote_vars_pa;
 991	int remote_rp_version;
 992	int reactivate = 0;
 993	unsigned long remote_rp_ts_jiffies = 0;
 994	short partid;
 995	struct xpc_partition *part;
 996	struct xpc_partition_sn2 *part_sn2;
 997	enum xp_retval ret;
 998
 999	/* pull over the reserved page structure */
1000
1001	remote_rp = (struct xpc_rsvd_page *)xpc_remote_copy_buffer_sn2;
1002
1003	ret = xpc_get_remote_rp(nasid, NULL, remote_rp, &remote_rp_pa);
1004	if (ret != xpSuccess) {
1005		dev_warn(xpc_part, "unable to get reserved page from nasid %d, "
1006			 "which sent interrupt, reason=%d\n", nasid, ret);
1007		return;
1008	}
1009
1010	remote_vars_pa = remote_rp->sn.sn2.vars_pa;
1011	remote_rp_version = remote_rp->version;
1012	remote_rp_ts_jiffies = remote_rp->ts_jiffies;
1013
1014	partid = remote_rp->SAL_partid;
1015	part = &xpc_partitions[partid];
1016	part_sn2 = &part->sn.sn2;
1017
1018	/* pull over the cross partition variables */
1019
1020	remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2;
1021
1022	ret = xpc_get_remote_vars_sn2(remote_vars_pa, remote_vars);
1023	if (ret != xpSuccess) {
1024		dev_warn(xpc_part, "unable to get XPC variables from nasid %d, "
1025			 "which sent interrupt, reason=%d\n", nasid, ret);
1026
1027		XPC_DEACTIVATE_PARTITION(part, ret);
1028		return;
1029	}
1030
1031	part->activate_IRQ_rcvd++;
1032
1033	dev_dbg(xpc_part, "partid for nasid %d is %d; IRQs = %d; HB = "
1034		"%lld:0x%lx\n", (int)nasid, (int)partid,
1035		part->activate_IRQ_rcvd,
1036		remote_vars->heartbeat, remote_vars->heartbeating_to_mask[0]);
1037
1038	if (xpc_partition_disengaged(part) &&
1039	    part->act_state == XPC_P_AS_INACTIVE) {
1040
1041		xpc_update_partition_info_sn2(part, remote_rp_version,
1042					      &remote_rp_ts_jiffies,
1043					      remote_rp_pa, remote_vars_pa,
1044					      remote_vars);
1045
1046		if (xpc_partition_deactivation_requested_sn2(partid)) {
1047			/*
1048			 * Other side is waiting on us to deactivate even though
1049			 * we already have.
1050			 */
1051			return;
1052		}
1053
1054		xpc_activate_partition(part);
1055		return;
1056	}
1057
1058	DBUG_ON(part->remote_rp_version == 0);
1059	DBUG_ON(part_sn2->remote_vars_version == 0);
1060
1061	if (remote_rp_ts_jiffies != part->remote_rp_ts_jiffies) {
1062
1063		/* the other side rebooted */
1064
1065		DBUG_ON(xpc_partition_engaged_sn2(partid));
1066		DBUG_ON(xpc_partition_deactivation_requested_sn2(partid));
1067
1068		xpc_update_partition_info_sn2(part, remote_rp_version,
1069					      &remote_rp_ts_jiffies,
1070					      remote_rp_pa, remote_vars_pa,
1071					      remote_vars);
1072		reactivate = 1;
1073	}
1074
1075	if (part->disengage_timeout > 0 && !xpc_partition_disengaged(part)) {
1076		/* still waiting on other side to disengage from us */
1077		return;
1078	}
1079
1080	if (reactivate)
1081		XPC_DEACTIVATE_PARTITION(part, xpReactivating);
1082	else if (xpc_partition_deactivation_requested_sn2(partid))
1083		XPC_DEACTIVATE_PARTITION(part, xpOtherGoingDown);
1084}
1085
1086/*
1087 * Loop through the activation amo variables and process any bits
1088 * which are set.  Each bit indicates a nasid sending a partition
1089 * activation or deactivation request.
1090 *
1091 * Return #of IRQs detected.
1092 */
1093int
1094xpc_identify_activate_IRQ_sender_sn2(void)
1095{
1096	int l;
1097	int b;
1098	unsigned long nasid_mask_long;
1099	u64 nasid;		/* remote nasid */
1100	int n_IRQs_detected = 0;
1101	struct amo *act_amos;
1102
1103	act_amos = xpc_vars_sn2->amos_page + XPC_ACTIVATE_IRQ_AMOS_SN2;
1104
1105	/* scan through activate amo variables looking for non-zero entries */
1106	for (l = 0; l < xpc_nasid_mask_nlongs; l++) {
1107
1108		if (xpc_exiting)
1109			break;
1110
1111		nasid_mask_long = xpc_receive_IRQ_amo_sn2(&act_amos[l]);
1112
1113		b = find_first_bit(&nasid_mask_long, BITS_PER_LONG);
1114		if (b >= BITS_PER_LONG) {
1115			/* no IRQs from nasids in this amo variable */
1116			continue;
1117		}
1118
1119		dev_dbg(xpc_part, "amo[%d] gave back 0x%lx\n", l,
1120			nasid_mask_long);
1121
1122		/*
1123		 * If this nasid has been added to the machine since
1124		 * our partition was reset, this will retain the
1125		 * remote nasid in our reserved pages machine mask.
1126		 * This is used in the event of module reload.
1127		 */
1128		xpc_mach_nasids[l] |= nasid_mask_long;
1129
1130		/* locate the nasid(s) which sent interrupts */
1131
1132		do {
1133			n_IRQs_detected++;
1134			nasid = (l * BITS_PER_LONG + b) * 2;
1135			dev_dbg(xpc_part, "interrupt from nasid %lld\n", nasid);
1136			xpc_identify_activate_IRQ_req_sn2(nasid);
1137
1138			b = find_next_bit(&nasid_mask_long, BITS_PER_LONG,
1139					  b + 1);
1140		} while (b < BITS_PER_LONG);
1141	}
1142	return n_IRQs_detected;
1143}
1144
1145static void
1146xpc_process_activate_IRQ_rcvd_sn2(void)
1147{
1148	unsigned long irq_flags;
1149	int n_IRQs_expected;
1150	int n_IRQs_detected;
1151
1152	spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
1153	n_IRQs_expected = xpc_activate_IRQ_rcvd;
1154	xpc_activate_IRQ_rcvd = 0;
1155	spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
1156
1157	n_IRQs_detected = xpc_identify_activate_IRQ_sender_sn2();
1158	if (n_IRQs_detected < n_IRQs_expected) {
1159		/* retry once to help avoid missing amo */
1160		(void)xpc_identify_activate_IRQ_sender_sn2();
1161	}
1162}
1163
1164/*
1165 * Setup the channel structures that are sn2 specific.
1166 */
1167static enum xp_retval
1168xpc_setup_ch_structures_sn2(struct xpc_partition *part)
1169{
1170	struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
1171	struct xpc_channel_sn2 *ch_sn2;
1172	enum xp_retval retval;
1173	int ret;
1174	int cpuid;
1175	int ch_number;
1176	struct timer_list *timer;
1177	short partid = XPC_PARTID(part);
1178
1179	/* allocate all the required GET/PUT values */
1180
1181	part_sn2->local_GPs =
1182	    xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL,
1183					  &part_sn2->local_GPs_base);
1184	if (part_sn2->local_GPs == NULL) {
1185		dev_err(xpc_chan, "can't get memory for local get/put "
1186			"values\n");
1187		return xpNoMemory;
1188	}
1189
1190	part_sn2->remote_GPs =
1191	    xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL,
1192					  &part_sn2->remote_GPs_base);
1193	if (part_sn2->remote_GPs == NULL) {
1194		dev_err(xpc_chan, "can't get memory for remote get/put "
1195			"values\n");
1196		retval = xpNoMemory;
1197		goto out_1;
1198	}
1199
1200	part_sn2->remote_GPs_pa = 0;
1201
1202	/* allocate all the required open and close args */
1203
1204	part_sn2->local_openclose_args =
1205	    xpc_kzalloc_cacheline_aligned(XPC_OPENCLOSE_ARGS_SIZE,
1206					  GFP_KERNEL, &part_sn2->
1207					  local_openclose_args_base);
1208	if (part_sn2->local_openclose_args == NULL) {
1209		dev_err(xpc_chan, "can't get memory for local connect args\n");
1210		retval = xpNoMemory;
1211		goto out_2;
1212	}
1213
1214	part_sn2->remote_openclose_args_pa = 0;
1215
1216	part_sn2->local_chctl_amo_va = xpc_init_IRQ_amo_sn2(partid);
1217
1218	part_sn2->notify_IRQ_nasid = 0;
1219	part_sn2->notify_IRQ_phys_cpuid = 0;
1220	part_sn2->remote_chctl_amo_va = NULL;
1221
1222	sprintf(part_sn2->notify_IRQ_owner, "xpc%02d", partid);
1223	ret = request_irq(SGI_XPC_NOTIFY, xpc_handle_notify_IRQ_sn2,
1224			  IRQF_SHARED, part_sn2->notify_IRQ_owner,
1225			  (void *)(u64)partid);
1226	if (ret != 0) {
1227		dev_err(xpc_chan, "can't register NOTIFY IRQ handler, "
1228			"errno=%d\n", -ret);
1229		retval = xpLackOfResources;
1230		goto out_3;
1231	}
1232
1233	/* Setup a timer to check for dropped notify IRQs */
1234	timer = &part_sn2->dropped_notify_IRQ_timer;
1235	init_timer(timer);
1236	timer->function =
1237	    (void (*)(unsigned long))xpc_check_for_dropped_notify_IRQ_sn2;
1238	timer->data = (unsigned long)part;
1239	timer->expires = jiffies + XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL;
1240	add_timer(timer);
1241
1242	for (ch_number = 0; ch_number < part->nchannels; ch_number++) {
1243		ch_sn2 = &part->channels[ch_number].sn.sn2;
1244
1245		ch_sn2->local_GP = &part_sn2->local_GPs[ch_number];
1246		ch_sn2->local_openclose_args =
1247		    &part_sn2->local_openclose_args[ch_number];
1248
1249		mutex_init(&ch_sn2->msg_to_pull_mutex);
1250	}
1251
1252	/*
1253	 * Setup the per partition specific variables required by the
1254	 * remote partition to establish channel connections with us.
1255	 *
1256	 * The setting of the magic # indicates that these per partition
1257	 * specific variables are ready to be used.
1258	 */
1259	xpc_vars_part_sn2[partid].GPs_pa = xp_pa(part_sn2->local_GPs);
1260	xpc_vars_part_sn2[partid].openclose_args_pa =
1261	    xp_pa(part_sn2->local_openclose_args);
1262	xpc_vars_part_sn2[partid].chctl_amo_pa =
1263	    xp_pa(part_sn2->local_chctl_amo_va);
1264	cpuid = raw_smp_processor_id();	/* any CPU in this partition will do */
1265	xpc_vars_part_sn2[partid].notify_IRQ_nasid = cpuid_to_nasid(cpuid);
1266	xpc_vars_part_sn2[partid].notify_IRQ_phys_cpuid =
1267	    cpu_physical_id(cpuid);
1268	xpc_vars_part_sn2[partid].nchannels = part->nchannels;
1269	xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC1_SN2;
1270
1271	return xpSuccess;
1272
1273	/* setup of ch structures failed */
1274out_3:
1275	kfree(part_sn2->local_openclose_args_base);
1276	part_sn2->local_openclose_args = NULL;
1277out_2:
1278	kfree(part_sn2->remote_GPs_base);
1279	part_sn2->remote_GPs = NULL;
1280out_1:
1281	kfree(part_sn2->local_GPs_base);
1282	part_sn2->local_GPs = NULL;
1283	return retval;
1284}
1285
1286/*
1287 * Teardown the channel structures that are sn2 specific.
1288 */
1289static void
1290xpc_teardown_ch_structures_sn2(struct xpc_partition *part)
1291{
1292	struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
1293	short partid = XPC_PARTID(part);
1294
1295	/*
1296	 * Indicate that the variables specific to the remote partition are no
1297	 * longer available for its use.
1298	 */
1299	xpc_vars_part_sn2[partid].magic = 0;
1300
1301	/* in case we've still got outstanding timers registered... */
1302	del_timer_sync(&part_sn2->dropped_notify_IRQ_timer);
1303	free_irq(SGI_XPC_NOTIFY, (void *)(u64)partid);
1304
1305	kfree(part_sn2->local_openclose_args_base);
1306	part_sn2->local_openclose_args = NULL;
1307	kfree(part_sn2->remote_GPs_base);
1308	part_sn2->remote_GPs = NULL;
1309	kfree(part_sn2->local_GPs_base);
1310	part_sn2->local_GPs = NULL;
1311	part_sn2->local_chctl_amo_va = NULL;
1312}
1313
1314/*
1315 * Create a wrapper that hides the underlying mechanism for pulling a cacheline
1316 * (or multiple cachelines) from a remote partition.
1317 *
1318 * src_pa must be a cacheline aligned physical address on the remote partition.
1319 * dst must be a cacheline aligned virtual address on this partition.
1320 * cnt must be cacheline sized
1321 */
1322/* ??? Replace this function by call to xp_remote_memcpy() or bte_copy()? */
1323static enum xp_retval
1324xpc_pull_remote_cachelines_sn2(struct xpc_partition *part, void *dst,
1325			       const unsigned long src_pa, size_t cnt)
1326{
1327	enum xp_retval ret;
1328
1329	DBUG_ON(src_pa != L1_CACHE_ALIGN(src_pa));
1330	DBUG_ON((unsigned long)dst != L1_CACHE_ALIGN((unsigned long)dst));
1331	DBUG_ON(cnt != L1_CACHE_ALIGN(cnt));
1332
1333	if (part->act_state == XPC_P_AS_DEACTIVATING)
1334		return part->reason;
1335
1336	ret = xp_remote_memcpy(xp_pa(dst), src_pa, cnt);
1337	if (ret != xpSuccess) {
1338		dev_dbg(xpc_chan, "xp_remote_memcpy() from partition %d failed,"
1339			" ret=%d\n", XPC_PARTID(part), ret);
1340	}
1341	return ret;
1342}
1343
1344/*
1345 * Pull the remote per partition specific variables from the specified
1346 * partition.
1347 */
1348static enum xp_retval
1349xpc_pull_remote_vars_part_sn2(struct xpc_partition *part)
1350{
1351	struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
1352	u8 buffer[L1_CACHE_BYTES * 2];
1353	struct xpc_vars_part_sn2 *pulled_entry_cacheline =
1354	    (struct xpc_vars_part_sn2 *)L1_CACHE_ALIGN((u64)buffer);
1355	struct xpc_vars_part_sn2 *pulled_entry;
1356	unsigned long remote_entry_cacheline_pa;
1357	unsigned long remote_entry_pa;
1358	short partid = XPC_PARTID(part);
1359	enum xp_retval ret;
1360
1361	/* pull the cacheline that contains the variables we're interested in */
1362
1363	DBUG_ON(part_sn2->remote_vars_part_pa !=
1364		L1_CACHE_ALIGN(part_sn2->remote_vars_part_pa));
1365	DBUG_ON(sizeof(struct xpc_vars_part_sn2) != L1_CACHE_BYTES / 2);
1366
1367	remote_entry_pa = part_sn2->remote_vars_part_pa +
1368	    sn_partition_id * sizeof(struct xpc_vars_part_sn2);
1369
1370	remote_entry_cacheline_pa = (remote_entry_pa & ~(L1_CACHE_BYTES - 1));
1371
1372	pulled_entry = (struct xpc_vars_part_sn2 *)((u64)pulled_entry_cacheline
1373						    + (remote_entry_pa &
1374						    (L1_CACHE_BYTES - 1)));
1375
1376	ret = xpc_pull_remote_cachelines_sn2(part, pulled_entry_cacheline,
1377					     remote_entry_cacheline_pa,
1378					     L1_CACHE_BYTES);
1379	if (ret != xpSuccess) {
1380		dev_dbg(xpc_chan, "failed to pull XPC vars_part from "
1381			"partition %d, ret=%d\n", partid, ret);
1382		return ret;
1383	}
1384
1385	/* see if they've been set up yet */
1386
1387	if (pulled_entry->magic != XPC_VP_MAGIC1_SN2 &&
1388	    pulled_entry->magic != XPC_VP_MAGIC2_SN2) {
1389
1390		if (pulled_entry->magic != 0) {
1391			dev_dbg(xpc_chan, "partition %d's XPC vars_part for "
1392				"partition %d has bad magic value (=0x%llx)\n",
1393				partid, sn_partition_id, pulled_entry->magic);
1394			return xpBadMagic;
1395		}
1396
1397		/* they've not been initialized yet */
1398		return xpRetry;
1399	}
1400
1401	if (xpc_vars_part_sn2[partid].magic == XPC_VP_MAGIC1_SN2) {
1402
1403		/* validate the variables */
1404
1405		if (pulled_entry->GPs_pa == 0 ||
1406		    pulled_entry->openclose_args_pa == 0 ||
1407		    pulled_entry->chctl_amo_pa == 0) {
1408
1409			dev_err(xpc_chan, "partition %d's XPC vars_part for "
1410				"partition %d are not valid\n", partid,
1411				sn_partition_id);
1412			return xpInvalidAddress;
1413		}
1414
1415		/* the variables we imported look to be valid */
1416
1417		part_sn2->remote_GPs_pa = pulled_entry->GPs_pa;
1418		part_sn2->remote_openclose_args_pa =
1419		    pulled_entry->openclose_args_pa;
1420		part_sn2->remote_chctl_amo_va =
1421		    (struct amo *)__va(pulled_entry->chctl_amo_pa);
1422		part_sn2->notify_IRQ_nasid = pulled_entry->notify_IRQ_nasid;
1423		part_sn2->notify_IRQ_phys_cpuid =
1424		    pulled_entry->notify_IRQ_phys_cpuid;
1425
1426		if (part->nchannels > pulled_entry->nchannels)
1427			part->nchannels = pulled_entry->nchannels;
1428
1429		/* let the other side know that we've pulled their variables */
1430
1431		xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC2_SN2;
1432	}
1433
1434	if (pulled_entry->magic == XPC_VP_MAGIC1_SN2)
1435		return xpRetry;
1436
1437	return xpSuccess;
1438}
1439
1440/*
1441 * Establish first contact with the remote partititon. This involves pulling
1442 * the XPC per partition variables from the remote partition and waiting for
1443 * the remote partition to pull ours.
1444 */
1445static enum xp_retval
1446xpc_make_first_contact_sn2(struct xpc_partition *part)
1447{
1448	struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
1449	enum xp_retval ret;
1450
1451	/*
1452	 * Register the remote partition's amos with SAL so it can handle
1453	 * and cleanup errors within that address range should the remote
1454	 * partition go down. We don't unregister this range because it is
1455	 * difficult to tell when outstanding writes to the remote partition
1456	 * are finished and thus when it is safe to unregister. This should
1457	 * not result in wasted space in the SAL xp_addr_region table because
1458	 * we should get the same page for remote_amos_page_pa after module
1459	 * reloads and system reboots.
1460	 */
1461	if (sn_register_xp_addr_region(part_sn2->remote_amos_page_pa,
1462				       PAGE_SIZE, 1) < 0) {
1463		dev_warn(xpc_part, "xpc_activating(%d) failed to register "
1464			 "xp_addr region\n", XPC_PARTID(part));
1465
1466		ret = xpPhysAddrRegFailed;
1467		XPC_DEACTIVATE_PARTITION(part, ret);
1468		return ret;
1469	}
1470
1471	/*
1472	 * Send activate IRQ to get other side to activate if they've not
1473	 * already begun to do so.
1474	 */
1475	xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa,
1476				  cnodeid_to_nasid(0),
1477				  part_sn2->activate_IRQ_nasid,
1478				  part_sn2->activate_IRQ_phys_cpuid);
1479
1480	while ((ret = xpc_pull_remote_vars_part_sn2(part)) != xpSuccess) {
1481		if (ret != xpRetry) {
1482			XPC_DEACTIVATE_PARTITION(part, ret);
1483			return ret;
1484		}
1485
1486		dev_dbg(xpc_part, "waiting to make first contact with "
1487			"partition %d\n", XPC_PARTID(part));
1488
1489		/* wait a 1/4 of a second or so */
1490		(void)msleep_interruptible(250);
1491
1492		if (part->act_state == XPC_P_AS_DEACTIVATING)
1493			return part->reason;
1494	}
1495
1496	return xpSuccess;
1497}
1498
1499/*
1500 * Get the chctl flags and pull the openclose args and/or remote GPs as needed.
1501 */
1502static u64
1503xpc_get_chctl_all_flags_sn2(struct xpc_partition *part)
1504{
1505	struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
1506	unsigned long irq_flags;
1507	union xpc_channel_ctl_flags chctl;
1508	enum xp_retval ret;
1509
1510	/*
1511	 * See if there are any chctl flags to be handled.
1512	 */
1513
1514	spin_lock_irqsave(&part->chctl_lock, irq_flags);
1515	chctl = part->chctl;
1516	if (chctl.all_flags != 0)
1517		part->chctl.all_flags = 0;
1518
1519	spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
1520
1521	if (xpc_any_openclose_chctl_flags_set(&chctl)) {
1522		ret = xpc_pull_remote_cachelines_sn2(part, part->
1523						     remote_openclose_args,
1524						     part_sn2->
1525						     remote_openclose_args_pa,
1526						     XPC_OPENCLOSE_ARGS_SIZE);
1527		if (ret != xpSuccess) {
1528			XPC_DEACTIVATE_PARTITION(part, ret);
1529
1530			dev_dbg(xpc_chan, "failed to pull openclose args from "
1531				"partition %d, ret=%d\n", XPC_PARTID(part),
1532				ret);
1533
1534			/* don't bother processing chctl flags anymore */
1535			chctl.all_flags = 0;
1536		}
1537	}
1538
1539	if (xpc_any_msg_chctl_flags_set(&chctl)) {
1540		ret = xpc_pull_remote_cachelines_sn2(part, part_sn2->remote_GPs,
1541						     part_sn2->remote_GPs_pa,
1542						     XPC_GP_SIZE);
1543		if (ret != xpSuccess) {
1544			XPC_DEACTIVATE_PARTITION(part, ret);
1545
1546			dev_dbg(xpc_chan, "failed to pull GPs from partition "
1547				"%d, ret=%d\n", XPC_PARTID(part), ret);
1548
1549			/* don't bother processing chctl flags anymore */
1550			chctl.all_flags = 0;
1551		}
1552	}
1553
1554	return chctl.all_flags;
1555}
1556
1557/*
1558 * Allocate the local message queue and the notify queue.
1559 */
1560static enum xp_retval
1561xpc_allocate_local_msgqueue_sn2(struct xpc_channel *ch)
1562{
1563	struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1564	unsigned long irq_flags;
1565	int nentries;
1566	size_t nbytes;
1567
1568	for (nentries = ch->local_nentries; nentries > 0; nentries--) {
1569
1570		nbytes = nentries * ch->entry_size;
1571		ch_sn2->local_msgqueue =
1572		    xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL,
1573						  &ch_sn2->local_msgqueue_base);
1574		if (ch_sn2->local_msgqueue == NULL)
1575			continue;
1576
1577		nbytes = nentries * sizeof(struct xpc_notify_sn2);
1578		ch_sn2->notify_queue = kzalloc(nbytes, GFP_KERNEL);
1579		if (ch_sn2->notify_queue == NULL) {
1580			kfree(ch_sn2->local_msgqueue_base);
1581			ch_sn2->local_msgqueue = NULL;
1582			continue;
1583		}
1584
1585		spin_lock_irqsave(&ch->lock, irq_flags);
1586		if (nentries < ch->local_nentries) {
1587			dev_dbg(xpc_chan, "nentries=%d local_nentries=%d, "
1588				"partid=%d, channel=%d\n", nentries,
1589				ch->local_nentries, ch->partid, ch->number);
1590
1591			ch->local_nentries = nentries;
1592		}
1593		spin_unlock_irqrestore(&ch->lock, irq_flags);
1594		return xpSuccess;
1595	}
1596
1597	dev_dbg(xpc_chan, "can't get memory for local message queue and notify "
1598		"queue, partid=%d, channel=%d\n", ch->partid, ch->number);
1599	return xpNoMemory;
1600}
1601
1602/*
1603 * Allocate the cached remote message queue.
1604 */
1605static enum xp_retval
1606xpc_allocate_remote_msgqueue_sn2(struct xpc_channel *ch)
1607{
1608	struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1609	unsigned long irq_flags;
1610	int nentries;
1611	size_t nbytes;
1612
1613	DBUG_ON(ch->remote_nentries <= 0);
1614
1615	for (nentries = ch->remote_nentries; nentries > 0; nentries--) {
1616
1617		nbytes = nentries * ch->entry_size;
1618		ch_sn2->remote_msgqueue =
1619		    xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL, &ch_sn2->
1620						  remote_msgqueue_base);
1621		if (ch_sn2->remote_msgqueue == NULL)
1622			continue;
1623
1624		spin_lock_irqsave(&ch->lock, irq_flags);
1625		if (nentries < ch->remote_nentries) {
1626			dev_dbg(xpc_chan, "nentries=%d remote_nentries=%d, "
1627				"partid=%d, channel=%d\n", nentries,
1628				ch->remote_nentries, ch->partid, ch->number);
1629
1630			ch->remote_nentries = nentries;
1631		}
1632		spin_unlock_irqrestore(&ch->lock, irq_flags);
1633		return xpSuccess;
1634	}
1635
1636	dev_dbg(xpc_chan, "can't get memory for cached remote message queue, "
1637		"partid=%d, channel=%d\n", ch->partid, ch->number);
1638	return xpNoMemory;
1639}
1640
1641/*
1642 * Allocate message queues and other stuff associated with a channel.
1643 *
1644 * Note: Assumes all of the channel sizes are filled in.
1645 */
1646static enum xp_retval
1647xpc_setup_msg_structures_sn2(struct xpc_channel *ch)
1648{
1649	struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1650	enum xp_retval ret;
1651
1652	DBUG_ON(ch->flags & XPC_C_SETUP);
1653
1654	ret = xpc_allocate_local_msgqueue_sn2(ch);
1655	if (ret == xpSuccess) {
1656
1657		ret = xpc_allocate_remote_msgqueue_sn2(ch);
1658		if (ret != xpSuccess) {
1659			kfree(ch_sn2->local_msgqueue_base);
1660			ch_sn2->local_msgqueue = NULL;
1661			kfree(ch_sn2->notify_queue);
1662			ch_sn2->notify_queue = NULL;
1663		}
1664	}
1665	return ret;
1666}
1667
1668/*
1669 * Free up message queues and other stuff that were allocated for the specified
1670 * channel.
1671 */
1672static void
1673xpc_teardown_msg_structures_sn2(struct xpc_channel *ch)
1674{
1675	struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1676
1677	DBUG_ON(!spin_is_locked(&ch->lock));
1678
1679	ch_sn2->remote_msgqueue_pa = 0;
1680
1681	ch_sn2->local_GP->get = 0;
1682	ch_sn2->local_GP->put = 0;
1683	ch_sn2->remote_GP.get = 0;
1684	ch_sn2->remote_GP.put = 0;
1685	ch_sn2->w_local_GP.get = 0;
1686	ch_sn2->w_local_GP.put = 0;
1687	ch_sn2->w_remote_GP.get = 0;
1688	ch_sn2->w_remote_GP.put = 0;
1689	ch_sn2->next_msg_to_pull = 0;
1690
1691	if (ch->flags & XPC_C_SETUP) {
1692		dev_dbg(xpc_chan, "ch->flags=0x%x, partid=%d, channel=%d\n",
1693			ch->flags, ch->partid, ch->number);
1694
1695		kfree(ch_sn2->local_msgqueue_base);
1696		ch_sn2->local_msgqueue = NULL;
1697		kfree(ch_sn2->remote_msgqueue_base);
1698		ch_sn2->remote_msgqueue = NULL;
1699		kfree(ch_sn2->notify_queue);
1700		ch_sn2->notify_queue = NULL;
1701	}
1702}
1703
1704/*
1705 * Notify those who wanted to be notified upon delivery of their message.
1706 */
1707static void
1708xpc_notify_senders_sn2(struct xpc_channel *ch, enum xp_retval reason, s64 put)
1709{
1710	struct xpc_notify_sn2 *notify;
1711	u8 notify_type;
1712	s64 get = ch->sn.sn2.w_remote_GP.get - 1;
1713
1714	while (++get < put && atomic_read(&ch->n_to_notify) > 0) {
1715
1716		notify = &ch->sn.sn2.notify_queue[get % ch->local_nentries];
1717
1718		/*
1719		 * See if the notify entry indicates it was associated with
1720		 * a message who's sender wants to be notified. It is possible
1721		 * that it is, but someone else is doing or has done the
1722		 * notification.
1723		 */
1724		notify_type = notify->type;
1725		if (notify_type == 0 ||
1726		    cmpxchg(&notify->type, notify_type, 0) != notify_type) {
1727			continue;
1728		}
1729
1730		DBUG_ON(notify_type != XPC_N_CALL);
1731
1732		atomic_dec(&ch->n_to_notify);
1733
1734		if (notify->func != NULL) {
1735			dev_dbg(xpc_chan, "notify->func() called, notify=0x%p "
1736				"msg_number=%lld partid=%d channel=%d\n",
1737				(void *)notify, get, ch->partid, ch->number);
1738
1739			notify->func(reason, ch->partid, ch->number,
1740				     notify->key);
1741
1742			dev_dbg(xpc_chan, "notify->func() returned, notify=0x%p"
1743				" msg_number=%lld partid=%d channel=%d\n",
1744				(void *)notify, get, ch->partid, ch->number);
1745		}
1746	}
1747}
1748
1749static void
1750xpc_notify_senders_of_disconnect_sn2(struct xpc_channel *ch)
1751{
1752	xpc_notify_senders_sn2(ch, ch->reason, ch->sn.sn2.w_local_GP.put);
1753}
1754
1755/*
1756 * Clear some of the msg flags in the local message queue.
1757 */
1758static inline void
1759xpc_clear_local_msgqueue_flags_sn2(struct xpc_channel *ch)
1760{
1761	struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1762	struct xpc_msg_sn2 *msg;
1763	s64 get;
1764
1765	get = ch_sn2->w_remote_GP.get;
1766	do {
1767		msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue +
1768					     (get % ch->local_nentries) *
1769					     ch->entry_size);
1770		DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
1771		msg->flags = 0;
1772	} while (++get < ch_sn2->remote_GP.get);
1773}
1774
1775/*
1776 * Clear some of the msg flags in the remote message queue.
1777 */
1778static inline void
1779xpc_clear_remote_msgqueue_flags_sn2(struct xpc_channel *ch)
1780{
1781	struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1782	struct xpc_msg_sn2 *msg;
1783	s64 put, remote_nentries = ch->remote_nentries;
1784
1785	/* flags are zeroed when the buffer is allocated */
1786	if (ch_sn2->remote_GP.put < remote_nentries)
1787		return;
1788
1789	put = max(ch_sn2->w_remote_GP.put, remote_nentries);
1790	do {
1791		msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue +
1792					     (put % remote_nentries) *
1793					     ch->entry_size);
1794		DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
1795		DBUG_ON(!(msg->flags & XPC_M_SN2_DONE));
1796		DBUG_ON(msg->number != put - remote_nentries);
1797		msg->flags = 0;
1798	} while (++put < ch_sn2->remote_GP.put);
1799}
1800
1801static int
1802xpc_n_of_deliverable_payloads_sn2(struct xpc_channel *ch)
1803{
1804	return ch->sn.sn2.w_remote_GP.put - ch->sn.sn2.w_local_GP.get;
1805}
1806
1807static void
1808xpc_process_msg_chctl_flags_sn2(struct xpc_partition *part, int ch_number)
1809{
1810	struct xpc_channel *ch = &part->channels[ch_number];
1811	struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1812	int npayloads_sent;
1813
1814	ch_sn2->remote_GP = part->sn.sn2.remote_GPs[ch_number];
1815
1816	/* See what, if anything, has changed for each connected channel */
1817
1818	xpc_msgqueue_ref(ch);
1819
1820	if (ch_sn2->w_remote_GP.get == ch_sn2->remote_GP.get &&
1821	    ch_sn2->w_remote_GP.put == ch_sn2->remote_GP.put) {
1822		/* nothing changed since GPs were last pulled */
1823		xpc_msgqueue_deref(ch);
1824		return;
1825	}
1826
1827	if (!(ch->flags & XPC_C_CONNECTED)) {
1828		xpc_msgqueue_deref(ch);
1829		return;
1830	}
1831
1832	/*
1833	 * First check to see if messages recently sent by us have been
1834	 * received by the other side. (The remote GET value will have
1835	 * changed since we last looked at it.)
1836	 */
1837
1838	if (ch_sn2->w_remote_GP.get != ch_sn2->remote_GP.get) {
1839
1840		/*
1841		 * We need to notify any senders that want to be notified
1842		 * that their sent messages have been received by their
1843		 * intended recipients. We need to do this before updating
1844		 * w_remote_GP.get so that we don't allocate the same message
1845		 * queue entries prematurely (see xpc_allocate_msg()).
1846		 */
1847		if (atomic_read(&ch->n_to_notify) > 0) {
1848			/*
1849			 * Notify senders that messages sent have been
1850			 * received and delivered by the other side.
1851			 */
1852			xpc_notify_senders_sn2(ch, xpMsgDelivered,
1853					       ch_sn2->remote_GP.get);
1854		}
1855
1856		/*
1857		 * Clear msg->flags in previously sent messages, so that
1858		 * they're ready for xpc_allocate_msg().
1859		 */
1860		xpc_clear_local_msgqueue_flags_sn2(ch);
1861
1862		ch_sn2->w_remote_GP.get = ch_sn2->remote_GP.get;
1863
1864		dev_dbg(xpc_chan, "w_remote_GP.get changed to %lld, partid=%d, "
1865			"channel=%d\n", ch_sn2->w_remote_GP.get, ch->partid,
1866			ch->number);
1867
1868		/*
1869		 * If anyone was waiting for message queue entries to become
1870		 * available, wake them up.
1871		 */
1872		if (atomic_read(&ch->n_on_msg_allocate_wq) > 0)
1873			wake_up(&ch->msg_allocate_wq);
1874	}
1875
1876	/*
1877	 * Now check for newly sent messages by the other side. (The remote
1878	 * PUT value will have changed since we last looked at it.)
1879	 */
1880
1881	if (ch_sn2->w_remote_GP.put != ch_sn2->remote_GP.put) {
1882		/*
1883		 * Clear msg->flags in previously received messages, so that
1884		 * they're ready for xpc_get_deliverable_payload_sn2().
1885		 */
1886		xpc_clear_remote_msgqueue_flags_sn2(ch);
1887
1888		smp_wmb(); /* ensure flags have been cleared before bte_copy */
1889		ch_sn2->w_remote_GP.put = ch_sn2->remote_GP.put;
1890
1891		dev_dbg(xpc_chan, "w_remote_GP.put changed to %lld, partid=%d, "
1892			"channel=%d\n", ch_sn2->w_remote_GP.put, ch->partid,
1893			ch->number);
1894
1895		npayloads_sent = xpc_n_of_deliverable_payloads_sn2(ch);
1896		if (npayloads_sent > 0) {
1897			dev_dbg(xpc_chan, "msgs waiting to be copied and "
1898				"delivered=%d, partid=%d, channel=%d\n",
1899				npayloads_sent, ch->partid, ch->number);
1900
1901			if (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE)
1902				xpc_activate_kthreads(ch, npayloads_sent);
1903		}
1904	}
1905
1906	xpc_msgqueue_deref(ch);
1907}
1908
1909static struct xpc_msg_sn2 *
1910xpc_pull_remote_msg_sn2(struct xpc_channel *ch, s64 get)
1911{
1912	struct xpc_partition *part = &xpc_partitions[ch->partid];
1913	struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1914	unsigned long remote_msg_pa;
1915	struct xpc_msg_sn2 *msg;
1916	u32 msg_index;
1917	u32 nmsgs;
1918	u64 msg_offset;
1919	enum xp_retval ret;
1920
1921	if (mutex_lock_interruptible(&ch_sn2->msg_to_pull_mutex) != 0) {
1922		/* we were interrupted by a signal */
1923		return NULL;
1924	}
1925
1926	while (get >= ch_sn2->next_msg_to_pull) {
1927
1928		/* pull as many messages as are ready and able to be pulled */
1929
1930		msg_index = ch_sn2->next_msg_to_pull % ch->remote_nentries;
1931
1932		DBUG_ON(ch_sn2->next_msg_to_pull >= ch_sn2->w_remote_GP.put);
1933		nmsgs = ch_sn2->w_remote_GP.put - ch_sn2->next_msg_to_pull;
1934		if (msg_index + nmsgs > ch->remote_nentries) {
1935			/* ignore the ones that wrap the msg queue for now */
1936			nmsgs = ch->remote_nentries - msg_index;
1937		}
1938
1939		msg_offset = msg_index * ch->entry_size;
1940		msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue +
1941		    msg_offset);
1942		remote_msg_pa = ch_sn2->remote_msgqueue_pa + msg_offset;
1943
1944		ret = xpc_pull_remote_cachelines_sn2(part, msg, remote_msg_pa,
1945						     nmsgs * ch->entry_size);
1946		if (ret != xpSuccess) {
1947
1948			dev_dbg(xpc_chan, "failed to pull %d msgs starting with"
1949				" msg %lld from partition %d, channel=%d, "
1950				"ret=%d\n", nmsgs, ch_sn2->next_msg_to_pull,
1951				ch->partid, ch->number, ret);
1952
1953			XPC_DEACTIVATE_PARTITION(part, ret);
1954
1955			mutex_unlock(&ch_sn2->msg_to_pull_mutex);
1956			return NULL;
1957		}
1958
1959		ch_sn2->next_msg_to_pull += nmsgs;
1960	}
1961
1962	mutex_unlock(&ch_sn2->msg_to_pull_mutex);
1963
1964	/* return the message we were looking for */
1965	msg_offset = (get % ch->remote_nentries) * ch->entry_size;
1966	msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + msg_offset);
1967
1968	return msg;
1969}
1970
1971/*
1972 * Get the next deliverable message's payload.
1973 */
1974static void *
1975xpc_get_deliverable_payload_sn2(struct xpc_channel *ch)
1976{
1977	struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1978	struct xpc_msg_sn2 *msg;
1979	void *payload = NULL;
1980	s64 get;
1981
1982	do {
1983		if (ch->flags & XPC_C_DISCONNECTING)
1984			break;
1985
1986		get = ch_sn2->w_local_GP.get;
1987		smp_rmb();	/* guarantee that .get loads before .put */
1988		if (get == ch_sn2->w_remote_GP.put)
1989			break;
1990
1991		/* There are messages waiting to be pulled and delivered.
1992		 * We need to try to secure one for ourselves. We'll do this
1993		 * by trying to increment w_local_GP.get and hope that no one
1994		 * else beats us to it. If they do, we'll we'll simply have
1995		 * to try again for the next one.
1996		 */
1997
1998		if (cmpxchg(&ch_sn2->w_local_GP.get, get, get + 1) == get) {
1999			/* we got the entry referenced by get */
2000
2001			dev_dbg(xpc_chan, "w_local_GP.get changed to %lld, "
2002				"partid=%d, channel=%d\n", get + 1,
2003				ch->partid, ch->number);
2004
2005			/* pull the message from the remote partition */
2006
2007			msg = xpc_pull_remote_msg_sn2(ch, get);
2008
2009			if (msg != NULL) {
2010				DBUG_ON(msg->number != get);
2011				DBUG_ON(msg->flags & XPC_M_SN2_DONE);
2012				DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
2013
2014				payload = &msg->payload;
2015			}
2016			break;
2017		}
2018
2019	} while (1);
2020
2021	return payload;
2022}
2023
2024/*
2025 * Now we actually send the messages that are ready to be sent by advancing
2026 * the local message queue's Put value and then send a chctl msgrequest to the
2027 * recipient partition.
2028 */
2029static void
2030xpc_send_msgs_sn2(struct xpc_channel *ch, s64 initial_put)
2031{
2032	struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
2033	struct xpc_msg_sn2 *msg;
2034	s64 put = initial_put + 1;
2035	int send_msgrequest = 0;
2036
2037	while (1) {
2038
2039		while (1) {
2040			if (put == ch_sn2->w_local_GP.put)
2041				break;
2042
2043			msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->
2044						     local_msgqueue + (put %
2045						     ch->local_nentries) *
2046						     ch->entry_size);
2047
2048			if (!(msg->flags & XPC_M_SN2_READY))
2049				break;
2050
2051			put++;
2052		}
2053
2054		if (put == initial_put) {
2055			/* nothing's changed */
2056			break;
2057		}
2058
2059		if (cmpxchg_rel(&ch_sn2->local_GP->put, initial_put, put) !=
2060		    initial_put) {
2061			/* someone else beat us to it */
2062			DBUG_ON(ch_sn2->local_GP->put < initial_put);
2063			break;
2064		}
2065
2066		/* we just set the new value of local_GP->put */
2067
2068		dev_dbg(xpc_chan, "local_GP->put changed to %lld, partid=%d, "
2069			"channel=%d\n", put, ch->partid, ch->number);
2070
2071		send_msgrequest = 1;
2072
2073		/*
2074		 * We need to ensure that the message referenced by
2075		 * local_GP->put is not XPC_M_SN2_READY or that local_GP->put
2076		 * equals w_local_GP.put, so we'll go have a look.
2077		 */
2078		initial_put = put;
2079	}
2080
2081	if (send_msgrequest)
2082		xpc_send_chctl_msgrequest_sn2(ch);
2083}
2084
2085/*
2086 * Allocate an entry for a message from the message queue associated with the
2087 * specified channel.
2088 */
2089static enum xp_retval
2090xpc_allocate_msg_sn2(struct xpc_channel *ch, u32 flags,
2091		     struct xpc_msg_sn2 **address_of_msg)
2092{
2093	struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
2094	struct xpc_msg_sn2 *msg;
2095	enum xp_retval ret;
2096	s64 put;
2097
2098	/*
2099	 * Get the next available message entry from the local message queue.
2100	 * If none are available, we'll make sure that we grab the latest
2101	 * GP values.
2102	 */
2103	ret = xpTimeout;
2104
2105	while (1) {
2106
2107		put = ch_sn2->w_local_GP.put;
2108		smp_rmb();	/* guarantee that .put loads before .get */
2109		if (put - ch_sn2->w_remote_GP.get < ch->local_nentries) {
2110
2111			/* There are available message entries. We need to try
2112			 * to secure one for ourselves. We'll do this by trying
2113			 * to increment w_local_GP.put as long as someone else
2114			 * doesn't beat us to it. If they do, we'll have to
2115			 * try again.
2116			 */
2117			if (cmpxchg(&ch_sn2->w_local_GP.put, put, put + 1) ==
2118			    put) {
2119				/* we got the entry referenced by put */
2120				break;
2121			}
2122			continue;	/* try again */
2123		}
2124
2125		/*
2126		 * There aren't any available msg entries at this time.
2127		 *
2128		 * In waiting for a message entry to become available,
2129		 * we set a timeout in case the other side is not sending
2130		 * completion interrupts. This lets us fake a notify IRQ
2131		 * that will cause the notify IRQ handler to fetch the latest
2132		 * GP values as if an interrupt was sent by the other side.
2133		 */
2134		if (ret == xpTimeout)
2135			xpc_send_chctl_local_msgrequest_sn2(ch);
2136
2137		if (flags & XPC_NOWAIT)
2138			return xpNoWait;
2139
2140		ret = xpc_allocate_msg_wait(ch);
2141		if (ret != xpInterrupted && ret != xpTimeout)
2142			return ret;
2143	}
2144
2145	/* get the message's address and initialize it */
2146	msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue +
2147				     (put % ch->local_nentries) *
2148				     ch->entry_size);
2149
2150	DBUG_ON(msg->flags != 0);
2151	msg->number = put;
2152
2153	dev_dbg(xpc_chan, "w_local_GP.put changed to %lld; msg=0x%p, "
2154		"msg_number=%lld, partid=%d, channel=%d\n", put + 1,
2155		(void *)msg, msg->number, ch->partid, ch->number);
2156
2157	*address_of_msg = msg;
2158	return xpSuccess;
2159}
2160
2161/*
2162 * Common code that does the actual sending of the message by advancing the
2163 * local message queue's Put value and sends a chctl msgrequest to the
2164 * partition the message is being sent to.
2165 */
2166static enum xp_retval
2167xpc_send_payload_sn2(struct xpc_channel *ch, u32 flags, void *payload,
2168		     u16 payload_size, u8 notify_type, xpc_notify_func func,
2169		     void *key)
2170{
2171	enum xp_retval ret = xpSuccess;
2172	struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
2173	struct xpc_msg_sn2 *msg = msg;
2174	struct xpc_notify_sn2 *notify = notify;
2175	s64 msg_number;
2176	s64 put;
2177
2178	DBUG_ON(notify_type == XPC_N_CALL && func == NULL);
2179
2180	if (XPC_MSG_SIZE(payload_size) > ch->entry_size)
2181		return xpPayloadTooBig;
2182
2183	xpc_msgqueue_ref(ch);
2184
2185	if (ch->flags & XPC_C_DISCONNECTING) {
2186		ret = ch->reason;
2187		goto out_1;
2188	}
2189	if (!(ch->flags & XPC_C_CONNECTED)) {
2190		ret = xpNotConnected;
2191		goto out_1;
2192	}
2193
2194	ret = xpc_allocate_msg_sn2(ch, flags, &msg);
2195	if (ret != xpSuccess)
2196		goto out_1;
2197
2198	msg_number = msg->number;
2199
2200	if (notify_type != 0) {
2201		/*
2202		 * Tell the remote side to send an ACK interrupt when the
2203		 * message has been delivered.
2204		 */
2205		msg->flags |= XPC_M_SN2_INTERRUPT;
2206
2207		atomic_inc(&ch->n_to_notify);
2208
2209		notify = &ch_sn2->notify_queue[msg_number % ch->local_nentries];
2210		notify->func = func;
2211		notify->key = key;
2212		notify->type = notify_type;
2213
2214		/* ??? Is a mb() needed here? */
2215
2216		if (ch->flags & XPC_C_DISCONNECTING) {
2217			/*
2218			 * An error occurred between our last error check and
2219			 * this one. We will try to clear the type field from
2220			 * the notify entry. If we succeed then
2221			 * xpc_disconnect_channel() didn't already process
2222			 * the notify entry.
2223			 */
2224			if (cmpxchg(&notify->type, notify_type, 0) ==
2225			    notify_type) {
2226				atomic_dec(&ch->n_to_notify);
2227				ret = ch->reason;
2228			}
2229			goto out_1;
2230		}
2231	}
2232
2233	memcpy(&msg->payload, payload, payload_size);
2234
2235	msg->flags |= XPC_M_SN2_READY;
2236
2237	/*
2238	 * The preceding store of msg->flags must occur before the following
2239	 * load of local_GP->put.
2240	 */
2241	smp_mb();
2242
2243	/* see if the message is next in line to be sent, if so send it */
2244
2245	put = ch_sn2->local_GP->put;
2246	if (put == msg_number)
2247		xpc_send_msgs_sn2(ch, put);
2248
2249out_1:
2250	xpc_msgqueue_deref(ch);
2251	return ret;
2252}
2253
2254/*
2255 * Now we actually acknowledge the messages that have been delivered and ack'd
2256 * by advancing the cached remote message queue's Get value and if requested
2257 * send a chctl msgrequest to the message sender's partition.
2258 *
2259 * If a message has XPC_M_SN2_INTERRUPT set, send an interrupt to the partition
2260 * that sent the message.
2261 */
2262static void
2263xpc_acknowledge_msgs_sn2(struct xpc_channel *ch, s64 initial_get, u8 msg_flags)
2264{
2265	struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
2266	struct xpc_msg_sn2 *msg;
2267	s64 get = initial_get + 1;
2268	int send_msgrequest = 0;
2269
2270	while (1) {
2271
2272		while (1) {
2273			if (get == ch_sn2->w_local_GP.get)
2274				break;
2275
2276			msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->
2277						     remote_msgqueue + (get %
2278						     ch->remote_nentries) *
2279						     ch->entry_size);
2280
2281			if (!(msg->flags & XPC_M_SN2_DONE))
2282				break;
2283
2284			msg_flags |= msg->flags;
2285			get++;
2286		}
2287
2288		if (get == initial_get) {
2289			/* nothing's changed */
2290			break;
2291		}
2292
2293		if (cmpxchg_rel(&ch_sn2->local_GP->get, initial_get, get) !=
2294		    initial_get) {
2295			/* someone else beat us to it */
2296			DBUG_ON(ch_sn2->local_GP->get <= initial_get);
2297			break;
2298		}
2299
2300		/* we just set the new value of local_GP->get */
2301
2302		dev_dbg(xpc_chan, "local_GP->get changed to %lld, partid=%d, "
2303			"channel=%d\n", get, ch->partid, ch->number);
2304
2305		send_msgrequest = (msg_flags & XPC_M_SN2_INTERRUPT);
2306
2307		/*
2308		 * We need to ensure that the message referenced by
2309		 * local_GP->get is not XPC_M_SN2_DONE or that local_GP->get
2310		 * equals w_local_GP.get, so we'll go have a look.
2311		 */
2312		initial_get = get;
2313	}
2314
2315	if (send_msgrequest)
2316		xpc_send_chctl_msgrequest_sn2(ch);
2317}
2318
2319static void
2320xpc_received_payload_sn2(struct xpc_channel *ch, void *payload)
2321{
2322	struct xpc_msg_sn2 *msg;
2323	s64 msg_number;
2324	s64 get;
2325
2326	msg = container_of(payload, struct xpc_msg_sn2, payload);
2327	msg_number = msg->number;
2328
2329	dev_dbg(xpc_chan, "msg=0x%p, msg_number=%lld, partid=%d, channel=%d\n",
2330		(void *)msg, msg_number, ch->partid, ch->number);
2331
2332	DBUG_ON((((u64)msg - (u64)ch->sn.sn2.remote_msgqueue) / ch->entry_size) !=
2333		msg_number % ch->remote_nentries);
2334	DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
2335	DBUG_ON(msg->flags & XPC_M_SN2_DONE);
2336
2337	msg->flags |= XPC_M_SN2_DONE;
2338
2339	/*
2340	 * The preceding store of msg->flags must occur before the following
2341	 * load of local_GP->get.
2342	 */
2343	smp_mb();
2344
2345	/*
2346	 * See if this message is next in line to be acknowledged as having
2347	 * been delivered.
2348	 */
2349	get = ch->sn.sn2.local_GP->get;
2350	if (get == msg_number)
2351		xpc_acknowledge_msgs_sn2(ch, get, msg->flags);
2352}
2353
2354static struct xpc_arch_operations xpc_arch_ops_sn2 = {
2355	.setup_partitions = xpc_setup_partitions_sn2,
2356	.teardown_partitions = xpc_teardown_partitions_sn2,
2357	.process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_sn2,
2358	.get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_sn2,
2359	.setup_rsvd_page = xpc_setup_rsvd_page_sn2,
2360
2361	.allow_hb = xpc_allow_hb_sn2,
2362	.disallow_hb = xpc_disallow_hb_sn2,
2363	.disallow_all_hbs = xpc_disallow_all_hbs_sn2,
2364	.increment_heartbeat = xpc_increment_heartbeat_sn2,
2365	.offline_heartbeat = xpc_offline_heartbeat_sn2,
2366	.online_heartbeat = xpc_online_heartbeat_sn2,
2367	.heartbeat_init = xpc_heartbeat_init_sn2,
2368	.heartbeat_exit = xpc_heartbeat_exit_sn2,
2369	.get_remote_heartbeat = xpc_get_remote_heartbeat_sn2,
2370
2371	.request_partition_activation =
2372		xpc_request_partition_activation_sn2,
2373	.request_partition_reactivation =
2374		xpc_request_partition_reactivation_sn2,
2375	.request_partition_deactivation =
2376		xpc_request_partition_deactivation_sn2,
2377	.cancel_partition_deactivation_request =
2378		xpc_cancel_partition_deactivation_request_sn2,
2379
2380	.setup_ch_structures = xpc_setup_ch_structures_sn2,
2381	.teardown_ch_structures = xpc_teardown_ch_structures_sn2,
2382
2383	.make_first_contact = xpc_make_first_contact_sn2,
2384
2385	.get_chctl_all_flags = xpc_get_chctl_all_flags_sn2,
2386	.send_chctl_closerequest = xpc_send_chctl_closerequest_sn2,
2387	.send_chctl_closereply = xpc_send_chctl_closereply_sn2,
2388	.send_chctl_openrequest = xpc_send_chctl_openrequest_sn2,
2389	.send_chctl_openreply = xpc_send_chctl_openreply_sn2,
2390	.send_chctl_opencomplete = xpc_send_chctl_opencomplete_sn2,
2391	.process_msg_chctl_flags = xpc_process_msg_chctl_flags_sn2,
2392
2393	.save_remote_msgqueue_pa = xpc_save_remote_msgqueue_pa_sn2,
2394
2395	.setup_msg_structures = xpc_setup_msg_structures_sn2,
2396	.teardown_msg_structures = xpc_teardown_msg_structures_sn2,
2397
2398	.indicate_partition_engaged = xpc_indicate_partition_engaged_sn2,
2399	.indicate_partition_disengaged = xpc_indicate_partition_disengaged_sn2,
2400	.partition_engaged = xpc_partition_engaged_sn2,
2401	.any_partition_engaged = xpc_any_partition_engaged_sn2,
2402	.assume_partition_disengaged = xpc_assume_partition_disengaged_sn2,
2403
2404	.n_of_deliverable_payloads = xpc_n_of_deliverable_payloads_sn2,
2405	.send_payload = xpc_send_payload_sn2,
2406	.get_deliverable_payload = xpc_get_deliverable_payload_sn2,
2407	.received_payload = xpc_received_payload_sn2,
2408	.notify_senders_of_disconnect = xpc_notify_senders_of_disconnect_sn2,
2409};
2410
2411int
2412xpc_init_sn2(void)
2413{
2414	int ret;
2415	size_t buf_size;
2416
2417	xpc_arch_ops = xpc_arch_ops_sn2;
2418
2419	if (offsetof(struct xpc_msg_sn2, payload) > XPC_MSG_HDR_MAX_SIZE) {
2420		dev_err(xpc_part, "header portion of struct xpc_msg_sn2 is "
2421			"larger than %d\n", XPC_MSG_HDR_MAX_SIZE);
2422		return -E2BIG;
2423	}
2424
2425	buf_size = max(XPC_RP_VARS_SIZE,
2426		       XPC_RP_HEADER_SIZE + XP_NASID_MASK_BYTES_SN2);
2427	xpc_remote_copy_buffer_sn2 = xpc_kmalloc_cacheline_aligned(buf_size,
2428								   GFP_KERNEL,
2429					      &xpc_remote_copy_buffer_base_sn2);
2430	if (xpc_remote_copy_buffer_sn2 == NULL) {
2431		dev_err(xpc_part, "can't get memory for remote copy buffer\n");
2432		return -ENOMEM;
2433	}
2434
2435	/* open up protections for IPI and [potentially] amo operations */
2436	xpc_allow_IPI_ops_sn2();
2437	xpc_allow_amo_ops_shub_wars_1_1_sn2();
2438
2439	/*
2440	 * This is safe to do before the xpc_hb_checker thread has started
2441	 * because the handler releases a wait queue.  If an interrupt is
2442	 * received before the thread is waiting, it will not go to sleep,
2443	 * but rather immediately process the interrupt.
2444	 */
2445	ret = request_irq(SGI_XPC_ACTIVATE, xpc_handle_activate_IRQ_sn2, 0,
2446			  "xpc hb", NULL);
2447	if (ret != 0) {
2448		dev_err(xpc_part, "can't register ACTIVATE IRQ handler, "
2449			"errno=%d\n", -ret);
2450		xpc_disallow_IPI_ops_sn2();
2451		kfree(xpc_remote_copy_buffer_base_sn2);
2452	}
2453	return ret;
2454}
2455
2456void
2457xpc_exit_sn2(void)
2458{
2459	free_irq(SGI_XPC_ACTIVATE, NULL);
2460	xpc_disallow_IPI_ops_sn2();
2461	kfree(xpc_remote_copy_buffer_base_sn2);
2462}