1/* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (c) 2008-2009 Silicon Graphics, Inc. All Rights Reserved. 7 */ 8 9/* 10 * Cross Partition Communication (XPC) sn2-based functions. 11 * 12 * Architecture specific implementation of common functions. 13 * 14 */ 15 16#include <linux/delay.h> 17#include <linux/slab.h> 18#include <asm/uncached.h> 19#include <asm/sn/mspec.h> 20#include <asm/sn/sn_sal.h> 21#include "xpc.h" 22 23/* 24 * Define the number of u64s required to represent all the C-brick nasids 25 * as a bitmap. The cross-partition kernel modules deal only with 26 * C-brick nasids, thus the need for bitmaps which don't account for 27 * odd-numbered (non C-brick) nasids. 28 */ 29#define XPC_MAX_PHYSNODES_SN2 (MAX_NUMALINK_NODES / 2) 30#define XP_NASID_MASK_BYTES_SN2 ((XPC_MAX_PHYSNODES_SN2 + 7) / 8) 31#define XP_NASID_MASK_WORDS_SN2 ((XPC_MAX_PHYSNODES_SN2 + 63) / 64) 32 33/* 34 * Memory for XPC's amo variables is allocated by the MSPEC driver. These 35 * pages are located in the lowest granule. The lowest granule uses 4k pages 36 * for cached references and an alternate TLB handler to never provide a 37 * cacheable mapping for the entire region. This will prevent speculative 38 * reading of cached copies of our lines from being issued which will cause 39 * a PI FSB Protocol error to be generated by the SHUB. For XPC, we need 64 40 * amo variables (based on XP_MAX_NPARTITIONS_SN2) to identify the senders of 41 * NOTIFY IRQs, 128 amo variables (based on XP_NASID_MASK_WORDS_SN2) to identify 42 * the senders of ACTIVATE IRQs, 1 amo variable to identify which remote 43 * partitions (i.e., XPCs) consider themselves currently engaged with the 44 * local XPC and 1 amo variable to request partition deactivation. 45 */ 46#define XPC_NOTIFY_IRQ_AMOS_SN2 0 47#define XPC_ACTIVATE_IRQ_AMOS_SN2 (XPC_NOTIFY_IRQ_AMOS_SN2 + \ 48 XP_MAX_NPARTITIONS_SN2) 49#define XPC_ENGAGED_PARTITIONS_AMO_SN2 (XPC_ACTIVATE_IRQ_AMOS_SN2 + \ 50 XP_NASID_MASK_WORDS_SN2) 51#define XPC_DEACTIVATE_REQUEST_AMO_SN2 (XPC_ENGAGED_PARTITIONS_AMO_SN2 + 1) 52 53/* 54 * Buffer used to store a local copy of portions of a remote partition's 55 * reserved page (either its header and part_nasids mask, or its vars). 56 */ 57static void *xpc_remote_copy_buffer_base_sn2; 58static char *xpc_remote_copy_buffer_sn2; 59 60static struct xpc_vars_sn2 *xpc_vars_sn2; 61static struct xpc_vars_part_sn2 *xpc_vars_part_sn2; 62 63static int 64xpc_setup_partitions_sn2(void) 65{ 66 /* nothing needs to be done */ 67 return 0; 68} 69 70static void 71xpc_teardown_partitions_sn2(void) 72{ 73 /* nothing needs to be done */ 74} 75 76/* SH_IPI_ACCESS shub register value on startup */ 77static u64 xpc_sh1_IPI_access_sn2; 78static u64 xpc_sh2_IPI_access0_sn2; 79static u64 xpc_sh2_IPI_access1_sn2; 80static u64 xpc_sh2_IPI_access2_sn2; 81static u64 xpc_sh2_IPI_access3_sn2; 82 83/* 84 * Change protections to allow IPI operations. 85 */ 86static void 87xpc_allow_IPI_ops_sn2(void) 88{ 89 int node; 90 int nasid; 91 92 /* !!! The following should get moved into SAL. */ 93 if (is_shub2()) { 94 xpc_sh2_IPI_access0_sn2 = 95 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS0)); 96 xpc_sh2_IPI_access1_sn2 = 97 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS1)); 98 xpc_sh2_IPI_access2_sn2 = 99 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS2)); 100 xpc_sh2_IPI_access3_sn2 = 101 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS3)); 102 103 for_each_online_node(node) { 104 nasid = cnodeid_to_nasid(node); 105 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0), 106 -1UL); 107 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1), 108 -1UL); 109 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2), 110 -1UL); 111 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3), 112 -1UL); 113 } 114 } else { 115 xpc_sh1_IPI_access_sn2 = 116 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH1_IPI_ACCESS)); 117 118 for_each_online_node(node) { 119 nasid = cnodeid_to_nasid(node); 120 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS), 121 -1UL); 122 } 123 } 124} 125 126/* 127 * Restrict protections to disallow IPI operations. 128 */ 129static void 130xpc_disallow_IPI_ops_sn2(void) 131{ 132 int node; 133 int nasid; 134 135 /* !!! The following should get moved into SAL. */ 136 if (is_shub2()) { 137 for_each_online_node(node) { 138 nasid = cnodeid_to_nasid(node); 139 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0), 140 xpc_sh2_IPI_access0_sn2); 141 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1), 142 xpc_sh2_IPI_access1_sn2); 143 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2), 144 xpc_sh2_IPI_access2_sn2); 145 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3), 146 xpc_sh2_IPI_access3_sn2); 147 } 148 } else { 149 for_each_online_node(node) { 150 nasid = cnodeid_to_nasid(node); 151 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS), 152 xpc_sh1_IPI_access_sn2); 153 } 154 } 155} 156 157/* 158 * The following set of functions are used for the sending and receiving of 159 * IRQs (also known as IPIs). There are two flavors of IRQs, one that is 160 * associated with partition activity (SGI_XPC_ACTIVATE) and the other that 161 * is associated with channel activity (SGI_XPC_NOTIFY). 162 */ 163 164static u64 165xpc_receive_IRQ_amo_sn2(struct amo *amo) 166{ 167 return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_CLEAR); 168} 169 170static enum xp_retval 171xpc_send_IRQ_sn2(struct amo *amo, u64 flag, int nasid, int phys_cpuid, 172 int vector) 173{ 174 int ret = 0; 175 unsigned long irq_flags; 176 177 local_irq_save(irq_flags); 178 179 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, flag); 180 sn_send_IPI_phys(nasid, phys_cpuid, vector, 0); 181 182 /* 183 * We must always use the nofault function regardless of whether we 184 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we 185 * didn't, we'd never know that the other partition is down and would 186 * keep sending IRQs and amos to it until the heartbeat times out. 187 */ 188 ret = xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->variable), 189 xp_nofault_PIOR_target)); 190 191 local_irq_restore(irq_flags); 192 193 return (ret == 0) ? xpSuccess : xpPioReadError; 194} 195 196static struct amo * 197xpc_init_IRQ_amo_sn2(int index) 198{ 199 struct amo *amo = xpc_vars_sn2->amos_page + index; 200 201 (void)xpc_receive_IRQ_amo_sn2(amo); /* clear amo variable */ 202 return amo; 203} 204 205/* 206 * Functions associated with SGI_XPC_ACTIVATE IRQ. 207 */ 208 209/* 210 * Notify the heartbeat check thread that an activate IRQ has been received. 211 */ 212static irqreturn_t 213xpc_handle_activate_IRQ_sn2(int irq, void *dev_id) 214{ 215 unsigned long irq_flags; 216 217 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); 218 xpc_activate_IRQ_rcvd++; 219 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); 220 221 wake_up_interruptible(&xpc_activate_IRQ_wq); 222 return IRQ_HANDLED; 223} 224 225/* 226 * Flag the appropriate amo variable and send an IRQ to the specified node. 227 */ 228static void 229xpc_send_activate_IRQ_sn2(unsigned long amos_page_pa, int from_nasid, 230 int to_nasid, int to_phys_cpuid) 231{ 232 struct amo *amos = (struct amo *)__va(amos_page_pa + 233 (XPC_ACTIVATE_IRQ_AMOS_SN2 * 234 sizeof(struct amo))); 235 236 (void)xpc_send_IRQ_sn2(&amos[BIT_WORD(from_nasid / 2)], 237 BIT_MASK(from_nasid / 2), to_nasid, 238 to_phys_cpuid, SGI_XPC_ACTIVATE); 239} 240 241static void 242xpc_send_local_activate_IRQ_sn2(int from_nasid) 243{ 244 unsigned long irq_flags; 245 struct amo *amos = (struct amo *)__va(xpc_vars_sn2->amos_page_pa + 246 (XPC_ACTIVATE_IRQ_AMOS_SN2 * 247 sizeof(struct amo))); 248 249 /* fake the sending and receipt of an activate IRQ from remote nasid */ 250 FETCHOP_STORE_OP(TO_AMO((u64)&amos[BIT_WORD(from_nasid / 2)].variable), 251 FETCHOP_OR, BIT_MASK(from_nasid / 2)); 252 253 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); 254 xpc_activate_IRQ_rcvd++; 255 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); 256 257 wake_up_interruptible(&xpc_activate_IRQ_wq); 258} 259 260/* 261 * Functions associated with SGI_XPC_NOTIFY IRQ. 262 */ 263 264/* 265 * Check to see if any chctl flags were sent from the specified partition. 266 */ 267static void 268xpc_check_for_sent_chctl_flags_sn2(struct xpc_partition *part) 269{ 270 union xpc_channel_ctl_flags chctl; 271 unsigned long irq_flags; 272 273 chctl.all_flags = xpc_receive_IRQ_amo_sn2(part->sn.sn2. 274 local_chctl_amo_va); 275 if (chctl.all_flags == 0) 276 return; 277 278 spin_lock_irqsave(&part->chctl_lock, irq_flags); 279 part->chctl.all_flags |= chctl.all_flags; 280 spin_unlock_irqrestore(&part->chctl_lock, irq_flags); 281 282 dev_dbg(xpc_chan, "received notify IRQ from partid=%d, chctl.all_flags=" 283 "0x%llx\n", XPC_PARTID(part), chctl.all_flags); 284 285 xpc_wakeup_channel_mgr(part); 286} 287 288/* 289 * Handle the receipt of a SGI_XPC_NOTIFY IRQ by seeing whether the specified 290 * partition actually sent it. Since SGI_XPC_NOTIFY IRQs may be shared by more 291 * than one partition, we use an amo structure per partition to indicate 292 * whether a partition has sent an IRQ or not. If it has, then wake up the 293 * associated kthread to handle it. 294 * 295 * All SGI_XPC_NOTIFY IRQs received by XPC are the result of IRQs sent by XPC 296 * running on other partitions. 297 * 298 * Noteworthy Arguments: 299 * 300 * irq - Interrupt ReQuest number. NOT USED. 301 * 302 * dev_id - partid of IRQ's potential sender. 303 */ 304static irqreturn_t 305xpc_handle_notify_IRQ_sn2(int irq, void *dev_id) 306{ 307 short partid = (short)(u64)dev_id; 308 struct xpc_partition *part = &xpc_partitions[partid]; 309 310 DBUG_ON(partid < 0 || partid >= XP_MAX_NPARTITIONS_SN2); 311 312 if (xpc_part_ref(part)) { 313 xpc_check_for_sent_chctl_flags_sn2(part); 314 315 xpc_part_deref(part); 316 } 317 return IRQ_HANDLED; 318} 319 320/* 321 * Check to see if xpc_handle_notify_IRQ_sn2() dropped any IRQs on the floor 322 * because the write to their associated amo variable completed after the IRQ 323 * was received. 324 */ 325static void 326xpc_check_for_dropped_notify_IRQ_sn2(struct xpc_partition *part) 327{ 328 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; 329 330 if (xpc_part_ref(part)) { 331 xpc_check_for_sent_chctl_flags_sn2(part); 332 333 part_sn2->dropped_notify_IRQ_timer.expires = jiffies + 334 XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL; 335 add_timer(&part_sn2->dropped_notify_IRQ_timer); 336 xpc_part_deref(part); 337 } 338} 339 340/* 341 * Send a notify IRQ to the remote partition that is associated with the 342 * specified channel. 343 */ 344static void 345xpc_send_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag, 346 char *chctl_flag_string, unsigned long *irq_flags) 347{ 348 struct xpc_partition *part = &xpc_partitions[ch->partid]; 349 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; 350 union xpc_channel_ctl_flags chctl = { 0 }; 351 enum xp_retval ret; 352 353 if (likely(part->act_state != XPC_P_AS_DEACTIVATING)) { 354 chctl.flags[ch->number] = chctl_flag; 355 ret = xpc_send_IRQ_sn2(part_sn2->remote_chctl_amo_va, 356 chctl.all_flags, 357 part_sn2->notify_IRQ_nasid, 358 part_sn2->notify_IRQ_phys_cpuid, 359 SGI_XPC_NOTIFY); 360 dev_dbg(xpc_chan, "%s sent to partid=%d, channel=%d, ret=%d\n", 361 chctl_flag_string, ch->partid, ch->number, ret); 362 if (unlikely(ret != xpSuccess)) { 363 if (irq_flags != NULL) 364 spin_unlock_irqrestore(&ch->lock, *irq_flags); 365 XPC_DEACTIVATE_PARTITION(part, ret); 366 if (irq_flags != NULL) 367 spin_lock_irqsave(&ch->lock, *irq_flags); 368 } 369 } 370} 371 372#define XPC_SEND_NOTIFY_IRQ_SN2(_ch, _ipi_f, _irq_f) \ 373 xpc_send_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f, _irq_f) 374 375/* 376 * Make it look like the remote partition, which is associated with the 377 * specified channel, sent us a notify IRQ. This faked IRQ will be handled 378 * by xpc_check_for_dropped_notify_IRQ_sn2(). 379 */ 380static void 381xpc_send_local_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag, 382 char *chctl_flag_string) 383{ 384 struct xpc_partition *part = &xpc_partitions[ch->partid]; 385 union xpc_channel_ctl_flags chctl = { 0 }; 386 387 chctl.flags[ch->number] = chctl_flag; 388 FETCHOP_STORE_OP(TO_AMO((u64)&part->sn.sn2.local_chctl_amo_va-> 389 variable), FETCHOP_OR, chctl.all_flags); 390 dev_dbg(xpc_chan, "%s sent local from partid=%d, channel=%d\n", 391 chctl_flag_string, ch->partid, ch->number); 392} 393 394#define XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(_ch, _ipi_f) \ 395 xpc_send_local_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f) 396 397static void 398xpc_send_chctl_closerequest_sn2(struct xpc_channel *ch, 399 unsigned long *irq_flags) 400{ 401 struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args; 402 403 args->reason = ch->reason; 404 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREQUEST, irq_flags); 405} 406 407static void 408xpc_send_chctl_closereply_sn2(struct xpc_channel *ch, unsigned long *irq_flags) 409{ 410 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREPLY, irq_flags); 411} 412 413static void 414xpc_send_chctl_openrequest_sn2(struct xpc_channel *ch, unsigned long *irq_flags) 415{ 416 struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args; 417 418 args->entry_size = ch->entry_size; 419 args->local_nentries = ch->local_nentries; 420 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREQUEST, irq_flags); 421} 422 423static void 424xpc_send_chctl_openreply_sn2(struct xpc_channel *ch, unsigned long *irq_flags) 425{ 426 struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args; 427 428 args->remote_nentries = ch->remote_nentries; 429 args->local_nentries = ch->local_nentries; 430 args->local_msgqueue_pa = xp_pa(ch->sn.sn2.local_msgqueue); 431 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREPLY, irq_flags); 432} 433 434static void 435xpc_send_chctl_opencomplete_sn2(struct xpc_channel *ch, 436 unsigned long *irq_flags) 437{ 438 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENCOMPLETE, irq_flags); 439} 440 441static void 442xpc_send_chctl_msgrequest_sn2(struct xpc_channel *ch) 443{ 444 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST, NULL); 445} 446 447static void 448xpc_send_chctl_local_msgrequest_sn2(struct xpc_channel *ch) 449{ 450 XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST); 451} 452 453static enum xp_retval 454xpc_save_remote_msgqueue_pa_sn2(struct xpc_channel *ch, 455 unsigned long msgqueue_pa) 456{ 457 ch->sn.sn2.remote_msgqueue_pa = msgqueue_pa; 458 return xpSuccess; 459} 460 461/* 462 * This next set of functions are used to keep track of when a partition is 463 * potentially engaged in accessing memory belonging to another partition. 464 */ 465 466static void 467xpc_indicate_partition_engaged_sn2(struct xpc_partition *part) 468{ 469 unsigned long irq_flags; 470 struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa + 471 (XPC_ENGAGED_PARTITIONS_AMO_SN2 * 472 sizeof(struct amo))); 473 474 local_irq_save(irq_flags); 475 476 /* set bit corresponding to our partid in remote partition's amo */ 477 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, 478 BIT(sn_partition_id)); 479 480 /* 481 * We must always use the nofault function regardless of whether we 482 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we 483 * didn't, we'd never know that the other partition is down and would 484 * keep sending IRQs and amos to it until the heartbeat times out. 485 */ 486 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> 487 variable), 488 xp_nofault_PIOR_target)); 489 490 local_irq_restore(irq_flags); 491} 492 493static void 494xpc_indicate_partition_disengaged_sn2(struct xpc_partition *part) 495{ 496 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; 497 unsigned long irq_flags; 498 struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa + 499 (XPC_ENGAGED_PARTITIONS_AMO_SN2 * 500 sizeof(struct amo))); 501 502 local_irq_save(irq_flags); 503 504 /* clear bit corresponding to our partid in remote partition's amo */ 505 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND, 506 ~BIT(sn_partition_id)); 507 508 /* 509 * We must always use the nofault function regardless of whether we 510 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we 511 * didn't, we'd never know that the other partition is down and would 512 * keep sending IRQs and amos to it until the heartbeat times out. 513 */ 514 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> 515 variable), 516 xp_nofault_PIOR_target)); 517 518 local_irq_restore(irq_flags); 519 520 /* 521 * Send activate IRQ to get other side to see that we've cleared our 522 * bit in their engaged partitions amo. 523 */ 524 xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa, 525 cnodeid_to_nasid(0), 526 part_sn2->activate_IRQ_nasid, 527 part_sn2->activate_IRQ_phys_cpuid); 528} 529 530static void 531xpc_assume_partition_disengaged_sn2(short partid) 532{ 533 struct amo *amo = xpc_vars_sn2->amos_page + 534 XPC_ENGAGED_PARTITIONS_AMO_SN2; 535 536 /* clear bit(s) based on partid mask in our partition's amo */ 537 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND, 538 ~BIT(partid)); 539} 540 541static int 542xpc_partition_engaged_sn2(short partid) 543{ 544 struct amo *amo = xpc_vars_sn2->amos_page + 545 XPC_ENGAGED_PARTITIONS_AMO_SN2; 546 547 /* our partition's amo variable ANDed with partid mask */ 548 return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) & 549 BIT(partid)) != 0; 550} 551 552static int 553xpc_any_partition_engaged_sn2(void) 554{ 555 struct amo *amo = xpc_vars_sn2->amos_page + 556 XPC_ENGAGED_PARTITIONS_AMO_SN2; 557 558 /* our partition's amo variable */ 559 return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) != 0; 560} 561 562/* original protection values for each node */ 563static u64 xpc_prot_vec_sn2[MAX_NUMNODES]; 564 565/* 566 * Change protections to allow amo operations on non-Shub 1.1 systems. 567 */ 568static enum xp_retval 569xpc_allow_amo_ops_sn2(struct amo *amos_page) 570{ 571 enum xp_retval ret = xpSuccess; 572 573 /* 574 * On SHUB 1.1, we cannot call sn_change_memprotect() since the BIST 575 * collides with memory operations. On those systems we call 576 * xpc_allow_amo_ops_shub_wars_1_1_sn2() instead. 577 */ 578 if (!enable_shub_wars_1_1()) 579 ret = xp_expand_memprotect(ia64_tpa((u64)amos_page), PAGE_SIZE); 580 581 return ret; 582} 583 584/* 585 * Change protections to allow amo operations on Shub 1.1 systems. 586 */ 587static void 588xpc_allow_amo_ops_shub_wars_1_1_sn2(void) 589{ 590 int node; 591 int nasid; 592 593 if (!enable_shub_wars_1_1()) 594 return; 595 596 for_each_online_node(node) { 597 nasid = cnodeid_to_nasid(node); 598 /* save current protection values */ 599 xpc_prot_vec_sn2[node] = 600 (u64)HUB_L((u64 *)GLOBAL_MMR_ADDR(nasid, 601 SH1_MD_DQLP_MMR_DIR_PRIVEC0)); 602 /* open up everything */ 603 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, 604 SH1_MD_DQLP_MMR_DIR_PRIVEC0), 605 -1UL); 606 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, 607 SH1_MD_DQRP_MMR_DIR_PRIVEC0), 608 -1UL); 609 } 610} 611 612static enum xp_retval 613xpc_get_partition_rsvd_page_pa_sn2(void *buf, u64 *cookie, unsigned long *rp_pa, 614 size_t *len) 615{ 616 s64 status; 617 enum xp_retval ret; 618 619 status = sn_partition_reserved_page_pa((u64)buf, cookie, 620 (u64 *)rp_pa, (u64 *)len); 621 if (status == SALRET_OK) 622 ret = xpSuccess; 623 else if (status == SALRET_MORE_PASSES) 624 ret = xpNeedMoreInfo; 625 else 626 ret = xpSalError; 627 628 return ret; 629} 630 631 632static int 633xpc_setup_rsvd_page_sn2(struct xpc_rsvd_page *rp) 634{ 635 struct amo *amos_page; 636 int i; 637 int ret; 638 639 xpc_vars_sn2 = XPC_RP_VARS(rp); 640 641 rp->sn.sn2.vars_pa = xp_pa(xpc_vars_sn2); 642 643 /* vars_part array follows immediately after vars */ 644 xpc_vars_part_sn2 = (struct xpc_vars_part_sn2 *)((u8 *)XPC_RP_VARS(rp) + 645 XPC_RP_VARS_SIZE); 646 647 /* 648 * Before clearing xpc_vars_sn2, see if a page of amos had been 649 * previously allocated. If not we'll need to allocate one and set 650 * permissions so that cross-partition amos are allowed. 651 * 652 * The allocated amo page needs MCA reporting to remain disabled after 653 * XPC has unloaded. To make this work, we keep a copy of the pointer 654 * to this page (i.e., amos_page) in the struct xpc_vars_sn2 structure, 655 * which is pointed to by the reserved page, and re-use that saved copy 656 * on subsequent loads of XPC. This amo page is never freed, and its 657 * memory protections are never restricted. 658 */ 659 amos_page = xpc_vars_sn2->amos_page; 660 if (amos_page == NULL) { 661 amos_page = (struct amo *)TO_AMO(uncached_alloc_page(0, 1)); 662 if (amos_page == NULL) { 663 dev_err(xpc_part, "can't allocate page of amos\n"); 664 return -ENOMEM; 665 } 666 667 /* 668 * Open up amo-R/W to cpu. This is done on Shub 1.1 systems 669 * when xpc_allow_amo_ops_shub_wars_1_1_sn2() is called. 670 */ 671 ret = xpc_allow_amo_ops_sn2(amos_page); 672 if (ret != xpSuccess) { 673 dev_err(xpc_part, "can't allow amo operations\n"); 674 uncached_free_page(__IA64_UNCACHED_OFFSET | 675 TO_PHYS((u64)amos_page), 1); 676 return -EPERM; 677 } 678 } 679 680 /* clear xpc_vars_sn2 */ 681 memset(xpc_vars_sn2, 0, sizeof(struct xpc_vars_sn2)); 682 683 xpc_vars_sn2->version = XPC_V_VERSION; 684 xpc_vars_sn2->activate_IRQ_nasid = cpuid_to_nasid(0); 685 xpc_vars_sn2->activate_IRQ_phys_cpuid = cpu_physical_id(0); 686 xpc_vars_sn2->vars_part_pa = xp_pa(xpc_vars_part_sn2); 687 xpc_vars_sn2->amos_page_pa = ia64_tpa((u64)amos_page); 688 xpc_vars_sn2->amos_page = amos_page; /* save for next load of XPC */ 689 690 /* clear xpc_vars_part_sn2 */ 691 memset((u64 *)xpc_vars_part_sn2, 0, sizeof(struct xpc_vars_part_sn2) * 692 XP_MAX_NPARTITIONS_SN2); 693 694 /* initialize the activate IRQ related amo variables */ 695 for (i = 0; i < xpc_nasid_mask_nlongs; i++) 696 (void)xpc_init_IRQ_amo_sn2(XPC_ACTIVATE_IRQ_AMOS_SN2 + i); 697 698 /* initialize the engaged remote partitions related amo variables */ 699 (void)xpc_init_IRQ_amo_sn2(XPC_ENGAGED_PARTITIONS_AMO_SN2); 700 (void)xpc_init_IRQ_amo_sn2(XPC_DEACTIVATE_REQUEST_AMO_SN2); 701 702 return 0; 703} 704 705static int 706xpc_hb_allowed_sn2(short partid, void *heartbeating_to_mask) 707{ 708 return test_bit(partid, heartbeating_to_mask); 709} 710 711static void 712xpc_allow_hb_sn2(short partid) 713{ 714 DBUG_ON(xpc_vars_sn2 == NULL); 715 set_bit(partid, xpc_vars_sn2->heartbeating_to_mask); 716} 717 718static void 719xpc_disallow_hb_sn2(short partid) 720{ 721 DBUG_ON(xpc_vars_sn2 == NULL); 722 clear_bit(partid, xpc_vars_sn2->heartbeating_to_mask); 723} 724 725static void 726xpc_disallow_all_hbs_sn2(void) 727{ 728 DBUG_ON(xpc_vars_sn2 == NULL); 729 bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, xp_max_npartitions); 730} 731 732static void 733xpc_increment_heartbeat_sn2(void) 734{ 735 xpc_vars_sn2->heartbeat++; 736} 737 738static void 739xpc_offline_heartbeat_sn2(void) 740{ 741 xpc_increment_heartbeat_sn2(); 742 xpc_vars_sn2->heartbeat_offline = 1; 743} 744 745static void 746xpc_online_heartbeat_sn2(void) 747{ 748 xpc_increment_heartbeat_sn2(); 749 xpc_vars_sn2->heartbeat_offline = 0; 750} 751 752static void 753xpc_heartbeat_init_sn2(void) 754{ 755 DBUG_ON(xpc_vars_sn2 == NULL); 756 757 bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, XP_MAX_NPARTITIONS_SN2); 758 xpc_online_heartbeat_sn2(); 759} 760 761static void 762xpc_heartbeat_exit_sn2(void) 763{ 764 xpc_offline_heartbeat_sn2(); 765} 766 767static enum xp_retval 768xpc_get_remote_heartbeat_sn2(struct xpc_partition *part) 769{ 770 struct xpc_vars_sn2 *remote_vars; 771 enum xp_retval ret; 772 773 remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2; 774 775 /* pull the remote vars structure that contains the heartbeat */ 776 ret = xp_remote_memcpy(xp_pa(remote_vars), 777 part->sn.sn2.remote_vars_pa, 778 XPC_RP_VARS_SIZE); 779 if (ret != xpSuccess) 780 return ret; 781 782 dev_dbg(xpc_part, "partid=%d, heartbeat=%lld, last_heartbeat=%lld, " 783 "heartbeat_offline=%lld, HB_mask[0]=0x%lx\n", XPC_PARTID(part), 784 remote_vars->heartbeat, part->last_heartbeat, 785 remote_vars->heartbeat_offline, 786 remote_vars->heartbeating_to_mask[0]); 787 788 if ((remote_vars->heartbeat == part->last_heartbeat && 789 !remote_vars->heartbeat_offline) || 790 !xpc_hb_allowed_sn2(sn_partition_id, 791 remote_vars->heartbeating_to_mask)) { 792 ret = xpNoHeartbeat; 793 } else { 794 part->last_heartbeat = remote_vars->heartbeat; 795 } 796 797 return ret; 798} 799 800/* 801 * Get a copy of the remote partition's XPC variables from the reserved page. 802 * 803 * remote_vars points to a buffer that is cacheline aligned for BTE copies and 804 * assumed to be of size XPC_RP_VARS_SIZE. 805 */ 806static enum xp_retval 807xpc_get_remote_vars_sn2(unsigned long remote_vars_pa, 808 struct xpc_vars_sn2 *remote_vars) 809{ 810 enum xp_retval ret; 811 812 if (remote_vars_pa == 0) 813 return xpVarsNotSet; 814 815 /* pull over the cross partition variables */ 816 ret = xp_remote_memcpy(xp_pa(remote_vars), remote_vars_pa, 817 XPC_RP_VARS_SIZE); 818 if (ret != xpSuccess) 819 return ret; 820 821 if (XPC_VERSION_MAJOR(remote_vars->version) != 822 XPC_VERSION_MAJOR(XPC_V_VERSION)) { 823 return xpBadVersion; 824 } 825 826 return xpSuccess; 827} 828 829static void 830xpc_request_partition_activation_sn2(struct xpc_rsvd_page *remote_rp, 831 unsigned long remote_rp_pa, int nasid) 832{ 833 xpc_send_local_activate_IRQ_sn2(nasid); 834} 835 836static void 837xpc_request_partition_reactivation_sn2(struct xpc_partition *part) 838{ 839 xpc_send_local_activate_IRQ_sn2(part->sn.sn2.activate_IRQ_nasid); 840} 841 842static void 843xpc_request_partition_deactivation_sn2(struct xpc_partition *part) 844{ 845 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; 846 unsigned long irq_flags; 847 struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa + 848 (XPC_DEACTIVATE_REQUEST_AMO_SN2 * 849 sizeof(struct amo))); 850 851 local_irq_save(irq_flags); 852 853 /* set bit corresponding to our partid in remote partition's amo */ 854 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, 855 BIT(sn_partition_id)); 856 857 /* 858 * We must always use the nofault function regardless of whether we 859 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we 860 * didn't, we'd never know that the other partition is down and would 861 * keep sending IRQs and amos to it until the heartbeat times out. 862 */ 863 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> 864 variable), 865 xp_nofault_PIOR_target)); 866 867 local_irq_restore(irq_flags); 868 869 /* 870 * Send activate IRQ to get other side to see that we've set our 871 * bit in their deactivate request amo. 872 */ 873 xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa, 874 cnodeid_to_nasid(0), 875 part_sn2->activate_IRQ_nasid, 876 part_sn2->activate_IRQ_phys_cpuid); 877} 878 879static void 880xpc_cancel_partition_deactivation_request_sn2(struct xpc_partition *part) 881{ 882 unsigned long irq_flags; 883 struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa + 884 (XPC_DEACTIVATE_REQUEST_AMO_SN2 * 885 sizeof(struct amo))); 886 887 local_irq_save(irq_flags); 888 889 /* clear bit corresponding to our partid in remote partition's amo */ 890 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND, 891 ~BIT(sn_partition_id)); 892 893 /* 894 * We must always use the nofault function regardless of whether we 895 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we 896 * didn't, we'd never know that the other partition is down and would 897 * keep sending IRQs and amos to it until the heartbeat times out. 898 */ 899 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> 900 variable), 901 xp_nofault_PIOR_target)); 902 903 local_irq_restore(irq_flags); 904} 905 906static int 907xpc_partition_deactivation_requested_sn2(short partid) 908{ 909 struct amo *amo = xpc_vars_sn2->amos_page + 910 XPC_DEACTIVATE_REQUEST_AMO_SN2; 911 912 /* our partition's amo variable ANDed with partid mask */ 913 return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) & 914 BIT(partid)) != 0; 915} 916 917/* 918 * Update the remote partition's info. 919 */ 920static void 921xpc_update_partition_info_sn2(struct xpc_partition *part, u8 remote_rp_version, 922 unsigned long *remote_rp_ts_jiffies, 923 unsigned long remote_rp_pa, 924 unsigned long remote_vars_pa, 925 struct xpc_vars_sn2 *remote_vars) 926{ 927 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; 928 929 part->remote_rp_version = remote_rp_version; 930 dev_dbg(xpc_part, " remote_rp_version = 0x%016x\n", 931 part->remote_rp_version); 932 933 part->remote_rp_ts_jiffies = *remote_rp_ts_jiffies; 934 dev_dbg(xpc_part, " remote_rp_ts_jiffies = 0x%016lx\n", 935 part->remote_rp_ts_jiffies); 936 937 part->remote_rp_pa = remote_rp_pa; 938 dev_dbg(xpc_part, " remote_rp_pa = 0x%016lx\n", part->remote_rp_pa); 939 940 part_sn2->remote_vars_pa = remote_vars_pa; 941 dev_dbg(xpc_part, " remote_vars_pa = 0x%016lx\n", 942 part_sn2->remote_vars_pa); 943 944 part->last_heartbeat = remote_vars->heartbeat - 1; 945 dev_dbg(xpc_part, " last_heartbeat = 0x%016llx\n", 946 part->last_heartbeat); 947 948 part_sn2->remote_vars_part_pa = remote_vars->vars_part_pa; 949 dev_dbg(xpc_part, " remote_vars_part_pa = 0x%016lx\n", 950 part_sn2->remote_vars_part_pa); 951 952 part_sn2->activate_IRQ_nasid = remote_vars->activate_IRQ_nasid; 953 dev_dbg(xpc_part, " activate_IRQ_nasid = 0x%x\n", 954 part_sn2->activate_IRQ_nasid); 955 956 part_sn2->activate_IRQ_phys_cpuid = 957 remote_vars->activate_IRQ_phys_cpuid; 958 dev_dbg(xpc_part, " activate_IRQ_phys_cpuid = 0x%x\n", 959 part_sn2->activate_IRQ_phys_cpuid); 960 961 part_sn2->remote_amos_page_pa = remote_vars->amos_page_pa; 962 dev_dbg(xpc_part, " remote_amos_page_pa = 0x%lx\n", 963 part_sn2->remote_amos_page_pa); 964 965 part_sn2->remote_vars_version = remote_vars->version; 966 dev_dbg(xpc_part, " remote_vars_version = 0x%x\n", 967 part_sn2->remote_vars_version); 968} 969 970/* 971 * Prior code has determined the nasid which generated a activate IRQ. 972 * Inspect that nasid to determine if its partition needs to be activated 973 * or deactivated. 974 * 975 * A partition is considered "awaiting activation" if our partition 976 * flags indicate it is not active and it has a heartbeat. A 977 * partition is considered "awaiting deactivation" if our partition 978 * flags indicate it is active but it has no heartbeat or it is not 979 * sending its heartbeat to us. 980 * 981 * To determine the heartbeat, the remote nasid must have a properly 982 * initialized reserved page. 983 */ 984static void 985xpc_identify_activate_IRQ_req_sn2(int nasid) 986{ 987 struct xpc_rsvd_page *remote_rp; 988 struct xpc_vars_sn2 *remote_vars; 989 unsigned long remote_rp_pa; 990 unsigned long remote_vars_pa; 991 int remote_rp_version; 992 int reactivate = 0; 993 unsigned long remote_rp_ts_jiffies = 0; 994 short partid; 995 struct xpc_partition *part; 996 struct xpc_partition_sn2 *part_sn2; 997 enum xp_retval ret; 998 999 /* pull over the reserved page structure */ 1000 1001 remote_rp = (struct xpc_rsvd_page *)xpc_remote_copy_buffer_sn2; 1002 1003 ret = xpc_get_remote_rp(nasid, NULL, remote_rp, &remote_rp_pa); 1004 if (ret != xpSuccess) { 1005 dev_warn(xpc_part, "unable to get reserved page from nasid %d, " 1006 "which sent interrupt, reason=%d\n", nasid, ret); 1007 return; 1008 } 1009 1010 remote_vars_pa = remote_rp->sn.sn2.vars_pa; 1011 remote_rp_version = remote_rp->version; 1012 remote_rp_ts_jiffies = remote_rp->ts_jiffies; 1013 1014 partid = remote_rp->SAL_partid; 1015 part = &xpc_partitions[partid]; 1016 part_sn2 = &part->sn.sn2; 1017 1018 /* pull over the cross partition variables */ 1019 1020 remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2; 1021 1022 ret = xpc_get_remote_vars_sn2(remote_vars_pa, remote_vars); 1023 if (ret != xpSuccess) { 1024 dev_warn(xpc_part, "unable to get XPC variables from nasid %d, " 1025 "which sent interrupt, reason=%d\n", nasid, ret); 1026 1027 XPC_DEACTIVATE_PARTITION(part, ret); 1028 return; 1029 } 1030 1031 part->activate_IRQ_rcvd++; 1032 1033 dev_dbg(xpc_part, "partid for nasid %d is %d; IRQs = %d; HB = " 1034 "%lld:0x%lx\n", (int)nasid, (int)partid, 1035 part->activate_IRQ_rcvd, 1036 remote_vars->heartbeat, remote_vars->heartbeating_to_mask[0]); 1037 1038 if (xpc_partition_disengaged(part) && 1039 part->act_state == XPC_P_AS_INACTIVE) { 1040 1041 xpc_update_partition_info_sn2(part, remote_rp_version, 1042 &remote_rp_ts_jiffies, 1043 remote_rp_pa, remote_vars_pa, 1044 remote_vars); 1045 1046 if (xpc_partition_deactivation_requested_sn2(partid)) { 1047 /* 1048 * Other side is waiting on us to deactivate even though 1049 * we already have. 1050 */ 1051 return; 1052 } 1053 1054 xpc_activate_partition(part); 1055 return; 1056 } 1057 1058 DBUG_ON(part->remote_rp_version == 0); 1059 DBUG_ON(part_sn2->remote_vars_version == 0); 1060 1061 if (remote_rp_ts_jiffies != part->remote_rp_ts_jiffies) { 1062 1063 /* the other side rebooted */ 1064 1065 DBUG_ON(xpc_partition_engaged_sn2(partid)); 1066 DBUG_ON(xpc_partition_deactivation_requested_sn2(partid)); 1067 1068 xpc_update_partition_info_sn2(part, remote_rp_version, 1069 &remote_rp_ts_jiffies, 1070 remote_rp_pa, remote_vars_pa, 1071 remote_vars); 1072 reactivate = 1; 1073 } 1074 1075 if (part->disengage_timeout > 0 && !xpc_partition_disengaged(part)) { 1076 /* still waiting on other side to disengage from us */ 1077 return; 1078 } 1079 1080 if (reactivate) 1081 XPC_DEACTIVATE_PARTITION(part, xpReactivating); 1082 else if (xpc_partition_deactivation_requested_sn2(partid)) 1083 XPC_DEACTIVATE_PARTITION(part, xpOtherGoingDown); 1084} 1085 1086/* 1087 * Loop through the activation amo variables and process any bits 1088 * which are set. Each bit indicates a nasid sending a partition 1089 * activation or deactivation request. 1090 * 1091 * Return #of IRQs detected. 1092 */ 1093int 1094xpc_identify_activate_IRQ_sender_sn2(void) 1095{ 1096 int l; 1097 int b; 1098 unsigned long nasid_mask_long; 1099 u64 nasid; /* remote nasid */ 1100 int n_IRQs_detected = 0; 1101 struct amo *act_amos; 1102 1103 act_amos = xpc_vars_sn2->amos_page + XPC_ACTIVATE_IRQ_AMOS_SN2; 1104 1105 /* scan through activate amo variables looking for non-zero entries */ 1106 for (l = 0; l < xpc_nasid_mask_nlongs; l++) { 1107 1108 if (xpc_exiting) 1109 break; 1110 1111 nasid_mask_long = xpc_receive_IRQ_amo_sn2(&act_amos[l]); 1112 1113 b = find_first_bit(&nasid_mask_long, BITS_PER_LONG); 1114 if (b >= BITS_PER_LONG) { 1115 /* no IRQs from nasids in this amo variable */ 1116 continue; 1117 } 1118 1119 dev_dbg(xpc_part, "amo[%d] gave back 0x%lx\n", l, 1120 nasid_mask_long); 1121 1122 /* 1123 * If this nasid has been added to the machine since 1124 * our partition was reset, this will retain the 1125 * remote nasid in our reserved pages machine mask. 1126 * This is used in the event of module reload. 1127 */ 1128 xpc_mach_nasids[l] |= nasid_mask_long; 1129 1130 /* locate the nasid(s) which sent interrupts */ 1131 1132 do { 1133 n_IRQs_detected++; 1134 nasid = (l * BITS_PER_LONG + b) * 2; 1135 dev_dbg(xpc_part, "interrupt from nasid %lld\n", nasid); 1136 xpc_identify_activate_IRQ_req_sn2(nasid); 1137 1138 b = find_next_bit(&nasid_mask_long, BITS_PER_LONG, 1139 b + 1); 1140 } while (b < BITS_PER_LONG); 1141 } 1142 return n_IRQs_detected; 1143} 1144 1145static void 1146xpc_process_activate_IRQ_rcvd_sn2(void) 1147{ 1148 unsigned long irq_flags; 1149 int n_IRQs_expected; 1150 int n_IRQs_detected; 1151 1152 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); 1153 n_IRQs_expected = xpc_activate_IRQ_rcvd; 1154 xpc_activate_IRQ_rcvd = 0; 1155 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); 1156 1157 n_IRQs_detected = xpc_identify_activate_IRQ_sender_sn2(); 1158 if (n_IRQs_detected < n_IRQs_expected) { 1159 /* retry once to help avoid missing amo */ 1160 (void)xpc_identify_activate_IRQ_sender_sn2(); 1161 } 1162} 1163 1164/* 1165 * Setup the channel structures that are sn2 specific. 1166 */ 1167static enum xp_retval 1168xpc_setup_ch_structures_sn2(struct xpc_partition *part) 1169{ 1170 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; 1171 struct xpc_channel_sn2 *ch_sn2; 1172 enum xp_retval retval; 1173 int ret; 1174 int cpuid; 1175 int ch_number; 1176 struct timer_list *timer; 1177 short partid = XPC_PARTID(part); 1178 1179 /* allocate all the required GET/PUT values */ 1180 1181 part_sn2->local_GPs = 1182 xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL, 1183 &part_sn2->local_GPs_base); 1184 if (part_sn2->local_GPs == NULL) { 1185 dev_err(xpc_chan, "can't get memory for local get/put " 1186 "values\n"); 1187 return xpNoMemory; 1188 } 1189 1190 part_sn2->remote_GPs = 1191 xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL, 1192 &part_sn2->remote_GPs_base); 1193 if (part_sn2->remote_GPs == NULL) { 1194 dev_err(xpc_chan, "can't get memory for remote get/put " 1195 "values\n"); 1196 retval = xpNoMemory; 1197 goto out_1; 1198 } 1199 1200 part_sn2->remote_GPs_pa = 0; 1201 1202 /* allocate all the required open and close args */ 1203 1204 part_sn2->local_openclose_args = 1205 xpc_kzalloc_cacheline_aligned(XPC_OPENCLOSE_ARGS_SIZE, 1206 GFP_KERNEL, &part_sn2-> 1207 local_openclose_args_base); 1208 if (part_sn2->local_openclose_args == NULL) { 1209 dev_err(xpc_chan, "can't get memory for local connect args\n"); 1210 retval = xpNoMemory; 1211 goto out_2; 1212 } 1213 1214 part_sn2->remote_openclose_args_pa = 0; 1215 1216 part_sn2->local_chctl_amo_va = xpc_init_IRQ_amo_sn2(partid); 1217 1218 part_sn2->notify_IRQ_nasid = 0; 1219 part_sn2->notify_IRQ_phys_cpuid = 0; 1220 part_sn2->remote_chctl_amo_va = NULL; 1221 1222 sprintf(part_sn2->notify_IRQ_owner, "xpc%02d", partid); 1223 ret = request_irq(SGI_XPC_NOTIFY, xpc_handle_notify_IRQ_sn2, 1224 IRQF_SHARED, part_sn2->notify_IRQ_owner, 1225 (void *)(u64)partid); 1226 if (ret != 0) { 1227 dev_err(xpc_chan, "can't register NOTIFY IRQ handler, " 1228 "errno=%d\n", -ret); 1229 retval = xpLackOfResources; 1230 goto out_3; 1231 } 1232 1233 /* Setup a timer to check for dropped notify IRQs */ 1234 timer = &part_sn2->dropped_notify_IRQ_timer; 1235 init_timer(timer); 1236 timer->function = 1237 (void (*)(unsigned long))xpc_check_for_dropped_notify_IRQ_sn2; 1238 timer->data = (unsigned long)part; 1239 timer->expires = jiffies + XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL; 1240 add_timer(timer); 1241 1242 for (ch_number = 0; ch_number < part->nchannels; ch_number++) { 1243 ch_sn2 = &part->channels[ch_number].sn.sn2; 1244 1245 ch_sn2->local_GP = &part_sn2->local_GPs[ch_number]; 1246 ch_sn2->local_openclose_args = 1247 &part_sn2->local_openclose_args[ch_number]; 1248 1249 mutex_init(&ch_sn2->msg_to_pull_mutex); 1250 } 1251 1252 /* 1253 * Setup the per partition specific variables required by the 1254 * remote partition to establish channel connections with us. 1255 * 1256 * The setting of the magic # indicates that these per partition 1257 * specific variables are ready to be used. 1258 */ 1259 xpc_vars_part_sn2[partid].GPs_pa = xp_pa(part_sn2->local_GPs); 1260 xpc_vars_part_sn2[partid].openclose_args_pa = 1261 xp_pa(part_sn2->local_openclose_args); 1262 xpc_vars_part_sn2[partid].chctl_amo_pa = 1263 xp_pa(part_sn2->local_chctl_amo_va); 1264 cpuid = raw_smp_processor_id(); /* any CPU in this partition will do */ 1265 xpc_vars_part_sn2[partid].notify_IRQ_nasid = cpuid_to_nasid(cpuid); 1266 xpc_vars_part_sn2[partid].notify_IRQ_phys_cpuid = 1267 cpu_physical_id(cpuid); 1268 xpc_vars_part_sn2[partid].nchannels = part->nchannels; 1269 xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC1_SN2; 1270 1271 return xpSuccess; 1272 1273 /* setup of ch structures failed */ 1274out_3: 1275 kfree(part_sn2->local_openclose_args_base); 1276 part_sn2->local_openclose_args = NULL; 1277out_2: 1278 kfree(part_sn2->remote_GPs_base); 1279 part_sn2->remote_GPs = NULL; 1280out_1: 1281 kfree(part_sn2->local_GPs_base); 1282 part_sn2->local_GPs = NULL; 1283 return retval; 1284} 1285 1286/* 1287 * Teardown the channel structures that are sn2 specific. 1288 */ 1289static void 1290xpc_teardown_ch_structures_sn2(struct xpc_partition *part) 1291{ 1292 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; 1293 short partid = XPC_PARTID(part); 1294 1295 /* 1296 * Indicate that the variables specific to the remote partition are no 1297 * longer available for its use. 1298 */ 1299 xpc_vars_part_sn2[partid].magic = 0; 1300 1301 /* in case we've still got outstanding timers registered... */ 1302 del_timer_sync(&part_sn2->dropped_notify_IRQ_timer); 1303 free_irq(SGI_XPC_NOTIFY, (void *)(u64)partid); 1304 1305 kfree(part_sn2->local_openclose_args_base); 1306 part_sn2->local_openclose_args = NULL; 1307 kfree(part_sn2->remote_GPs_base); 1308 part_sn2->remote_GPs = NULL; 1309 kfree(part_sn2->local_GPs_base); 1310 part_sn2->local_GPs = NULL; 1311 part_sn2->local_chctl_amo_va = NULL; 1312} 1313 1314/* 1315 * Create a wrapper that hides the underlying mechanism for pulling a cacheline 1316 * (or multiple cachelines) from a remote partition. 1317 * 1318 * src_pa must be a cacheline aligned physical address on the remote partition. 1319 * dst must be a cacheline aligned virtual address on this partition. 1320 * cnt must be cacheline sized 1321 */ 1322/* ??? Replace this function by call to xp_remote_memcpy() or bte_copy()? */ 1323static enum xp_retval 1324xpc_pull_remote_cachelines_sn2(struct xpc_partition *part, void *dst, 1325 const unsigned long src_pa, size_t cnt) 1326{ 1327 enum xp_retval ret; 1328 1329 DBUG_ON(src_pa != L1_CACHE_ALIGN(src_pa)); 1330 DBUG_ON((unsigned long)dst != L1_CACHE_ALIGN((unsigned long)dst)); 1331 DBUG_ON(cnt != L1_CACHE_ALIGN(cnt)); 1332 1333 if (part->act_state == XPC_P_AS_DEACTIVATING) 1334 return part->reason; 1335 1336 ret = xp_remote_memcpy(xp_pa(dst), src_pa, cnt); 1337 if (ret != xpSuccess) { 1338 dev_dbg(xpc_chan, "xp_remote_memcpy() from partition %d failed," 1339 " ret=%d\n", XPC_PARTID(part), ret); 1340 } 1341 return ret; 1342} 1343 1344/* 1345 * Pull the remote per partition specific variables from the specified 1346 * partition. 1347 */ 1348static enum xp_retval 1349xpc_pull_remote_vars_part_sn2(struct xpc_partition *part) 1350{ 1351 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; 1352 u8 buffer[L1_CACHE_BYTES * 2]; 1353 struct xpc_vars_part_sn2 *pulled_entry_cacheline = 1354 (struct xpc_vars_part_sn2 *)L1_CACHE_ALIGN((u64)buffer); 1355 struct xpc_vars_part_sn2 *pulled_entry; 1356 unsigned long remote_entry_cacheline_pa; 1357 unsigned long remote_entry_pa; 1358 short partid = XPC_PARTID(part); 1359 enum xp_retval ret; 1360 1361 /* pull the cacheline that contains the variables we're interested in */ 1362 1363 DBUG_ON(part_sn2->remote_vars_part_pa != 1364 L1_CACHE_ALIGN(part_sn2->remote_vars_part_pa)); 1365 DBUG_ON(sizeof(struct xpc_vars_part_sn2) != L1_CACHE_BYTES / 2); 1366 1367 remote_entry_pa = part_sn2->remote_vars_part_pa + 1368 sn_partition_id * sizeof(struct xpc_vars_part_sn2); 1369 1370 remote_entry_cacheline_pa = (remote_entry_pa & ~(L1_CACHE_BYTES - 1)); 1371 1372 pulled_entry = (struct xpc_vars_part_sn2 *)((u64)pulled_entry_cacheline 1373 + (remote_entry_pa & 1374 (L1_CACHE_BYTES - 1))); 1375 1376 ret = xpc_pull_remote_cachelines_sn2(part, pulled_entry_cacheline, 1377 remote_entry_cacheline_pa, 1378 L1_CACHE_BYTES); 1379 if (ret != xpSuccess) { 1380 dev_dbg(xpc_chan, "failed to pull XPC vars_part from " 1381 "partition %d, ret=%d\n", partid, ret); 1382 return ret; 1383 } 1384 1385 /* see if they've been set up yet */ 1386 1387 if (pulled_entry->magic != XPC_VP_MAGIC1_SN2 && 1388 pulled_entry->magic != XPC_VP_MAGIC2_SN2) { 1389 1390 if (pulled_entry->magic != 0) { 1391 dev_dbg(xpc_chan, "partition %d's XPC vars_part for " 1392 "partition %d has bad magic value (=0x%llx)\n", 1393 partid, sn_partition_id, pulled_entry->magic); 1394 return xpBadMagic; 1395 } 1396 1397 /* they've not been initialized yet */ 1398 return xpRetry; 1399 } 1400 1401 if (xpc_vars_part_sn2[partid].magic == XPC_VP_MAGIC1_SN2) { 1402 1403 /* validate the variables */ 1404 1405 if (pulled_entry->GPs_pa == 0 || 1406 pulled_entry->openclose_args_pa == 0 || 1407 pulled_entry->chctl_amo_pa == 0) { 1408 1409 dev_err(xpc_chan, "partition %d's XPC vars_part for " 1410 "partition %d are not valid\n", partid, 1411 sn_partition_id); 1412 return xpInvalidAddress; 1413 } 1414 1415 /* the variables we imported look to be valid */ 1416 1417 part_sn2->remote_GPs_pa = pulled_entry->GPs_pa; 1418 part_sn2->remote_openclose_args_pa = 1419 pulled_entry->openclose_args_pa; 1420 part_sn2->remote_chctl_amo_va = 1421 (struct amo *)__va(pulled_entry->chctl_amo_pa); 1422 part_sn2->notify_IRQ_nasid = pulled_entry->notify_IRQ_nasid; 1423 part_sn2->notify_IRQ_phys_cpuid = 1424 pulled_entry->notify_IRQ_phys_cpuid; 1425 1426 if (part->nchannels > pulled_entry->nchannels) 1427 part->nchannels = pulled_entry->nchannels; 1428 1429 /* let the other side know that we've pulled their variables */ 1430 1431 xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC2_SN2; 1432 } 1433 1434 if (pulled_entry->magic == XPC_VP_MAGIC1_SN2) 1435 return xpRetry; 1436 1437 return xpSuccess; 1438} 1439 1440/* 1441 * Establish first contact with the remote partititon. This involves pulling 1442 * the XPC per partition variables from the remote partition and waiting for 1443 * the remote partition to pull ours. 1444 */ 1445static enum xp_retval 1446xpc_make_first_contact_sn2(struct xpc_partition *part) 1447{ 1448 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; 1449 enum xp_retval ret; 1450 1451 /* 1452 * Register the remote partition's amos with SAL so it can handle 1453 * and cleanup errors within that address range should the remote 1454 * partition go down. We don't unregister this range because it is 1455 * difficult to tell when outstanding writes to the remote partition 1456 * are finished and thus when it is safe to unregister. This should 1457 * not result in wasted space in the SAL xp_addr_region table because 1458 * we should get the same page for remote_amos_page_pa after module 1459 * reloads and system reboots. 1460 */ 1461 if (sn_register_xp_addr_region(part_sn2->remote_amos_page_pa, 1462 PAGE_SIZE, 1) < 0) { 1463 dev_warn(xpc_part, "xpc_activating(%d) failed to register " 1464 "xp_addr region\n", XPC_PARTID(part)); 1465 1466 ret = xpPhysAddrRegFailed; 1467 XPC_DEACTIVATE_PARTITION(part, ret); 1468 return ret; 1469 } 1470 1471 /* 1472 * Send activate IRQ to get other side to activate if they've not 1473 * already begun to do so. 1474 */ 1475 xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa, 1476 cnodeid_to_nasid(0), 1477 part_sn2->activate_IRQ_nasid, 1478 part_sn2->activate_IRQ_phys_cpuid); 1479 1480 while ((ret = xpc_pull_remote_vars_part_sn2(part)) != xpSuccess) { 1481 if (ret != xpRetry) { 1482 XPC_DEACTIVATE_PARTITION(part, ret); 1483 return ret; 1484 } 1485 1486 dev_dbg(xpc_part, "waiting to make first contact with " 1487 "partition %d\n", XPC_PARTID(part)); 1488 1489 /* wait a 1/4 of a second or so */ 1490 (void)msleep_interruptible(250); 1491 1492 if (part->act_state == XPC_P_AS_DEACTIVATING) 1493 return part->reason; 1494 } 1495 1496 return xpSuccess; 1497} 1498 1499/* 1500 * Get the chctl flags and pull the openclose args and/or remote GPs as needed. 1501 */ 1502static u64 1503xpc_get_chctl_all_flags_sn2(struct xpc_partition *part) 1504{ 1505 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; 1506 unsigned long irq_flags; 1507 union xpc_channel_ctl_flags chctl; 1508 enum xp_retval ret; 1509 1510 /* 1511 * See if there are any chctl flags to be handled. 1512 */ 1513 1514 spin_lock_irqsave(&part->chctl_lock, irq_flags); 1515 chctl = part->chctl; 1516 if (chctl.all_flags != 0) 1517 part->chctl.all_flags = 0; 1518 1519 spin_unlock_irqrestore(&part->chctl_lock, irq_flags); 1520 1521 if (xpc_any_openclose_chctl_flags_set(&chctl)) { 1522 ret = xpc_pull_remote_cachelines_sn2(part, part-> 1523 remote_openclose_args, 1524 part_sn2-> 1525 remote_openclose_args_pa, 1526 XPC_OPENCLOSE_ARGS_SIZE); 1527 if (ret != xpSuccess) { 1528 XPC_DEACTIVATE_PARTITION(part, ret); 1529 1530 dev_dbg(xpc_chan, "failed to pull openclose args from " 1531 "partition %d, ret=%d\n", XPC_PARTID(part), 1532 ret); 1533 1534 /* don't bother processing chctl flags anymore */ 1535 chctl.all_flags = 0; 1536 } 1537 } 1538 1539 if (xpc_any_msg_chctl_flags_set(&chctl)) { 1540 ret = xpc_pull_remote_cachelines_sn2(part, part_sn2->remote_GPs, 1541 part_sn2->remote_GPs_pa, 1542 XPC_GP_SIZE); 1543 if (ret != xpSuccess) { 1544 XPC_DEACTIVATE_PARTITION(part, ret); 1545 1546 dev_dbg(xpc_chan, "failed to pull GPs from partition " 1547 "%d, ret=%d\n", XPC_PARTID(part), ret); 1548 1549 /* don't bother processing chctl flags anymore */ 1550 chctl.all_flags = 0; 1551 } 1552 } 1553 1554 return chctl.all_flags; 1555} 1556 1557/* 1558 * Allocate the local message queue and the notify queue. 1559 */ 1560static enum xp_retval 1561xpc_allocate_local_msgqueue_sn2(struct xpc_channel *ch) 1562{ 1563 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; 1564 unsigned long irq_flags; 1565 int nentries; 1566 size_t nbytes; 1567 1568 for (nentries = ch->local_nentries; nentries > 0; nentries--) { 1569 1570 nbytes = nentries * ch->entry_size; 1571 ch_sn2->local_msgqueue = 1572 xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL, 1573 &ch_sn2->local_msgqueue_base); 1574 if (ch_sn2->local_msgqueue == NULL) 1575 continue; 1576 1577 nbytes = nentries * sizeof(struct xpc_notify_sn2); 1578 ch_sn2->notify_queue = kzalloc(nbytes, GFP_KERNEL); 1579 if (ch_sn2->notify_queue == NULL) { 1580 kfree(ch_sn2->local_msgqueue_base); 1581 ch_sn2->local_msgqueue = NULL; 1582 continue; 1583 } 1584 1585 spin_lock_irqsave(&ch->lock, irq_flags); 1586 if (nentries < ch->local_nentries) { 1587 dev_dbg(xpc_chan, "nentries=%d local_nentries=%d, " 1588 "partid=%d, channel=%d\n", nentries, 1589 ch->local_nentries, ch->partid, ch->number); 1590 1591 ch->local_nentries = nentries; 1592 } 1593 spin_unlock_irqrestore(&ch->lock, irq_flags); 1594 return xpSuccess; 1595 } 1596 1597 dev_dbg(xpc_chan, "can't get memory for local message queue and notify " 1598 "queue, partid=%d, channel=%d\n", ch->partid, ch->number); 1599 return xpNoMemory; 1600} 1601 1602/* 1603 * Allocate the cached remote message queue. 1604 */ 1605static enum xp_retval 1606xpc_allocate_remote_msgqueue_sn2(struct xpc_channel *ch) 1607{ 1608 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; 1609 unsigned long irq_flags; 1610 int nentries; 1611 size_t nbytes; 1612 1613 DBUG_ON(ch->remote_nentries <= 0); 1614 1615 for (nentries = ch->remote_nentries; nentries > 0; nentries--) { 1616 1617 nbytes = nentries * ch->entry_size; 1618 ch_sn2->remote_msgqueue = 1619 xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL, &ch_sn2-> 1620 remote_msgqueue_base); 1621 if (ch_sn2->remote_msgqueue == NULL) 1622 continue; 1623 1624 spin_lock_irqsave(&ch->lock, irq_flags); 1625 if (nentries < ch->remote_nentries) { 1626 dev_dbg(xpc_chan, "nentries=%d remote_nentries=%d, " 1627 "partid=%d, channel=%d\n", nentries, 1628 ch->remote_nentries, ch->partid, ch->number); 1629 1630 ch->remote_nentries = nentries; 1631 } 1632 spin_unlock_irqrestore(&ch->lock, irq_flags); 1633 return xpSuccess; 1634 } 1635 1636 dev_dbg(xpc_chan, "can't get memory for cached remote message queue, " 1637 "partid=%d, channel=%d\n", ch->partid, ch->number); 1638 return xpNoMemory; 1639} 1640 1641/* 1642 * Allocate message queues and other stuff associated with a channel. 1643 * 1644 * Note: Assumes all of the channel sizes are filled in. 1645 */ 1646static enum xp_retval 1647xpc_setup_msg_structures_sn2(struct xpc_channel *ch) 1648{ 1649 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; 1650 enum xp_retval ret; 1651 1652 DBUG_ON(ch->flags & XPC_C_SETUP); 1653 1654 ret = xpc_allocate_local_msgqueue_sn2(ch); 1655 if (ret == xpSuccess) { 1656 1657 ret = xpc_allocate_remote_msgqueue_sn2(ch); 1658 if (ret != xpSuccess) { 1659 kfree(ch_sn2->local_msgqueue_base); 1660 ch_sn2->local_msgqueue = NULL; 1661 kfree(ch_sn2->notify_queue); 1662 ch_sn2->notify_queue = NULL; 1663 } 1664 } 1665 return ret; 1666} 1667 1668/* 1669 * Free up message queues and other stuff that were allocated for the specified 1670 * channel. 1671 */ 1672static void 1673xpc_teardown_msg_structures_sn2(struct xpc_channel *ch) 1674{ 1675 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; 1676 1677 DBUG_ON(!spin_is_locked(&ch->lock)); 1678 1679 ch_sn2->remote_msgqueue_pa = 0; 1680 1681 ch_sn2->local_GP->get = 0; 1682 ch_sn2->local_GP->put = 0; 1683 ch_sn2->remote_GP.get = 0; 1684 ch_sn2->remote_GP.put = 0; 1685 ch_sn2->w_local_GP.get = 0; 1686 ch_sn2->w_local_GP.put = 0; 1687 ch_sn2->w_remote_GP.get = 0; 1688 ch_sn2->w_remote_GP.put = 0; 1689 ch_sn2->next_msg_to_pull = 0; 1690 1691 if (ch->flags & XPC_C_SETUP) { 1692 dev_dbg(xpc_chan, "ch->flags=0x%x, partid=%d, channel=%d\n", 1693 ch->flags, ch->partid, ch->number); 1694 1695 kfree(ch_sn2->local_msgqueue_base); 1696 ch_sn2->local_msgqueue = NULL; 1697 kfree(ch_sn2->remote_msgqueue_base); 1698 ch_sn2->remote_msgqueue = NULL; 1699 kfree(ch_sn2->notify_queue); 1700 ch_sn2->notify_queue = NULL; 1701 } 1702} 1703 1704/* 1705 * Notify those who wanted to be notified upon delivery of their message. 1706 */ 1707static void 1708xpc_notify_senders_sn2(struct xpc_channel *ch, enum xp_retval reason, s64 put) 1709{ 1710 struct xpc_notify_sn2 *notify; 1711 u8 notify_type; 1712 s64 get = ch->sn.sn2.w_remote_GP.get - 1; 1713 1714 while (++get < put && atomic_read(&ch->n_to_notify) > 0) { 1715 1716 notify = &ch->sn.sn2.notify_queue[get % ch->local_nentries]; 1717 1718 /* 1719 * See if the notify entry indicates it was associated with 1720 * a message who's sender wants to be notified. It is possible 1721 * that it is, but someone else is doing or has done the 1722 * notification. 1723 */ 1724 notify_type = notify->type; 1725 if (notify_type == 0 || 1726 cmpxchg(¬ify->type, notify_type, 0) != notify_type) { 1727 continue; 1728 } 1729 1730 DBUG_ON(notify_type != XPC_N_CALL); 1731 1732 atomic_dec(&ch->n_to_notify); 1733 1734 if (notify->func != NULL) { 1735 dev_dbg(xpc_chan, "notify->func() called, notify=0x%p " 1736 "msg_number=%lld partid=%d channel=%d\n", 1737 (void *)notify, get, ch->partid, ch->number); 1738 1739 notify->func(reason, ch->partid, ch->number, 1740 notify->key); 1741 1742 dev_dbg(xpc_chan, "notify->func() returned, notify=0x%p" 1743 " msg_number=%lld partid=%d channel=%d\n", 1744 (void *)notify, get, ch->partid, ch->number); 1745 } 1746 } 1747} 1748 1749static void 1750xpc_notify_senders_of_disconnect_sn2(struct xpc_channel *ch) 1751{ 1752 xpc_notify_senders_sn2(ch, ch->reason, ch->sn.sn2.w_local_GP.put); 1753} 1754 1755/* 1756 * Clear some of the msg flags in the local message queue. 1757 */ 1758static inline void 1759xpc_clear_local_msgqueue_flags_sn2(struct xpc_channel *ch) 1760{ 1761 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; 1762 struct xpc_msg_sn2 *msg; 1763 s64 get; 1764 1765 get = ch_sn2->w_remote_GP.get; 1766 do { 1767 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue + 1768 (get % ch->local_nentries) * 1769 ch->entry_size); 1770 DBUG_ON(!(msg->flags & XPC_M_SN2_READY)); 1771 msg->flags = 0; 1772 } while (++get < ch_sn2->remote_GP.get); 1773} 1774 1775/* 1776 * Clear some of the msg flags in the remote message queue. 1777 */ 1778static inline void 1779xpc_clear_remote_msgqueue_flags_sn2(struct xpc_channel *ch) 1780{ 1781 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; 1782 struct xpc_msg_sn2 *msg; 1783 s64 put, remote_nentries = ch->remote_nentries; 1784 1785 /* flags are zeroed when the buffer is allocated */ 1786 if (ch_sn2->remote_GP.put < remote_nentries) 1787 return; 1788 1789 put = max(ch_sn2->w_remote_GP.put, remote_nentries); 1790 do { 1791 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + 1792 (put % remote_nentries) * 1793 ch->entry_size); 1794 DBUG_ON(!(msg->flags & XPC_M_SN2_READY)); 1795 DBUG_ON(!(msg->flags & XPC_M_SN2_DONE)); 1796 DBUG_ON(msg->number != put - remote_nentries); 1797 msg->flags = 0; 1798 } while (++put < ch_sn2->remote_GP.put); 1799} 1800 1801static int 1802xpc_n_of_deliverable_payloads_sn2(struct xpc_channel *ch) 1803{ 1804 return ch->sn.sn2.w_remote_GP.put - ch->sn.sn2.w_local_GP.get; 1805} 1806 1807static void 1808xpc_process_msg_chctl_flags_sn2(struct xpc_partition *part, int ch_number) 1809{ 1810 struct xpc_channel *ch = &part->channels[ch_number]; 1811 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; 1812 int npayloads_sent; 1813 1814 ch_sn2->remote_GP = part->sn.sn2.remote_GPs[ch_number]; 1815 1816 /* See what, if anything, has changed for each connected channel */ 1817 1818 xpc_msgqueue_ref(ch); 1819 1820 if (ch_sn2->w_remote_GP.get == ch_sn2->remote_GP.get && 1821 ch_sn2->w_remote_GP.put == ch_sn2->remote_GP.put) { 1822 /* nothing changed since GPs were last pulled */ 1823 xpc_msgqueue_deref(ch); 1824 return; 1825 } 1826 1827 if (!(ch->flags & XPC_C_CONNECTED)) { 1828 xpc_msgqueue_deref(ch); 1829 return; 1830 } 1831 1832 /* 1833 * First check to see if messages recently sent by us have been 1834 * received by the other side. (The remote GET value will have 1835 * changed since we last looked at it.) 1836 */ 1837 1838 if (ch_sn2->w_remote_GP.get != ch_sn2->remote_GP.get) { 1839 1840 /* 1841 * We need to notify any senders that want to be notified 1842 * that their sent messages have been received by their 1843 * intended recipients. We need to do this before updating 1844 * w_remote_GP.get so that we don't allocate the same message 1845 * queue entries prematurely (see xpc_allocate_msg()). 1846 */ 1847 if (atomic_read(&ch->n_to_notify) > 0) { 1848 /* 1849 * Notify senders that messages sent have been 1850 * received and delivered by the other side. 1851 */ 1852 xpc_notify_senders_sn2(ch, xpMsgDelivered, 1853 ch_sn2->remote_GP.get); 1854 } 1855 1856 /* 1857 * Clear msg->flags in previously sent messages, so that 1858 * they're ready for xpc_allocate_msg(). 1859 */ 1860 xpc_clear_local_msgqueue_flags_sn2(ch); 1861 1862 ch_sn2->w_remote_GP.get = ch_sn2->remote_GP.get; 1863 1864 dev_dbg(xpc_chan, "w_remote_GP.get changed to %lld, partid=%d, " 1865 "channel=%d\n", ch_sn2->w_remote_GP.get, ch->partid, 1866 ch->number); 1867 1868 /* 1869 * If anyone was waiting for message queue entries to become 1870 * available, wake them up. 1871 */ 1872 if (atomic_read(&ch->n_on_msg_allocate_wq) > 0) 1873 wake_up(&ch->msg_allocate_wq); 1874 } 1875 1876 /* 1877 * Now check for newly sent messages by the other side. (The remote 1878 * PUT value will have changed since we last looked at it.) 1879 */ 1880 1881 if (ch_sn2->w_remote_GP.put != ch_sn2->remote_GP.put) { 1882 /* 1883 * Clear msg->flags in previously received messages, so that 1884 * they're ready for xpc_get_deliverable_payload_sn2(). 1885 */ 1886 xpc_clear_remote_msgqueue_flags_sn2(ch); 1887 1888 smp_wmb(); /* ensure flags have been cleared before bte_copy */ 1889 ch_sn2->w_remote_GP.put = ch_sn2->remote_GP.put; 1890 1891 dev_dbg(xpc_chan, "w_remote_GP.put changed to %lld, partid=%d, " 1892 "channel=%d\n", ch_sn2->w_remote_GP.put, ch->partid, 1893 ch->number); 1894 1895 npayloads_sent = xpc_n_of_deliverable_payloads_sn2(ch); 1896 if (npayloads_sent > 0) { 1897 dev_dbg(xpc_chan, "msgs waiting to be copied and " 1898 "delivered=%d, partid=%d, channel=%d\n", 1899 npayloads_sent, ch->partid, ch->number); 1900 1901 if (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE) 1902 xpc_activate_kthreads(ch, npayloads_sent); 1903 } 1904 } 1905 1906 xpc_msgqueue_deref(ch); 1907} 1908 1909static struct xpc_msg_sn2 * 1910xpc_pull_remote_msg_sn2(struct xpc_channel *ch, s64 get) 1911{ 1912 struct xpc_partition *part = &xpc_partitions[ch->partid]; 1913 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; 1914 unsigned long remote_msg_pa; 1915 struct xpc_msg_sn2 *msg; 1916 u32 msg_index; 1917 u32 nmsgs; 1918 u64 msg_offset; 1919 enum xp_retval ret; 1920 1921 if (mutex_lock_interruptible(&ch_sn2->msg_to_pull_mutex) != 0) { 1922 /* we were interrupted by a signal */ 1923 return NULL; 1924 } 1925 1926 while (get >= ch_sn2->next_msg_to_pull) { 1927 1928 /* pull as many messages as are ready and able to be pulled */ 1929 1930 msg_index = ch_sn2->next_msg_to_pull % ch->remote_nentries; 1931 1932 DBUG_ON(ch_sn2->next_msg_to_pull >= ch_sn2->w_remote_GP.put); 1933 nmsgs = ch_sn2->w_remote_GP.put - ch_sn2->next_msg_to_pull; 1934 if (msg_index + nmsgs > ch->remote_nentries) { 1935 /* ignore the ones that wrap the msg queue for now */ 1936 nmsgs = ch->remote_nentries - msg_index; 1937 } 1938 1939 msg_offset = msg_index * ch->entry_size; 1940 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + 1941 msg_offset); 1942 remote_msg_pa = ch_sn2->remote_msgqueue_pa + msg_offset; 1943 1944 ret = xpc_pull_remote_cachelines_sn2(part, msg, remote_msg_pa, 1945 nmsgs * ch->entry_size); 1946 if (ret != xpSuccess) { 1947 1948 dev_dbg(xpc_chan, "failed to pull %d msgs starting with" 1949 " msg %lld from partition %d, channel=%d, " 1950 "ret=%d\n", nmsgs, ch_sn2->next_msg_to_pull, 1951 ch->partid, ch->number, ret); 1952 1953 XPC_DEACTIVATE_PARTITION(part, ret); 1954 1955 mutex_unlock(&ch_sn2->msg_to_pull_mutex); 1956 return NULL; 1957 } 1958 1959 ch_sn2->next_msg_to_pull += nmsgs; 1960 } 1961 1962 mutex_unlock(&ch_sn2->msg_to_pull_mutex); 1963 1964 /* return the message we were looking for */ 1965 msg_offset = (get % ch->remote_nentries) * ch->entry_size; 1966 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + msg_offset); 1967 1968 return msg; 1969} 1970 1971/* 1972 * Get the next deliverable message's payload. 1973 */ 1974static void * 1975xpc_get_deliverable_payload_sn2(struct xpc_channel *ch) 1976{ 1977 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; 1978 struct xpc_msg_sn2 *msg; 1979 void *payload = NULL; 1980 s64 get; 1981 1982 do { 1983 if (ch->flags & XPC_C_DISCONNECTING) 1984 break; 1985 1986 get = ch_sn2->w_local_GP.get; 1987 smp_rmb(); /* guarantee that .get loads before .put */ 1988 if (get == ch_sn2->w_remote_GP.put) 1989 break; 1990 1991 /* There are messages waiting to be pulled and delivered. 1992 * We need to try to secure one for ourselves. We'll do this 1993 * by trying to increment w_local_GP.get and hope that no one 1994 * else beats us to it. If they do, we'll we'll simply have 1995 * to try again for the next one. 1996 */ 1997 1998 if (cmpxchg(&ch_sn2->w_local_GP.get, get, get + 1) == get) { 1999 /* we got the entry referenced by get */ 2000 2001 dev_dbg(xpc_chan, "w_local_GP.get changed to %lld, " 2002 "partid=%d, channel=%d\n", get + 1, 2003 ch->partid, ch->number); 2004 2005 /* pull the message from the remote partition */ 2006 2007 msg = xpc_pull_remote_msg_sn2(ch, get); 2008 2009 if (msg != NULL) { 2010 DBUG_ON(msg->number != get); 2011 DBUG_ON(msg->flags & XPC_M_SN2_DONE); 2012 DBUG_ON(!(msg->flags & XPC_M_SN2_READY)); 2013 2014 payload = &msg->payload; 2015 } 2016 break; 2017 } 2018 2019 } while (1); 2020 2021 return payload; 2022} 2023 2024/* 2025 * Now we actually send the messages that are ready to be sent by advancing 2026 * the local message queue's Put value and then send a chctl msgrequest to the 2027 * recipient partition. 2028 */ 2029static void 2030xpc_send_msgs_sn2(struct xpc_channel *ch, s64 initial_put) 2031{ 2032 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; 2033 struct xpc_msg_sn2 *msg; 2034 s64 put = initial_put + 1; 2035 int send_msgrequest = 0; 2036 2037 while (1) { 2038 2039 while (1) { 2040 if (put == ch_sn2->w_local_GP.put) 2041 break; 2042 2043 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2-> 2044 local_msgqueue + (put % 2045 ch->local_nentries) * 2046 ch->entry_size); 2047 2048 if (!(msg->flags & XPC_M_SN2_READY)) 2049 break; 2050 2051 put++; 2052 } 2053 2054 if (put == initial_put) { 2055 /* nothing's changed */ 2056 break; 2057 } 2058 2059 if (cmpxchg_rel(&ch_sn2->local_GP->put, initial_put, put) != 2060 initial_put) { 2061 /* someone else beat us to it */ 2062 DBUG_ON(ch_sn2->local_GP->put < initial_put); 2063 break; 2064 } 2065 2066 /* we just set the new value of local_GP->put */ 2067 2068 dev_dbg(xpc_chan, "local_GP->put changed to %lld, partid=%d, " 2069 "channel=%d\n", put, ch->partid, ch->number); 2070 2071 send_msgrequest = 1; 2072 2073 /* 2074 * We need to ensure that the message referenced by 2075 * local_GP->put is not XPC_M_SN2_READY or that local_GP->put 2076 * equals w_local_GP.put, so we'll go have a look. 2077 */ 2078 initial_put = put; 2079 } 2080 2081 if (send_msgrequest) 2082 xpc_send_chctl_msgrequest_sn2(ch); 2083} 2084 2085/* 2086 * Allocate an entry for a message from the message queue associated with the 2087 * specified channel. 2088 */ 2089static enum xp_retval 2090xpc_allocate_msg_sn2(struct xpc_channel *ch, u32 flags, 2091 struct xpc_msg_sn2 **address_of_msg) 2092{ 2093 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; 2094 struct xpc_msg_sn2 *msg; 2095 enum xp_retval ret; 2096 s64 put; 2097 2098 /* 2099 * Get the next available message entry from the local message queue. 2100 * If none are available, we'll make sure that we grab the latest 2101 * GP values. 2102 */ 2103 ret = xpTimeout; 2104 2105 while (1) { 2106 2107 put = ch_sn2->w_local_GP.put; 2108 smp_rmb(); /* guarantee that .put loads before .get */ 2109 if (put - ch_sn2->w_remote_GP.get < ch->local_nentries) { 2110 2111 /* There are available message entries. We need to try 2112 * to secure one for ourselves. We'll do this by trying 2113 * to increment w_local_GP.put as long as someone else 2114 * doesn't beat us to it. If they do, we'll have to 2115 * try again. 2116 */ 2117 if (cmpxchg(&ch_sn2->w_local_GP.put, put, put + 1) == 2118 put) { 2119 /* we got the entry referenced by put */ 2120 break; 2121 } 2122 continue; /* try again */ 2123 } 2124 2125 /* 2126 * There aren't any available msg entries at this time. 2127 * 2128 * In waiting for a message entry to become available, 2129 * we set a timeout in case the other side is not sending 2130 * completion interrupts. This lets us fake a notify IRQ 2131 * that will cause the notify IRQ handler to fetch the latest 2132 * GP values as if an interrupt was sent by the other side. 2133 */ 2134 if (ret == xpTimeout) 2135 xpc_send_chctl_local_msgrequest_sn2(ch); 2136 2137 if (flags & XPC_NOWAIT) 2138 return xpNoWait; 2139 2140 ret = xpc_allocate_msg_wait(ch); 2141 if (ret != xpInterrupted && ret != xpTimeout) 2142 return ret; 2143 } 2144 2145 /* get the message's address and initialize it */ 2146 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue + 2147 (put % ch->local_nentries) * 2148 ch->entry_size); 2149 2150 DBUG_ON(msg->flags != 0); 2151 msg->number = put; 2152 2153 dev_dbg(xpc_chan, "w_local_GP.put changed to %lld; msg=0x%p, " 2154 "msg_number=%lld, partid=%d, channel=%d\n", put + 1, 2155 (void *)msg, msg->number, ch->partid, ch->number); 2156 2157 *address_of_msg = msg; 2158 return xpSuccess; 2159} 2160 2161/* 2162 * Common code that does the actual sending of the message by advancing the 2163 * local message queue's Put value and sends a chctl msgrequest to the 2164 * partition the message is being sent to. 2165 */ 2166static enum xp_retval 2167xpc_send_payload_sn2(struct xpc_channel *ch, u32 flags, void *payload, 2168 u16 payload_size, u8 notify_type, xpc_notify_func func, 2169 void *key) 2170{ 2171 enum xp_retval ret = xpSuccess; 2172 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; 2173 struct xpc_msg_sn2 *msg = msg; 2174 struct xpc_notify_sn2 *notify = notify; 2175 s64 msg_number; 2176 s64 put; 2177 2178 DBUG_ON(notify_type == XPC_N_CALL && func == NULL); 2179 2180 if (XPC_MSG_SIZE(payload_size) > ch->entry_size) 2181 return xpPayloadTooBig; 2182 2183 xpc_msgqueue_ref(ch); 2184 2185 if (ch->flags & XPC_C_DISCONNECTING) { 2186 ret = ch->reason; 2187 goto out_1; 2188 } 2189 if (!(ch->flags & XPC_C_CONNECTED)) { 2190 ret = xpNotConnected; 2191 goto out_1; 2192 } 2193 2194 ret = xpc_allocate_msg_sn2(ch, flags, &msg); 2195 if (ret != xpSuccess) 2196 goto out_1; 2197 2198 msg_number = msg->number; 2199 2200 if (notify_type != 0) { 2201 /* 2202 * Tell the remote side to send an ACK interrupt when the 2203 * message has been delivered. 2204 */ 2205 msg->flags |= XPC_M_SN2_INTERRUPT; 2206 2207 atomic_inc(&ch->n_to_notify); 2208 2209 notify = &ch_sn2->notify_queue[msg_number % ch->local_nentries]; 2210 notify->func = func; 2211 notify->key = key; 2212 notify->type = notify_type; 2213 2214 /* ??? Is a mb() needed here? */ 2215 2216 if (ch->flags & XPC_C_DISCONNECTING) { 2217 /* 2218 * An error occurred between our last error check and 2219 * this one. We will try to clear the type field from 2220 * the notify entry. If we succeed then 2221 * xpc_disconnect_channel() didn't already process 2222 * the notify entry. 2223 */ 2224 if (cmpxchg(¬ify->type, notify_type, 0) == 2225 notify_type) { 2226 atomic_dec(&ch->n_to_notify); 2227 ret = ch->reason; 2228 } 2229 goto out_1; 2230 } 2231 } 2232 2233 memcpy(&msg->payload, payload, payload_size); 2234 2235 msg->flags |= XPC_M_SN2_READY; 2236 2237 /* 2238 * The preceding store of msg->flags must occur before the following 2239 * load of local_GP->put. 2240 */ 2241 smp_mb(); 2242 2243 /* see if the message is next in line to be sent, if so send it */ 2244 2245 put = ch_sn2->local_GP->put; 2246 if (put == msg_number) 2247 xpc_send_msgs_sn2(ch, put); 2248 2249out_1: 2250 xpc_msgqueue_deref(ch); 2251 return ret; 2252} 2253 2254/* 2255 * Now we actually acknowledge the messages that have been delivered and ack'd 2256 * by advancing the cached remote message queue's Get value and if requested 2257 * send a chctl msgrequest to the message sender's partition. 2258 * 2259 * If a message has XPC_M_SN2_INTERRUPT set, send an interrupt to the partition 2260 * that sent the message. 2261 */ 2262static void 2263xpc_acknowledge_msgs_sn2(struct xpc_channel *ch, s64 initial_get, u8 msg_flags) 2264{ 2265 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; 2266 struct xpc_msg_sn2 *msg; 2267 s64 get = initial_get + 1; 2268 int send_msgrequest = 0; 2269 2270 while (1) { 2271 2272 while (1) { 2273 if (get == ch_sn2->w_local_GP.get) 2274 break; 2275 2276 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2-> 2277 remote_msgqueue + (get % 2278 ch->remote_nentries) * 2279 ch->entry_size); 2280 2281 if (!(msg->flags & XPC_M_SN2_DONE)) 2282 break; 2283 2284 msg_flags |= msg->flags; 2285 get++; 2286 } 2287 2288 if (get == initial_get) { 2289 /* nothing's changed */ 2290 break; 2291 } 2292 2293 if (cmpxchg_rel(&ch_sn2->local_GP->get, initial_get, get) != 2294 initial_get) { 2295 /* someone else beat us to it */ 2296 DBUG_ON(ch_sn2->local_GP->get <= initial_get); 2297 break; 2298 } 2299 2300 /* we just set the new value of local_GP->get */ 2301 2302 dev_dbg(xpc_chan, "local_GP->get changed to %lld, partid=%d, " 2303 "channel=%d\n", get, ch->partid, ch->number); 2304 2305 send_msgrequest = (msg_flags & XPC_M_SN2_INTERRUPT); 2306 2307 /* 2308 * We need to ensure that the message referenced by 2309 * local_GP->get is not XPC_M_SN2_DONE or that local_GP->get 2310 * equals w_local_GP.get, so we'll go have a look. 2311 */ 2312 initial_get = get; 2313 } 2314 2315 if (send_msgrequest) 2316 xpc_send_chctl_msgrequest_sn2(ch); 2317} 2318 2319static void 2320xpc_received_payload_sn2(struct xpc_channel *ch, void *payload) 2321{ 2322 struct xpc_msg_sn2 *msg; 2323 s64 msg_number; 2324 s64 get; 2325 2326 msg = container_of(payload, struct xpc_msg_sn2, payload); 2327 msg_number = msg->number; 2328 2329 dev_dbg(xpc_chan, "msg=0x%p, msg_number=%lld, partid=%d, channel=%d\n", 2330 (void *)msg, msg_number, ch->partid, ch->number); 2331 2332 DBUG_ON((((u64)msg - (u64)ch->sn.sn2.remote_msgqueue) / ch->entry_size) != 2333 msg_number % ch->remote_nentries); 2334 DBUG_ON(!(msg->flags & XPC_M_SN2_READY)); 2335 DBUG_ON(msg->flags & XPC_M_SN2_DONE); 2336 2337 msg->flags |= XPC_M_SN2_DONE; 2338 2339 /* 2340 * The preceding store of msg->flags must occur before the following 2341 * load of local_GP->get. 2342 */ 2343 smp_mb(); 2344 2345 /* 2346 * See if this message is next in line to be acknowledged as having 2347 * been delivered. 2348 */ 2349 get = ch->sn.sn2.local_GP->get; 2350 if (get == msg_number) 2351 xpc_acknowledge_msgs_sn2(ch, get, msg->flags); 2352} 2353 2354static struct xpc_arch_operations xpc_arch_ops_sn2 = { 2355 .setup_partitions = xpc_setup_partitions_sn2, 2356 .teardown_partitions = xpc_teardown_partitions_sn2, 2357 .process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_sn2, 2358 .get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_sn2, 2359 .setup_rsvd_page = xpc_setup_rsvd_page_sn2, 2360 2361 .allow_hb = xpc_allow_hb_sn2, 2362 .disallow_hb = xpc_disallow_hb_sn2, 2363 .disallow_all_hbs = xpc_disallow_all_hbs_sn2, 2364 .increment_heartbeat = xpc_increment_heartbeat_sn2, 2365 .offline_heartbeat = xpc_offline_heartbeat_sn2, 2366 .online_heartbeat = xpc_online_heartbeat_sn2, 2367 .heartbeat_init = xpc_heartbeat_init_sn2, 2368 .heartbeat_exit = xpc_heartbeat_exit_sn2, 2369 .get_remote_heartbeat = xpc_get_remote_heartbeat_sn2, 2370 2371 .request_partition_activation = 2372 xpc_request_partition_activation_sn2, 2373 .request_partition_reactivation = 2374 xpc_request_partition_reactivation_sn2, 2375 .request_partition_deactivation = 2376 xpc_request_partition_deactivation_sn2, 2377 .cancel_partition_deactivation_request = 2378 xpc_cancel_partition_deactivation_request_sn2, 2379 2380 .setup_ch_structures = xpc_setup_ch_structures_sn2, 2381 .teardown_ch_structures = xpc_teardown_ch_structures_sn2, 2382 2383 .make_first_contact = xpc_make_first_contact_sn2, 2384 2385 .get_chctl_all_flags = xpc_get_chctl_all_flags_sn2, 2386 .send_chctl_closerequest = xpc_send_chctl_closerequest_sn2, 2387 .send_chctl_closereply = xpc_send_chctl_closereply_sn2, 2388 .send_chctl_openrequest = xpc_send_chctl_openrequest_sn2, 2389 .send_chctl_openreply = xpc_send_chctl_openreply_sn2, 2390 .send_chctl_opencomplete = xpc_send_chctl_opencomplete_sn2, 2391 .process_msg_chctl_flags = xpc_process_msg_chctl_flags_sn2, 2392 2393 .save_remote_msgqueue_pa = xpc_save_remote_msgqueue_pa_sn2, 2394 2395 .setup_msg_structures = xpc_setup_msg_structures_sn2, 2396 .teardown_msg_structures = xpc_teardown_msg_structures_sn2, 2397 2398 .indicate_partition_engaged = xpc_indicate_partition_engaged_sn2, 2399 .indicate_partition_disengaged = xpc_indicate_partition_disengaged_sn2, 2400 .partition_engaged = xpc_partition_engaged_sn2, 2401 .any_partition_engaged = xpc_any_partition_engaged_sn2, 2402 .assume_partition_disengaged = xpc_assume_partition_disengaged_sn2, 2403 2404 .n_of_deliverable_payloads = xpc_n_of_deliverable_payloads_sn2, 2405 .send_payload = xpc_send_payload_sn2, 2406 .get_deliverable_payload = xpc_get_deliverable_payload_sn2, 2407 .received_payload = xpc_received_payload_sn2, 2408 .notify_senders_of_disconnect = xpc_notify_senders_of_disconnect_sn2, 2409}; 2410 2411int 2412xpc_init_sn2(void) 2413{ 2414 int ret; 2415 size_t buf_size; 2416 2417 xpc_arch_ops = xpc_arch_ops_sn2; 2418 2419 if (offsetof(struct xpc_msg_sn2, payload) > XPC_MSG_HDR_MAX_SIZE) { 2420 dev_err(xpc_part, "header portion of struct xpc_msg_sn2 is " 2421 "larger than %d\n", XPC_MSG_HDR_MAX_SIZE); 2422 return -E2BIG; 2423 } 2424 2425 buf_size = max(XPC_RP_VARS_SIZE, 2426 XPC_RP_HEADER_SIZE + XP_NASID_MASK_BYTES_SN2); 2427 xpc_remote_copy_buffer_sn2 = xpc_kmalloc_cacheline_aligned(buf_size, 2428 GFP_KERNEL, 2429 &xpc_remote_copy_buffer_base_sn2); 2430 if (xpc_remote_copy_buffer_sn2 == NULL) { 2431 dev_err(xpc_part, "can't get memory for remote copy buffer\n"); 2432 return -ENOMEM; 2433 } 2434 2435 /* open up protections for IPI and [potentially] amo operations */ 2436 xpc_allow_IPI_ops_sn2(); 2437 xpc_allow_amo_ops_shub_wars_1_1_sn2(); 2438 2439 /* 2440 * This is safe to do before the xpc_hb_checker thread has started 2441 * because the handler releases a wait queue. If an interrupt is 2442 * received before the thread is waiting, it will not go to sleep, 2443 * but rather immediately process the interrupt. 2444 */ 2445 ret = request_irq(SGI_XPC_ACTIVATE, xpc_handle_activate_IRQ_sn2, 0, 2446 "xpc hb", NULL); 2447 if (ret != 0) { 2448 dev_err(xpc_part, "can't register ACTIVATE IRQ handler, " 2449 "errno=%d\n", -ret); 2450 xpc_disallow_IPI_ops_sn2(); 2451 kfree(xpc_remote_copy_buffer_base_sn2); 2452 } 2453 return ret; 2454} 2455 2456void 2457xpc_exit_sn2(void) 2458{ 2459 free_irq(SGI_XPC_ACTIVATE, NULL); 2460 xpc_disallow_IPI_ops_sn2(); 2461 kfree(xpc_remote_copy_buffer_base_sn2); 2462} 2463