194bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson/* 294bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson * This file is subject to the terms and conditions of the GNU General Public 394bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson * License. See the file "COPYING" in the main directory of this archive 494bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson * for more details. 594bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson * 6361916a943cd9dbda1c0b00879d0225cc919d868Dean Nelson * Copyright (c) 2008-2009 Silicon Graphics, Inc. All Rights Reserved. 794bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson */ 894bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson 994bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson/* 1094bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson * Cross Partition Communication (XPC) uv-based functions. 1194bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson * 1294bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson * Architecture specific implementation of common functions. 1394bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson * 1494bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson */ 1594bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson 1694bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson#include <linux/kernel.h> 175b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson#include <linux/mm.h> 185b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson#include <linux/interrupt.h> 195b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson#include <linux/delay.h> 205b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson#include <linux/device.h> 217838f994b4fceff24c343f4e26a6cf4393869579Robin Holt#include <linux/cpu.h> 227838f994b4fceff24c343f4e26a6cf4393869579Robin Holt#include <linux/module.h> 232525789b4694d78df4f001063f042b2b74227d26Dean Nelson#include <linux/err.h> 245a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 25261f3b4979db88d29fc86aad9f76fbc0c2c6d21aDean Nelson#include <asm/uv/uv_hub.h> 262525789b4694d78df4f001063f042b2b74227d26Dean Nelson#if defined CONFIG_X86_64 272525789b4694d78df4f001063f042b2b74227d26Dean Nelson#include <asm/uv/bios.h> 282525789b4694d78df4f001063f042b2b74227d26Dean Nelson#include <asm/uv/uv_irq.h> 292525789b4694d78df4f001063f042b2b74227d26Dean Nelson#elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV 302525789b4694d78df4f001063f042b2b74227d26Dean Nelson#include <asm/sn/intr.h> 312525789b4694d78df4f001063f042b2b74227d26Dean Nelson#include <asm/sn/sn_sal.h> 322525789b4694d78df4f001063f042b2b74227d26Dean Nelson#endif 335b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson#include "../sgi-gru/gru.h" 34261f3b4979db88d29fc86aad9f76fbc0c2c6d21aDean Nelson#include "../sgi-gru/grukservices.h" 3594bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson#include "xpc.h" 3694bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson 376f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner#if defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV 386f2584f47474d29ce829604bfc8b56c10b352fdbJack Steinerstruct uv_IO_APIC_route_entry { 396f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner __u64 vector : 8, 406f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner delivery_mode : 3, 416f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner dest_mode : 1, 426f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner delivery_status : 1, 436f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner polarity : 1, 446f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner __reserved_1 : 1, 456f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner trigger : 1, 466f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner mask : 1, 476f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner __reserved_2 : 15, 486f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner dest : 32; 496f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner}; 506f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner#endif 516f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner 52a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holtstatic struct xpc_heartbeat_uv *xpc_heartbeat_uv; 5333ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson 545b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson#define XPC_ACTIVATE_MSG_SIZE_UV (1 * GRU_CACHE_LINE_BYTES) 552525789b4694d78df4f001063f042b2b74227d26Dean Nelson#define XPC_ACTIVATE_MQ_SIZE_UV (4 * XP_MAX_NPARTITIONS_UV * \ 562525789b4694d78df4f001063f042b2b74227d26Dean Nelson XPC_ACTIVATE_MSG_SIZE_UV) 572525789b4694d78df4f001063f042b2b74227d26Dean Nelson#define XPC_ACTIVATE_IRQ_NAME "xpc_activate" 585b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 592525789b4694d78df4f001063f042b2b74227d26Dean Nelson#define XPC_NOTIFY_MSG_SIZE_UV (2 * GRU_CACHE_LINE_BYTES) 602525789b4694d78df4f001063f042b2b74227d26Dean Nelson#define XPC_NOTIFY_MQ_SIZE_UV (4 * XP_MAX_NPARTITIONS_UV * \ 612525789b4694d78df4f001063f042b2b74227d26Dean Nelson XPC_NOTIFY_MSG_SIZE_UV) 622525789b4694d78df4f001063f042b2b74227d26Dean Nelson#define XPC_NOTIFY_IRQ_NAME "xpc_notify" 635b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 647838f994b4fceff24c343f4e26a6cf4393869579Robin Holtstatic int xpc_mq_node = -1; 657838f994b4fceff24c343f4e26a6cf4393869579Robin Holt 662525789b4694d78df4f001063f042b2b74227d26Dean Nelsonstatic struct xpc_gru_mq_uv *xpc_activate_mq_uv; 672525789b4694d78df4f001063f042b2b74227d26Dean Nelsonstatic struct xpc_gru_mq_uv *xpc_notify_mq_uv; 685b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 695b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic int 70a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holtxpc_setup_partitions_uv(void) 715b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 725b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson short partid; 735b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson struct xpc_partition_uv *part_uv; 745b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 755b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson for (partid = 0; partid < XP_MAX_NPARTITIONS_UV; partid++) { 765b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson part_uv = &xpc_partitions[partid].sn.uv; 775b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 786f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner mutex_init(&part_uv->cached_activate_gru_mq_desc_mutex); 795b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson spin_lock_init(&part_uv->flags_lock); 805b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson part_uv->remote_act_state = XPC_P_AS_INACTIVE; 815b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson } 825b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson return 0; 835b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 845b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 856f2584f47474d29ce829604bfc8b56c10b352fdbJack Steinerstatic void 86a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holtxpc_teardown_partitions_uv(void) 876f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner{ 886f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner short partid; 896f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner struct xpc_partition_uv *part_uv; 906f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner unsigned long irq_flags; 916f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner 926f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner for (partid = 0; partid < XP_MAX_NPARTITIONS_UV; partid++) { 936f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner part_uv = &xpc_partitions[partid].sn.uv; 946f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner 956f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner if (part_uv->cached_activate_gru_mq_desc != NULL) { 966f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner mutex_lock(&part_uv->cached_activate_gru_mq_desc_mutex); 976f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner spin_lock_irqsave(&part_uv->flags_lock, irq_flags); 986f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner part_uv->flags &= ~XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV; 996f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags); 1006f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner kfree(part_uv->cached_activate_gru_mq_desc); 1016f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner part_uv->cached_activate_gru_mq_desc = NULL; 1026f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner mutex_unlock(&part_uv-> 1036f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner cached_activate_gru_mq_desc_mutex); 1046f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner } 1056f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner } 1066f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner} 1076f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner 1082525789b4694d78df4f001063f042b2b74227d26Dean Nelsonstatic int 1092525789b4694d78df4f001063f042b2b74227d26Dean Nelsonxpc_get_gru_mq_irq_uv(struct xpc_gru_mq_uv *mq, int cpu, char *irq_name) 1102525789b4694d78df4f001063f042b2b74227d26Dean Nelson{ 1116f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner int mmr_pnode = uv_blade_to_pnode(mq->mmr_blade); 1126f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner 1132525789b4694d78df4f001063f042b2b74227d26Dean Nelson#if defined CONFIG_X86_64 1146c2c502910247d2820cb630e7b28fb6bdecdbf45Dimitri Sivanich mq->irq = uv_setup_irq(irq_name, cpu, mq->mmr_blade, mq->mmr_offset, 1156c2c502910247d2820cb630e7b28fb6bdecdbf45Dimitri Sivanich UV_AFFINITY_CPU); 1167838f994b4fceff24c343f4e26a6cf4393869579Robin Holt if (mq->irq < 0) 1176f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner return mq->irq; 1182525789b4694d78df4f001063f042b2b74227d26Dean Nelson 1196f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner mq->mmr_value = uv_read_global_mmr64(mmr_pnode, mq->mmr_offset); 1202525789b4694d78df4f001063f042b2b74227d26Dean Nelson 1216f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner#elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV 1222525789b4694d78df4f001063f042b2b74227d26Dean Nelson if (strcmp(irq_name, XPC_ACTIVATE_IRQ_NAME) == 0) 1232525789b4694d78df4f001063f042b2b74227d26Dean Nelson mq->irq = SGI_XPC_ACTIVATE; 1242525789b4694d78df4f001063f042b2b74227d26Dean Nelson else if (strcmp(irq_name, XPC_NOTIFY_IRQ_NAME) == 0) 1252525789b4694d78df4f001063f042b2b74227d26Dean Nelson mq->irq = SGI_XPC_NOTIFY; 1262525789b4694d78df4f001063f042b2b74227d26Dean Nelson else 1272525789b4694d78df4f001063f042b2b74227d26Dean Nelson return -EINVAL; 1282525789b4694d78df4f001063f042b2b74227d26Dean Nelson 1296f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner mq->mmr_value = (unsigned long)cpu_physical_id(cpu) << 32 | mq->irq; 1306f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner uv_write_global_mmr64(mmr_pnode, mq->mmr_offset, mq->mmr_value); 1312525789b4694d78df4f001063f042b2b74227d26Dean Nelson#else 1322525789b4694d78df4f001063f042b2b74227d26Dean Nelson #error not a supported configuration 1332525789b4694d78df4f001063f042b2b74227d26Dean Nelson#endif 1342525789b4694d78df4f001063f042b2b74227d26Dean Nelson 1352525789b4694d78df4f001063f042b2b74227d26Dean Nelson return 0; 1362525789b4694d78df4f001063f042b2b74227d26Dean Nelson} 1372525789b4694d78df4f001063f042b2b74227d26Dean Nelson 1382525789b4694d78df4f001063f042b2b74227d26Dean Nelsonstatic void 1392525789b4694d78df4f001063f042b2b74227d26Dean Nelsonxpc_release_gru_mq_irq_uv(struct xpc_gru_mq_uv *mq) 1402525789b4694d78df4f001063f042b2b74227d26Dean Nelson{ 1412525789b4694d78df4f001063f042b2b74227d26Dean Nelson#if defined CONFIG_X86_64 1426c2c502910247d2820cb630e7b28fb6bdecdbf45Dimitri Sivanich uv_teardown_irq(mq->irq); 1432525789b4694d78df4f001063f042b2b74227d26Dean Nelson 1442525789b4694d78df4f001063f042b2b74227d26Dean Nelson#elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV 1452525789b4694d78df4f001063f042b2b74227d26Dean Nelson int mmr_pnode; 1462525789b4694d78df4f001063f042b2b74227d26Dean Nelson unsigned long mmr_value; 1472525789b4694d78df4f001063f042b2b74227d26Dean Nelson 1482525789b4694d78df4f001063f042b2b74227d26Dean Nelson mmr_pnode = uv_blade_to_pnode(mq->mmr_blade); 1492525789b4694d78df4f001063f042b2b74227d26Dean Nelson mmr_value = 1UL << 16; 1502525789b4694d78df4f001063f042b2b74227d26Dean Nelson 1512525789b4694d78df4f001063f042b2b74227d26Dean Nelson uv_write_global_mmr64(mmr_pnode, mq->mmr_offset, mmr_value); 1522525789b4694d78df4f001063f042b2b74227d26Dean Nelson#else 1532525789b4694d78df4f001063f042b2b74227d26Dean Nelson #error not a supported configuration 1542525789b4694d78df4f001063f042b2b74227d26Dean Nelson#endif 1552525789b4694d78df4f001063f042b2b74227d26Dean Nelson} 1562525789b4694d78df4f001063f042b2b74227d26Dean Nelson 1572525789b4694d78df4f001063f042b2b74227d26Dean Nelsonstatic int 1582525789b4694d78df4f001063f042b2b74227d26Dean Nelsonxpc_gru_mq_watchlist_alloc_uv(struct xpc_gru_mq_uv *mq) 1592525789b4694d78df4f001063f042b2b74227d26Dean Nelson{ 1602525789b4694d78df4f001063f042b2b74227d26Dean Nelson int ret; 1612525789b4694d78df4f001063f042b2b74227d26Dean Nelson 162c2c9f115741453715d6b4da1cd2de65af8c7ad86Robin Holt#if defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV 163c2c9f115741453715d6b4da1cd2de65af8c7ad86Robin Holt int mmr_pnode = uv_blade_to_pnode(mq->mmr_blade); 164c2c9f115741453715d6b4da1cd2de65af8c7ad86Robin Holt 165c2c9f115741453715d6b4da1cd2de65af8c7ad86Robin Holt ret = sn_mq_watchlist_alloc(mmr_pnode, (void *)uv_gpa(mq->address), 166c8182f0016fb65a721c4fbe487909a2d56178135Russ Anderson mq->order, &mq->mmr_offset); 1672525789b4694d78df4f001063f042b2b74227d26Dean Nelson if (ret < 0) { 1682525789b4694d78df4f001063f042b2b74227d26Dean Nelson dev_err(xpc_part, "sn_mq_watchlist_alloc() failed, ret=%d\n", 1692525789b4694d78df4f001063f042b2b74227d26Dean Nelson ret); 1702525789b4694d78df4f001063f042b2b74227d26Dean Nelson return -EBUSY; 1712525789b4694d78df4f001063f042b2b74227d26Dean Nelson } 172c2c9f115741453715d6b4da1cd2de65af8c7ad86Robin Holt#elif defined CONFIG_X86_64 173c2c9f115741453715d6b4da1cd2de65af8c7ad86Robin Holt ret = uv_bios_mq_watchlist_alloc(uv_gpa(mq->address), 174c2c9f115741453715d6b4da1cd2de65af8c7ad86Robin Holt mq->order, &mq->mmr_offset); 175c2c9f115741453715d6b4da1cd2de65af8c7ad86Robin Holt if (ret < 0) { 176c2c9f115741453715d6b4da1cd2de65af8c7ad86Robin Holt dev_err(xpc_part, "uv_bios_mq_watchlist_alloc() failed, " 177c2c9f115741453715d6b4da1cd2de65af8c7ad86Robin Holt "ret=%d\n", ret); 178c2c9f115741453715d6b4da1cd2de65af8c7ad86Robin Holt return ret; 179c2c9f115741453715d6b4da1cd2de65af8c7ad86Robin Holt } 1802525789b4694d78df4f001063f042b2b74227d26Dean Nelson#else 1812525789b4694d78df4f001063f042b2b74227d26Dean Nelson #error not a supported configuration 1822525789b4694d78df4f001063f042b2b74227d26Dean Nelson#endif 1832525789b4694d78df4f001063f042b2b74227d26Dean Nelson 1842525789b4694d78df4f001063f042b2b74227d26Dean Nelson mq->watchlist_num = ret; 1852525789b4694d78df4f001063f042b2b74227d26Dean Nelson return 0; 1862525789b4694d78df4f001063f042b2b74227d26Dean Nelson} 1872525789b4694d78df4f001063f042b2b74227d26Dean Nelson 1882525789b4694d78df4f001063f042b2b74227d26Dean Nelsonstatic void 1892525789b4694d78df4f001063f042b2b74227d26Dean Nelsonxpc_gru_mq_watchlist_free_uv(struct xpc_gru_mq_uv *mq) 1902525789b4694d78df4f001063f042b2b74227d26Dean Nelson{ 1912525789b4694d78df4f001063f042b2b74227d26Dean Nelson int ret; 192c2c9f115741453715d6b4da1cd2de65af8c7ad86Robin Holt int mmr_pnode = uv_blade_to_pnode(mq->mmr_blade); 1932525789b4694d78df4f001063f042b2b74227d26Dean Nelson 1942525789b4694d78df4f001063f042b2b74227d26Dean Nelson#if defined CONFIG_X86_64 195c2c9f115741453715d6b4da1cd2de65af8c7ad86Robin Holt ret = uv_bios_mq_watchlist_free(mmr_pnode, mq->watchlist_num); 1962525789b4694d78df4f001063f042b2b74227d26Dean Nelson BUG_ON(ret != BIOS_STATUS_SUCCESS); 1972525789b4694d78df4f001063f042b2b74227d26Dean Nelson#elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV 198c2c9f115741453715d6b4da1cd2de65af8c7ad86Robin Holt ret = sn_mq_watchlist_free(mmr_pnode, mq->watchlist_num); 1992525789b4694d78df4f001063f042b2b74227d26Dean Nelson BUG_ON(ret != SALRET_OK); 2002525789b4694d78df4f001063f042b2b74227d26Dean Nelson#else 2012525789b4694d78df4f001063f042b2b74227d26Dean Nelson #error not a supported configuration 2022525789b4694d78df4f001063f042b2b74227d26Dean Nelson#endif 2032525789b4694d78df4f001063f042b2b74227d26Dean Nelson} 2042525789b4694d78df4f001063f042b2b74227d26Dean Nelson 2052525789b4694d78df4f001063f042b2b74227d26Dean Nelsonstatic struct xpc_gru_mq_uv * 2062525789b4694d78df4f001063f042b2b74227d26Dean Nelsonxpc_create_gru_mq_uv(unsigned int mq_size, int cpu, char *irq_name, 2075b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson irq_handler_t irq_handler) 2085b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 2092525789b4694d78df4f001063f042b2b74227d26Dean Nelson enum xp_retval xp_ret; 2105b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson int ret; 2115b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson int nid; 21257e6d258b1e41cd7ceb26fa43ce116939d8440b1Robin Holt int nasid; 2132525789b4694d78df4f001063f042b2b74227d26Dean Nelson int pg_order; 2145b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson struct page *page; 2152525789b4694d78df4f001063f042b2b74227d26Dean Nelson struct xpc_gru_mq_uv *mq; 2166f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner struct uv_IO_APIC_route_entry *mmr_value; 2172525789b4694d78df4f001063f042b2b74227d26Dean Nelson 2182525789b4694d78df4f001063f042b2b74227d26Dean Nelson mq = kmalloc(sizeof(struct xpc_gru_mq_uv), GFP_KERNEL); 2192525789b4694d78df4f001063f042b2b74227d26Dean Nelson if (mq == NULL) { 2202525789b4694d78df4f001063f042b2b74227d26Dean Nelson dev_err(xpc_part, "xpc_create_gru_mq_uv() failed to kmalloc() " 2212525789b4694d78df4f001063f042b2b74227d26Dean Nelson "a xpc_gru_mq_uv structure\n"); 2222525789b4694d78df4f001063f042b2b74227d26Dean Nelson ret = -ENOMEM; 2236f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner goto out_0; 2246f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner } 2256f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner 2266f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner mq->gru_mq_desc = kzalloc(sizeof(struct gru_message_queue_desc), 2276f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner GFP_KERNEL); 2286f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner if (mq->gru_mq_desc == NULL) { 2296f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner dev_err(xpc_part, "xpc_create_gru_mq_uv() failed to kmalloc() " 2306f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner "a gru_message_queue_desc structure\n"); 2316f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner ret = -ENOMEM; 2322525789b4694d78df4f001063f042b2b74227d26Dean Nelson goto out_1; 2332525789b4694d78df4f001063f042b2b74227d26Dean Nelson } 2345b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 2352525789b4694d78df4f001063f042b2b74227d26Dean Nelson pg_order = get_order(mq_size); 2362525789b4694d78df4f001063f042b2b74227d26Dean Nelson mq->order = pg_order + PAGE_SHIFT; 2372525789b4694d78df4f001063f042b2b74227d26Dean Nelson mq_size = 1UL << mq->order; 2382525789b4694d78df4f001063f042b2b74227d26Dean Nelson 2392525789b4694d78df4f001063f042b2b74227d26Dean Nelson mq->mmr_blade = uv_cpu_to_blade_id(cpu); 2402525789b4694d78df4f001063f042b2b74227d26Dean Nelson 2412525789b4694d78df4f001063f042b2b74227d26Dean Nelson nid = cpu_to_node(cpu); 2427838f994b4fceff24c343f4e26a6cf4393869579Robin Holt page = alloc_pages_exact_node(nid, 243e97ca8e5b864f88b028c1759ba8536fa827d6d96Johannes Weiner GFP_KERNEL | __GFP_ZERO | __GFP_THISNODE, 2447838f994b4fceff24c343f4e26a6cf4393869579Robin Holt pg_order); 245bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (page == NULL) { 246bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson dev_err(xpc_part, "xpc_create_gru_mq_uv() failed to alloc %d " 247bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson "bytes of memory on nid=%d for GRU mq\n", mq_size, nid); 2482525789b4694d78df4f001063f042b2b74227d26Dean Nelson ret = -ENOMEM; 2492525789b4694d78df4f001063f042b2b74227d26Dean Nelson goto out_2; 250bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 2512525789b4694d78df4f001063f042b2b74227d26Dean Nelson mq->address = page_address(page); 2525b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 2532525789b4694d78df4f001063f042b2b74227d26Dean Nelson /* enable generation of irq when GRU mq operation occurs to this mq */ 2542525789b4694d78df4f001063f042b2b74227d26Dean Nelson ret = xpc_gru_mq_watchlist_alloc_uv(mq); 2552525789b4694d78df4f001063f042b2b74227d26Dean Nelson if (ret != 0) 2562525789b4694d78df4f001063f042b2b74227d26Dean Nelson goto out_3; 2575b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 2582525789b4694d78df4f001063f042b2b74227d26Dean Nelson ret = xpc_get_gru_mq_irq_uv(mq, cpu, irq_name); 2592525789b4694d78df4f001063f042b2b74227d26Dean Nelson if (ret != 0) 2602525789b4694d78df4f001063f042b2b74227d26Dean Nelson goto out_4; 2612525789b4694d78df4f001063f042b2b74227d26Dean Nelson 2622525789b4694d78df4f001063f042b2b74227d26Dean Nelson ret = request_irq(mq->irq, irq_handler, 0, irq_name, NULL); 2635b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson if (ret != 0) { 2645b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson dev_err(xpc_part, "request_irq(irq=%d) returned error=%d\n", 2656f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner mq->irq, -ret); 2662525789b4694d78df4f001063f042b2b74227d26Dean Nelson goto out_5; 2675b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson } 2685b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 26957e6d258b1e41cd7ceb26fa43ce116939d8440b1Robin Holt nasid = UV_PNODE_TO_NASID(uv_cpu_to_pnode(cpu)); 27057e6d258b1e41cd7ceb26fa43ce116939d8440b1Robin Holt 2716f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner mmr_value = (struct uv_IO_APIC_route_entry *)&mq->mmr_value; 2726f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner ret = gru_create_message_queue(mq->gru_mq_desc, mq->address, mq_size, 27357e6d258b1e41cd7ceb26fa43ce116939d8440b1Robin Holt nasid, mmr_value->vector, mmr_value->dest); 2746f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner if (ret != 0) { 2756f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner dev_err(xpc_part, "gru_create_message_queue() returned " 2766f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner "error=%d\n", ret); 2776f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner ret = -EINVAL; 2786f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner goto out_6; 2796f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner } 2806f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner 2812525789b4694d78df4f001063f042b2b74227d26Dean Nelson /* allow other partitions to access this GRU mq */ 2822525789b4694d78df4f001063f042b2b74227d26Dean Nelson xp_ret = xp_expand_memprotect(xp_pa(mq->address), mq_size); 2832525789b4694d78df4f001063f042b2b74227d26Dean Nelson if (xp_ret != xpSuccess) { 2842525789b4694d78df4f001063f042b2b74227d26Dean Nelson ret = -EACCES; 2852525789b4694d78df4f001063f042b2b74227d26Dean Nelson goto out_6; 2862525789b4694d78df4f001063f042b2b74227d26Dean Nelson } 2875b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 2885b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson return mq; 2892525789b4694d78df4f001063f042b2b74227d26Dean Nelson 2902525789b4694d78df4f001063f042b2b74227d26Dean Nelson /* something went wrong */ 2912525789b4694d78df4f001063f042b2b74227d26Dean Nelsonout_6: 2922525789b4694d78df4f001063f042b2b74227d26Dean Nelson free_irq(mq->irq, NULL); 2932525789b4694d78df4f001063f042b2b74227d26Dean Nelsonout_5: 2942525789b4694d78df4f001063f042b2b74227d26Dean Nelson xpc_release_gru_mq_irq_uv(mq); 2952525789b4694d78df4f001063f042b2b74227d26Dean Nelsonout_4: 2962525789b4694d78df4f001063f042b2b74227d26Dean Nelson xpc_gru_mq_watchlist_free_uv(mq); 2972525789b4694d78df4f001063f042b2b74227d26Dean Nelsonout_3: 2982525789b4694d78df4f001063f042b2b74227d26Dean Nelson free_pages((unsigned long)mq->address, pg_order); 2992525789b4694d78df4f001063f042b2b74227d26Dean Nelsonout_2: 3006f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner kfree(mq->gru_mq_desc); 3012525789b4694d78df4f001063f042b2b74227d26Dean Nelsonout_1: 3026f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner kfree(mq); 3036f2584f47474d29ce829604bfc8b56c10b352fdbJack Steinerout_0: 3042525789b4694d78df4f001063f042b2b74227d26Dean Nelson return ERR_PTR(ret); 3055b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 30694bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson 30733ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelsonstatic void 3082525789b4694d78df4f001063f042b2b74227d26Dean Nelsonxpc_destroy_gru_mq_uv(struct xpc_gru_mq_uv *mq) 3095b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 3102525789b4694d78df4f001063f042b2b74227d26Dean Nelson unsigned int mq_size; 3112525789b4694d78df4f001063f042b2b74227d26Dean Nelson int pg_order; 3122525789b4694d78df4f001063f042b2b74227d26Dean Nelson int ret; 3132525789b4694d78df4f001063f042b2b74227d26Dean Nelson 3142525789b4694d78df4f001063f042b2b74227d26Dean Nelson /* disallow other partitions to access GRU mq */ 3152525789b4694d78df4f001063f042b2b74227d26Dean Nelson mq_size = 1UL << mq->order; 3162525789b4694d78df4f001063f042b2b74227d26Dean Nelson ret = xp_restrict_memprotect(xp_pa(mq->address), mq_size); 3172525789b4694d78df4f001063f042b2b74227d26Dean Nelson BUG_ON(ret != xpSuccess); 3182525789b4694d78df4f001063f042b2b74227d26Dean Nelson 3192525789b4694d78df4f001063f042b2b74227d26Dean Nelson /* unregister irq handler and release mq irq/vector mapping */ 3202525789b4694d78df4f001063f042b2b74227d26Dean Nelson free_irq(mq->irq, NULL); 3212525789b4694d78df4f001063f042b2b74227d26Dean Nelson xpc_release_gru_mq_irq_uv(mq); 3225b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 3232525789b4694d78df4f001063f042b2b74227d26Dean Nelson /* disable generation of irq when GRU mq op occurs to this mq */ 3242525789b4694d78df4f001063f042b2b74227d26Dean Nelson xpc_gru_mq_watchlist_free_uv(mq); 3255b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 3262525789b4694d78df4f001063f042b2b74227d26Dean Nelson pg_order = mq->order - PAGE_SHIFT; 3272525789b4694d78df4f001063f042b2b74227d26Dean Nelson free_pages((unsigned long)mq->address, pg_order); 3285b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 3292525789b4694d78df4f001063f042b2b74227d26Dean Nelson kfree(mq); 3305b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 3315b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 3325b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic enum xp_retval 3336f2584f47474d29ce829604bfc8b56c10b352fdbJack Steinerxpc_send_gru_msg(struct gru_message_queue_desc *gru_mq_desc, void *msg, 3346f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner size_t msg_size) 33533ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson{ 3365b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson enum xp_retval xp_ret; 3375b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson int ret; 3385b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 3395b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson while (1) { 3406f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner ret = gru_send_message_gpa(gru_mq_desc, msg, msg_size); 3415b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson if (ret == MQE_OK) { 3425b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson xp_ret = xpSuccess; 3435b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson break; 3445b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson } 3455b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 3465b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson if (ret == MQE_QUEUE_FULL) { 3475b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson dev_dbg(xpc_chan, "gru_send_message_gpa() returned " 3485b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson "error=MQE_QUEUE_FULL\n"); 3495b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson /* !!! handle QLimit reached; delay & try again */ 3505b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson /* ??? Do we add a limit to the number of retries? */ 3515b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson (void)msleep_interruptible(10); 3525b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson } else if (ret == MQE_CONGESTION) { 3535b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson dev_dbg(xpc_chan, "gru_send_message_gpa() returned " 3545b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson "error=MQE_CONGESTION\n"); 3555b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson /* !!! handle LB Overflow; simply try again */ 3565b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson /* ??? Do we add a limit to the number of retries? */ 3575b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson } else { 3585b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson /* !!! Currently this is MQE_UNEXPECTED_CB_ERR */ 3595b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson dev_err(xpc_chan, "gru_send_message_gpa() returned " 3605b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson "error=%d\n", ret); 3615b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson xp_ret = xpGruSendMqError; 3625b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson break; 3635b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson } 3645b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson } 3655b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson return xp_ret; 3665b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 3675b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 3685b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic void 3695b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_process_activate_IRQ_rcvd_uv(void) 3705b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 3715b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson unsigned long irq_flags; 3725b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson short partid; 3735b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson struct xpc_partition *part; 3745b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson u8 act_state_req; 3755b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 3765b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson DBUG_ON(xpc_activate_IRQ_rcvd == 0); 3775b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 3785b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); 3795b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson for (partid = 0; partid < XP_MAX_NPARTITIONS_UV; partid++) { 3805b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson part = &xpc_partitions[partid]; 3815b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 3825b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson if (part->sn.uv.act_state_req == 0) 3835b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson continue; 3845b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 3855b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson xpc_activate_IRQ_rcvd--; 3865b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson BUG_ON(xpc_activate_IRQ_rcvd < 0); 3875b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 3885b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson act_state_req = part->sn.uv.act_state_req; 3895b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson part->sn.uv.act_state_req = 0; 3905b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); 3915b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 3925b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson if (act_state_req == XPC_P_ASR_ACTIVATE_UV) { 3935b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson if (part->act_state == XPC_P_AS_INACTIVE) 3945b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson xpc_activate_partition(part); 3955b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson else if (part->act_state == XPC_P_AS_DEACTIVATING) 3965b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson XPC_DEACTIVATE_PARTITION(part, xpReactivating); 3975b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 3985b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson } else if (act_state_req == XPC_P_ASR_REACTIVATE_UV) { 3995b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson if (part->act_state == XPC_P_AS_INACTIVE) 4005b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson xpc_activate_partition(part); 4015b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson else 4025b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson XPC_DEACTIVATE_PARTITION(part, xpReactivating); 4035b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 4045b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson } else if (act_state_req == XPC_P_ASR_DEACTIVATE_UV) { 4055b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson XPC_DEACTIVATE_PARTITION(part, part->sn.uv.reason); 4065b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 4075b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson } else { 4085b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson BUG(); 4095b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson } 4105b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 4115b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); 4125b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson if (xpc_activate_IRQ_rcvd == 0) 4135b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson break; 4145b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson } 4155b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); 4165b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 4175b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 4185b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 419bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonstatic void 420bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_handle_activate_mq_msg_uv(struct xpc_partition *part, 421bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_activate_mq_msghdr_uv *msg_hdr, 42209358972bff5ce99de496bbba97c85d417b3c054Robin Holt int part_setup, 423bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson int *wakeup_hb_checker) 4245b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 4255b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson unsigned long irq_flags; 426bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_partition_uv *part_uv = &part->sn.uv; 4275b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson struct xpc_openclose_args *args; 4285b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 429bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson part_uv->remote_act_state = msg_hdr->act_state; 4305b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 431bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson switch (msg_hdr->type) { 432bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson case XPC_ACTIVATE_MQ_MSG_SYNC_ACT_STATE_UV: 433bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson /* syncing of remote_act_state was just done above */ 434bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson break; 4355b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 436bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson case XPC_ACTIVATE_MQ_MSG_ACTIVATE_REQ_UV: { 437bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_activate_mq_msg_activate_req_uv *msg; 4385b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 439bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson /* 440bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * ??? Do we deal here with ts_jiffies being different 441bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * ??? if act_state != XPC_P_AS_INACTIVE instead of 442bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * ??? below? 443bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson */ 444bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg = container_of(msg_hdr, struct 445bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_activate_mq_msg_activate_req_uv, hdr); 4465b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 447bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); 448bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (part_uv->act_state_req == 0) 449bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_activate_IRQ_rcvd++; 450bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson part_uv->act_state_req = XPC_P_ASR_ACTIVATE_UV; 451bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson part->remote_rp_pa = msg->rp_gpa; /* !!! _pa is _gpa */ 452bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson part->remote_rp_ts_jiffies = msg_hdr->rp_ts_jiffies; 453a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt part_uv->heartbeat_gpa = msg->heartbeat_gpa; 4546f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner 4556f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner if (msg->activate_gru_mq_desc_gpa != 4566f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner part_uv->activate_gru_mq_desc_gpa) { 4578875408abd935a77b6e1cb11c21c438aa2e7ec75Dan Carpenter spin_lock(&part_uv->flags_lock); 4586f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner part_uv->flags &= ~XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV; 4598875408abd935a77b6e1cb11c21c438aa2e7ec75Dan Carpenter spin_unlock(&part_uv->flags_lock); 4606f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner part_uv->activate_gru_mq_desc_gpa = 4616f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner msg->activate_gru_mq_desc_gpa; 4626f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner } 463bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); 4645b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 465bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson (*wakeup_hb_checker)++; 466bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson break; 467bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 468bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson case XPC_ACTIVATE_MQ_MSG_DEACTIVATE_REQ_UV: { 469bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_activate_mq_msg_deactivate_req_uv *msg; 4705b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 471bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg = container_of(msg_hdr, struct 472bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_activate_mq_msg_deactivate_req_uv, hdr); 4735b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 474bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); 475bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (part_uv->act_state_req == 0) 476bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_activate_IRQ_rcvd++; 477bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson part_uv->act_state_req = XPC_P_ASR_DEACTIVATE_UV; 478bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson part_uv->reason = msg->reason; 479bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); 480bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 481bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson (*wakeup_hb_checker)++; 482bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return; 483bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 484bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson case XPC_ACTIVATE_MQ_MSG_CHCTL_CLOSEREQUEST_UV: { 485bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_activate_mq_msg_chctl_closerequest_uv *msg; 4865b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 48709358972bff5ce99de496bbba97c85d417b3c054Robin Holt if (!part_setup) 48809358972bff5ce99de496bbba97c85d417b3c054Robin Holt break; 48909358972bff5ce99de496bbba97c85d417b3c054Robin Holt 490bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg = container_of(msg_hdr, struct 491bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_activate_mq_msg_chctl_closerequest_uv, 492bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson hdr); 493bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson args = &part->remote_openclose_args[msg->ch_number]; 494bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson args->reason = msg->reason; 4955b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 496bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_lock_irqsave(&part->chctl_lock, irq_flags); 497bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson part->chctl.flags[msg->ch_number] |= XPC_CHCTL_CLOSEREQUEST; 498bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_unlock_irqrestore(&part->chctl_lock, irq_flags); 4995b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 500bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_wakeup_channel_mgr(part); 501bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson break; 502bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 503bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson case XPC_ACTIVATE_MQ_MSG_CHCTL_CLOSEREPLY_UV: { 504bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_activate_mq_msg_chctl_closereply_uv *msg; 5055b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 50609358972bff5ce99de496bbba97c85d417b3c054Robin Holt if (!part_setup) 50709358972bff5ce99de496bbba97c85d417b3c054Robin Holt break; 50809358972bff5ce99de496bbba97c85d417b3c054Robin Holt 509bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg = container_of(msg_hdr, struct 510bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_activate_mq_msg_chctl_closereply_uv, 511bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson hdr); 5125b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 513bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_lock_irqsave(&part->chctl_lock, irq_flags); 514bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson part->chctl.flags[msg->ch_number] |= XPC_CHCTL_CLOSEREPLY; 515bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_unlock_irqrestore(&part->chctl_lock, irq_flags); 5165b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 517bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_wakeup_channel_mgr(part); 518bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson break; 519bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 520bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson case XPC_ACTIVATE_MQ_MSG_CHCTL_OPENREQUEST_UV: { 521bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_activate_mq_msg_chctl_openrequest_uv *msg; 522bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 52309358972bff5ce99de496bbba97c85d417b3c054Robin Holt if (!part_setup) 52409358972bff5ce99de496bbba97c85d417b3c054Robin Holt break; 52509358972bff5ce99de496bbba97c85d417b3c054Robin Holt 526bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg = container_of(msg_hdr, struct 527bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_activate_mq_msg_chctl_openrequest_uv, 528bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson hdr); 529bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson args = &part->remote_openclose_args[msg->ch_number]; 530bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson args->entry_size = msg->entry_size; 531bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson args->local_nentries = msg->local_nentries; 532bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 533bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_lock_irqsave(&part->chctl_lock, irq_flags); 534bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson part->chctl.flags[msg->ch_number] |= XPC_CHCTL_OPENREQUEST; 535bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_unlock_irqrestore(&part->chctl_lock, irq_flags); 536bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 537bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_wakeup_channel_mgr(part); 538bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson break; 539bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 540bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson case XPC_ACTIVATE_MQ_MSG_CHCTL_OPENREPLY_UV: { 541bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_activate_mq_msg_chctl_openreply_uv *msg; 542bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 54309358972bff5ce99de496bbba97c85d417b3c054Robin Holt if (!part_setup) 54409358972bff5ce99de496bbba97c85d417b3c054Robin Holt break; 54509358972bff5ce99de496bbba97c85d417b3c054Robin Holt 546bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg = container_of(msg_hdr, struct 547bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_activate_mq_msg_chctl_openreply_uv, hdr); 548bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson args = &part->remote_openclose_args[msg->ch_number]; 549bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson args->remote_nentries = msg->remote_nentries; 550bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson args->local_nentries = msg->local_nentries; 5516f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner args->local_msgqueue_pa = msg->notify_gru_mq_desc_gpa; 552bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 553bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_lock_irqsave(&part->chctl_lock, irq_flags); 554bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson part->chctl.flags[msg->ch_number] |= XPC_CHCTL_OPENREPLY; 555bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_unlock_irqrestore(&part->chctl_lock, irq_flags); 556bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 557bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_wakeup_channel_mgr(part); 558bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson break; 559bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 560efdd06ed181a88a11e612238c1ac04668e665395Robin Holt case XPC_ACTIVATE_MQ_MSG_CHCTL_OPENCOMPLETE_UV: { 561efdd06ed181a88a11e612238c1ac04668e665395Robin Holt struct xpc_activate_mq_msg_chctl_opencomplete_uv *msg; 562efdd06ed181a88a11e612238c1ac04668e665395Robin Holt 56309358972bff5ce99de496bbba97c85d417b3c054Robin Holt if (!part_setup) 56409358972bff5ce99de496bbba97c85d417b3c054Robin Holt break; 56509358972bff5ce99de496bbba97c85d417b3c054Robin Holt 566efdd06ed181a88a11e612238c1ac04668e665395Robin Holt msg = container_of(msg_hdr, struct 567efdd06ed181a88a11e612238c1ac04668e665395Robin Holt xpc_activate_mq_msg_chctl_opencomplete_uv, hdr); 568efdd06ed181a88a11e612238c1ac04668e665395Robin Holt spin_lock_irqsave(&part->chctl_lock, irq_flags); 569efdd06ed181a88a11e612238c1ac04668e665395Robin Holt part->chctl.flags[msg->ch_number] |= XPC_CHCTL_OPENCOMPLETE; 570efdd06ed181a88a11e612238c1ac04668e665395Robin Holt spin_unlock_irqrestore(&part->chctl_lock, irq_flags); 571efdd06ed181a88a11e612238c1ac04668e665395Robin Holt 572efdd06ed181a88a11e612238c1ac04668e665395Robin Holt xpc_wakeup_channel_mgr(part); 573efdd06ed181a88a11e612238c1ac04668e665395Robin Holt } 574bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson case XPC_ACTIVATE_MQ_MSG_MARK_ENGAGED_UV: 575bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_lock_irqsave(&part_uv->flags_lock, irq_flags); 576bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson part_uv->flags |= XPC_P_ENGAGED_UV; 577bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags); 578bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson break; 579bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 580bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson case XPC_ACTIVATE_MQ_MSG_MARK_DISENGAGED_UV: 581bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_lock_irqsave(&part_uv->flags_lock, irq_flags); 582bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson part_uv->flags &= ~XPC_P_ENGAGED_UV; 583bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags); 584bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson break; 585bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 586bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson default: 587bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson dev_err(xpc_part, "received unknown activate_mq msg type=%d " 588bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson "from partition=%d\n", msg_hdr->type, XPC_PARTID(part)); 589bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 590bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson /* get hb checker to deactivate from the remote partition */ 591bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); 592bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (part_uv->act_state_req == 0) 593bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_activate_IRQ_rcvd++; 594bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson part_uv->act_state_req = XPC_P_ASR_DEACTIVATE_UV; 595bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson part_uv->reason = xpBadMsgType; 596bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); 5975b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 598bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson (*wakeup_hb_checker)++; 599bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return; 600bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 6015b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 602bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (msg_hdr->rp_ts_jiffies != part->remote_rp_ts_jiffies && 603bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson part->remote_rp_ts_jiffies != 0) { 604bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson /* 605bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * ??? Does what we do here need to be sensitive to 606bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * ??? act_state or remote_act_state? 607bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson */ 608bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); 609bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (part_uv->act_state_req == 0) 610bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_activate_IRQ_rcvd++; 611bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson part_uv->act_state_req = XPC_P_ASR_REACTIVATE_UV; 612bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); 6135b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 614bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson (*wakeup_hb_checker)++; 615bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 616bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson} 617bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 618bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonstatic irqreturn_t 619bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_handle_activate_IRQ_uv(int irq, void *dev_id) 620bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson{ 621bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_activate_mq_msghdr_uv *msg_hdr; 622bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson short partid; 623bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_partition *part; 624bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson int wakeup_hb_checker = 0; 6256f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner int part_referenced; 626bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 6272525789b4694d78df4f001063f042b2b74227d26Dean Nelson while (1) { 6286f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner msg_hdr = gru_get_next_message(xpc_activate_mq_uv->gru_mq_desc); 6292525789b4694d78df4f001063f042b2b74227d26Dean Nelson if (msg_hdr == NULL) 6302525789b4694d78df4f001063f042b2b74227d26Dean Nelson break; 631bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 632bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson partid = msg_hdr->partid; 633bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (partid < 0 || partid >= XP_MAX_NPARTITIONS_UV) { 634bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson dev_err(xpc_part, "xpc_handle_activate_IRQ_uv() " 635bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson "received invalid partid=0x%x in message\n", 636bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson partid); 637bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } else { 638bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson part = &xpc_partitions[partid]; 6396f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner 6406f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner part_referenced = xpc_part_ref(part); 6416f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner xpc_handle_activate_mq_msg_uv(part, msg_hdr, 64209358972bff5ce99de496bbba97c85d417b3c054Robin Holt part_referenced, 6436f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner &wakeup_hb_checker); 6446f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner if (part_referenced) 645bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_part_deref(part); 6465b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson } 6475b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 6486f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner gru_free_message(xpc_activate_mq_uv->gru_mq_desc, msg_hdr); 6495b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson } 6505b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 6515b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson if (wakeup_hb_checker) 6525b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson wake_up_interruptible(&xpc_activate_IRQ_wq); 6535b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 6545b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson return IRQ_HANDLED; 6555b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 6565b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 6575b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic enum xp_retval 6586f2584f47474d29ce829604bfc8b56c10b352fdbJack Steinerxpc_cache_remote_gru_mq_desc_uv(struct gru_message_queue_desc *gru_mq_desc, 6596f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner unsigned long gru_mq_desc_gpa) 6606f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner{ 6616f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner enum xp_retval ret; 6626f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner 6636f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner ret = xp_remote_memcpy(uv_gpa(gru_mq_desc), gru_mq_desc_gpa, 6646f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner sizeof(struct gru_message_queue_desc)); 6656f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner if (ret == xpSuccess) 6666f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner gru_mq_desc->mq = NULL; 6676f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner 6686f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner return ret; 6696f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner} 6706f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner 6716f2584f47474d29ce829604bfc8b56c10b352fdbJack Steinerstatic enum xp_retval 6725b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_send_activate_IRQ_uv(struct xpc_partition *part, void *msg, size_t msg_size, 6735b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson int msg_type) 6745b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 6755b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson struct xpc_activate_mq_msghdr_uv *msg_hdr = msg; 6766f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner struct xpc_partition_uv *part_uv = &part->sn.uv; 6776f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner struct gru_message_queue_desc *gru_mq_desc; 6786f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner unsigned long irq_flags; 6796f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner enum xp_retval ret; 6805b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 6815b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson DBUG_ON(msg_size > XPC_ACTIVATE_MSG_SIZE_UV); 6825b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 6835b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson msg_hdr->type = msg_type; 6846f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner msg_hdr->partid = xp_partition_id; 6855b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson msg_hdr->act_state = part->act_state; 6865b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson msg_hdr->rp_ts_jiffies = xpc_rsvd_page->ts_jiffies; 6875b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 6886f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner mutex_lock(&part_uv->cached_activate_gru_mq_desc_mutex); 6896f2584f47474d29ce829604bfc8b56c10b352fdbJack Steineragain: 6906f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner if (!(part_uv->flags & XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV)) { 6916f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner gru_mq_desc = part_uv->cached_activate_gru_mq_desc; 6926f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner if (gru_mq_desc == NULL) { 6936f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner gru_mq_desc = kmalloc(sizeof(struct 6946f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner gru_message_queue_desc), 6956f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner GFP_KERNEL); 6966f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner if (gru_mq_desc == NULL) { 6976f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner ret = xpNoMemory; 6986f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner goto done; 6996f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner } 7006f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner part_uv->cached_activate_gru_mq_desc = gru_mq_desc; 7016f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner } 7026f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner 7036f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner ret = xpc_cache_remote_gru_mq_desc_uv(gru_mq_desc, 7046f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner part_uv-> 7056f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner activate_gru_mq_desc_gpa); 7066f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner if (ret != xpSuccess) 7076f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner goto done; 7086f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner 7096f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner spin_lock_irqsave(&part_uv->flags_lock, irq_flags); 7106f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner part_uv->flags |= XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV; 7116f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags); 7126f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner } 7136f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner 7145b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson /* ??? Is holding a spin_lock (ch->lock) during this call a bad idea? */ 7156f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner ret = xpc_send_gru_msg(part_uv->cached_activate_gru_mq_desc, msg, 7166f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner msg_size); 7176f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner if (ret != xpSuccess) { 7186f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner smp_rmb(); /* ensure a fresh copy of part_uv->flags */ 7196f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner if (!(part_uv->flags & XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV)) 7206f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner goto again; 7216f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner } 7226f2584f47474d29ce829604bfc8b56c10b352fdbJack Steinerdone: 7236f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner mutex_unlock(&part_uv->cached_activate_gru_mq_desc_mutex); 7246f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner return ret; 7255b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 7265b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 7275b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic void 7285b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_send_activate_IRQ_part_uv(struct xpc_partition *part, void *msg, 7295b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson size_t msg_size, int msg_type) 7305b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 7315b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson enum xp_retval ret; 7325b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 7335b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson ret = xpc_send_activate_IRQ_uv(part, msg, msg_size, msg_type); 7345b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson if (unlikely(ret != xpSuccess)) 7355b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson XPC_DEACTIVATE_PARTITION(part, ret); 7365b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 7375b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 7385b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic void 7395b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_send_activate_IRQ_ch_uv(struct xpc_channel *ch, unsigned long *irq_flags, 7405b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson void *msg, size_t msg_size, int msg_type) 7415b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 7426f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner struct xpc_partition *part = &xpc_partitions[ch->partid]; 7435b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson enum xp_retval ret; 7445b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 7455b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson ret = xpc_send_activate_IRQ_uv(part, msg, msg_size, msg_type); 7465b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson if (unlikely(ret != xpSuccess)) { 7475b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson if (irq_flags != NULL) 7485b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson spin_unlock_irqrestore(&ch->lock, *irq_flags); 7495b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 7505b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson XPC_DEACTIVATE_PARTITION(part, ret); 7515b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 7525b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson if (irq_flags != NULL) 7535b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson spin_lock_irqsave(&ch->lock, *irq_flags); 7545b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson } 7555b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 7565b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 7575b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic void 7585b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_send_local_activate_IRQ_uv(struct xpc_partition *part, int act_state_req) 7595b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 7605b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson unsigned long irq_flags; 7615b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson struct xpc_partition_uv *part_uv = &part->sn.uv; 7625b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 76333ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson /* 7647d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson * !!! Make our side think that the remote partition sent an activate 765a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt * !!! mq message our way by doing what the activate IRQ handler would 766ea57f80c8c0e59cfc5095f7e856ce7c8e6ac2984Dean Nelson * !!! do had one really been sent. 76733ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson */ 7685b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 7695b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); 7705b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson if (part_uv->act_state_req == 0) 7715b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson xpc_activate_IRQ_rcvd++; 7725b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson part_uv->act_state_req = act_state_req; 7735b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); 7745b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 7755b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson wake_up_interruptible(&xpc_activate_IRQ_wq); 77633ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson} 77733ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson 77894bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelsonstatic enum xp_retval 7795b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_get_partition_rsvd_page_pa_uv(void *buf, u64 *cookie, unsigned long *rp_pa, 7805b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson size_t *len) 78194bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson{ 7827d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson s64 status; 7837d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson enum xp_retval ret; 7847d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson 7857d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson#if defined CONFIG_X86_64 7867d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson status = uv_bios_reserved_page_pa((u64)buf, cookie, (u64 *)rp_pa, 7877d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson (u64 *)len); 7887d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson if (status == BIOS_STATUS_SUCCESS) 7897d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson ret = xpSuccess; 7907d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson else if (status == BIOS_STATUS_MORE_PASSES) 7917d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson ret = xpNeedMoreInfo; 7927d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson else 7937d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson ret = xpBiosError; 7947d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson 7957d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson#elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV 7967d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson status = sn_partition_reserved_page_pa((u64)buf, cookie, rp_pa, len); 7977d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson if (status == SALRET_OK) 7987d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson ret = xpSuccess; 7997d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson else if (status == SALRET_MORE_PASSES) 8007d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson ret = xpNeedMoreInfo; 8017d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson else 8027d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson ret = xpSalError; 8037d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson 8047d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson#else 8057d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson #error not a supported configuration 8067d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson#endif 8077d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson 8087d9d1f25c3872080ce599e5dd0dac3305d0a028bDean Nelson return ret; 8095b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 8105b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 8115b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic int 812a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holtxpc_setup_rsvd_page_uv(struct xpc_rsvd_page *rp) 8135b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 814a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt xpc_heartbeat_uv = 815a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt &xpc_partitions[sn_partition_id].sn.uv.cached_heartbeat; 816a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt rp->sn.uv.heartbeat_gpa = uv_gpa(xpc_heartbeat_uv); 817a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt rp->sn.uv.activate_gru_mq_desc_gpa = 8186f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner uv_gpa(xpc_activate_mq_uv->gru_mq_desc); 8195b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson return 0; 8205b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 8215b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 8225b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic void 823a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holtxpc_allow_hb_uv(short partid) 8245b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 825a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt} 8265b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 827a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holtstatic void 828a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holtxpc_disallow_hb_uv(short partid) 829a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt{ 830a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt} 8315b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 832a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holtstatic void 833a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holtxpc_disallow_all_hbs_uv(void) 834a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt{ 83594bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson} 83694bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson 83733ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelsonstatic void 83833ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelsonxpc_increment_heartbeat_uv(void) 83933ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson{ 840a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt xpc_heartbeat_uv->value++; 8415b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 8425b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 8435b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic void 8445b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_offline_heartbeat_uv(void) 8455b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 846a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt xpc_increment_heartbeat_uv(); 847a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt xpc_heartbeat_uv->offline = 1; 8485b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 8495b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 8505b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic void 8515b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_online_heartbeat_uv(void) 8525b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 853a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt xpc_increment_heartbeat_uv(); 854a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt xpc_heartbeat_uv->offline = 0; 85533ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson} 85633ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson 85733ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelsonstatic void 85833ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelsonxpc_heartbeat_init_uv(void) 85933ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson{ 860a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt xpc_heartbeat_uv->value = 1; 861a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt xpc_heartbeat_uv->offline = 0; 86233ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson} 86333ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson 86433ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelsonstatic void 86533ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelsonxpc_heartbeat_exit_uv(void) 86633ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson{ 867a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt xpc_offline_heartbeat_uv(); 8685b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 8695b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 8705b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic enum xp_retval 8715b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_get_remote_heartbeat_uv(struct xpc_partition *part) 8725b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 8735b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson struct xpc_partition_uv *part_uv = &part->sn.uv; 874a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt enum xp_retval ret; 8755b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 876a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt ret = xp_remote_memcpy(uv_gpa(&part_uv->cached_heartbeat), 877a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt part_uv->heartbeat_gpa, 878a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt sizeof(struct xpc_heartbeat_uv)); 879a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt if (ret != xpSuccess) 880a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt return ret; 8815b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 882a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt if (part_uv->cached_heartbeat.value == part->last_heartbeat && 883a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt !part_uv->cached_heartbeat.offline) { 8845b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 885a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt ret = xpNoHeartbeat; 886a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt } else { 887a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt part->last_heartbeat = part_uv->cached_heartbeat.value; 8885b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson } 8895b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson return ret; 89033ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson} 89133ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson 89233ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelsonstatic void 893a47d5dac9d8481766382f8cf1483dd581df38b99Dean Nelsonxpc_request_partition_activation_uv(struct xpc_rsvd_page *remote_rp, 8945b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson unsigned long remote_rp_gpa, int nasid) 89533ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson{ 89633ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson short partid = remote_rp->SAL_partid; 89733ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson struct xpc_partition *part = &xpc_partitions[partid]; 8985b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson struct xpc_activate_mq_msg_activate_req_uv msg; 89933ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson 9005b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson part->remote_rp_pa = remote_rp_gpa; /* !!! _pa here is really _gpa */ 9015b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson part->remote_rp_ts_jiffies = remote_rp->ts_jiffies; 902a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt part->sn.uv.heartbeat_gpa = remote_rp->sn.uv.heartbeat_gpa; 9036f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner part->sn.uv.activate_gru_mq_desc_gpa = 904a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt remote_rp->sn.uv.activate_gru_mq_desc_gpa; 9055b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 9065b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson /* 9075b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson * ??? Is it a good idea to make this conditional on what is 9085b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson * ??? potentially stale state information? 9095b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson */ 9105b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson if (part->sn.uv.remote_act_state == XPC_P_AS_INACTIVE) { 9115b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson msg.rp_gpa = uv_gpa(xpc_rsvd_page); 912a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt msg.heartbeat_gpa = xpc_rsvd_page->sn.uv.heartbeat_gpa; 9136f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner msg.activate_gru_mq_desc_gpa = 914a374c57b0764432a80303abee3d1afd1939b5a0aRobin Holt xpc_rsvd_page->sn.uv.activate_gru_mq_desc_gpa; 9155b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg), 9165b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson XPC_ACTIVATE_MQ_MSG_ACTIVATE_REQ_UV); 9175b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson } 91833ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson 9195b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson if (part->act_state == XPC_P_AS_INACTIVE) 9205b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson xpc_send_local_activate_IRQ_uv(part, XPC_P_ASR_ACTIVATE_UV); 92133ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson} 92233ba3c7724be79f7cdbfc611335572c056d9a05aDean Nelson 923a47d5dac9d8481766382f8cf1483dd581df38b99Dean Nelsonstatic void 924a47d5dac9d8481766382f8cf1483dd581df38b99Dean Nelsonxpc_request_partition_reactivation_uv(struct xpc_partition *part) 925a47d5dac9d8481766382f8cf1483dd581df38b99Dean Nelson{ 9265b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson xpc_send_local_activate_IRQ_uv(part, XPC_P_ASR_ACTIVATE_UV); 9275b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 9285b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 9295b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic void 9305b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_request_partition_deactivation_uv(struct xpc_partition *part) 9315b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 9325b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson struct xpc_activate_mq_msg_deactivate_req_uv msg; 9335b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 9345b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson /* 9355b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson * ??? Is it a good idea to make this conditional on what is 9365b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson * ??? potentially stale state information? 9375b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson */ 9385b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson if (part->sn.uv.remote_act_state != XPC_P_AS_DEACTIVATING && 9395b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson part->sn.uv.remote_act_state != XPC_P_AS_INACTIVE) { 9405b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 9415b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson msg.reason = part->reason; 9425b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg), 9435b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson XPC_ACTIVATE_MQ_MSG_DEACTIVATE_REQ_UV); 9445b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson } 945a47d5dac9d8481766382f8cf1483dd581df38b99Dean Nelson} 946a47d5dac9d8481766382f8cf1483dd581df38b99Dean Nelson 947bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonstatic void 948bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_cancel_partition_deactivation_request_uv(struct xpc_partition *part) 949bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson{ 950bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson /* nothing needs to be done */ 951bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return; 952bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson} 953bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 954bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonstatic void 955bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_init_fifo_uv(struct xpc_fifo_head_uv *head) 956bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson{ 957bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson head->first = NULL; 958bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson head->last = NULL; 959bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_lock_init(&head->lock); 960bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson head->n_entries = 0; 961bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson} 962bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 963bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonstatic void * 964bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_get_fifo_entry_uv(struct xpc_fifo_head_uv *head) 965bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson{ 966bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson unsigned long irq_flags; 967bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_fifo_entry_uv *first; 968bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 969bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_lock_irqsave(&head->lock, irq_flags); 970bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson first = head->first; 971bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (head->first != NULL) { 972bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson head->first = first->next; 973bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (head->first == NULL) 974bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson head->last = NULL; 97515b87d67ff3dc042bee42f991858d6b121b3b3caRobin Holt 97615b87d67ff3dc042bee42f991858d6b121b3b3caRobin Holt head->n_entries--; 97715b87d67ff3dc042bee42f991858d6b121b3b3caRobin Holt BUG_ON(head->n_entries < 0); 97815b87d67ff3dc042bee42f991858d6b121b3b3caRobin Holt 97915b87d67ff3dc042bee42f991858d6b121b3b3caRobin Holt first->next = NULL; 980bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 981bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_unlock_irqrestore(&head->lock, irq_flags); 982bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return first; 983bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson} 984bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 985bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonstatic void 986bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_put_fifo_entry_uv(struct xpc_fifo_head_uv *head, 987bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_fifo_entry_uv *last) 988bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson{ 989bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson unsigned long irq_flags; 990bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 991bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson last->next = NULL; 992bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_lock_irqsave(&head->lock, irq_flags); 993bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (head->last != NULL) 994bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson head->last->next = last; 995bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson else 996bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson head->first = last; 997bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson head->last = last; 9986f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner head->n_entries++; 999bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_unlock_irqrestore(&head->lock, irq_flags); 1000bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson} 1001bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1002bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonstatic int 1003bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_n_of_fifo_entries_uv(struct xpc_fifo_head_uv *head) 1004bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson{ 1005bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return head->n_entries; 1006bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson} 1007bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1008e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelson/* 10095b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson * Setup the channel structures that are uv specific. 1010e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelson */ 1011e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelsonstatic enum xp_retval 1012a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holtxpc_setup_ch_structures_uv(struct xpc_partition *part) 1013e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelson{ 1014bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_channel_uv *ch_uv; 1015bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson int ch_number; 1016bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1017bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson for (ch_number = 0; ch_number < part->nchannels; ch_number++) { 1018bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson ch_uv = &part->channels[ch_number].sn.uv; 1019bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1020bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_init_fifo_uv(&ch_uv->msg_slot_free_list); 1021bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_init_fifo_uv(&ch_uv->recv_msg_list); 1022bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1023bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1024bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return xpSuccess; 1025e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelson} 1026e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelson 1027e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelson/* 10285b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson * Teardown the channel structures that are uv specific. 1029e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelson */ 1030e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelsonstatic void 1031a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holtxpc_teardown_ch_structures_uv(struct xpc_partition *part) 1032e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelson{ 1033bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson /* nothing needs to be done */ 1034e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelson return; 1035e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelson} 1036e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelson 1037e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelsonstatic enum xp_retval 1038e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelsonxpc_make_first_contact_uv(struct xpc_partition *part) 1039e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelson{ 10405b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson struct xpc_activate_mq_msg_uv msg; 10415b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 10425b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson /* 10435b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson * We send a sync msg to get the remote partition's remote_act_state 10445b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson * updated to our current act_state which at this point should 10455b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson * be XPC_P_AS_ACTIVATING. 10465b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson */ 10475b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg), 10485b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson XPC_ACTIVATE_MQ_MSG_SYNC_ACT_STATE_UV); 10495b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 1050dbd2918ec65c35f36bb102c88eafe87be0552f6fRobin Holt while (!((part->sn.uv.remote_act_state == XPC_P_AS_ACTIVATING) || 1051dbd2918ec65c35f36bb102c88eafe87be0552f6fRobin Holt (part->sn.uv.remote_act_state == XPC_P_AS_ACTIVE))) { 10525b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 10535b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson dev_dbg(xpc_part, "waiting to make first contact with " 10545b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson "partition %d\n", XPC_PARTID(part)); 10555b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 10565b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson /* wait a 1/4 of a second or so */ 10575b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson (void)msleep_interruptible(250); 10585b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 10595b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson if (part->act_state == XPC_P_AS_DEACTIVATING) 10605b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson return part->reason; 10615b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson } 10625b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 10635b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson return xpSuccess; 1064e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelson} 1065e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelson 1066e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelsonstatic u64 10677fb5e59d63deda89a8eefdbd5b3c8d622076afd4Dean Nelsonxpc_get_chctl_all_flags_uv(struct xpc_partition *part) 1068e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelson{ 10695b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson unsigned long irq_flags; 10705b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson union xpc_channel_ctl_flags chctl; 10715b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 10725b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson spin_lock_irqsave(&part->chctl_lock, irq_flags); 10735b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson chctl = part->chctl; 10745b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson if (chctl.all_flags != 0) 10755b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson part->chctl.all_flags = 0; 10765b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 10775b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson spin_unlock_irqrestore(&part->chctl_lock, irq_flags); 10785b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson return chctl.all_flags; 10795b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 10805b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 10815b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic enum xp_retval 1082bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_allocate_send_msg_slot_uv(struct xpc_channel *ch) 1083bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson{ 1084bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_channel_uv *ch_uv = &ch->sn.uv; 1085bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_send_msg_slot_uv *msg_slot; 1086bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson unsigned long irq_flags; 1087bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson int nentries; 1088bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson int entry; 1089bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson size_t nbytes; 1090bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1091bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson for (nentries = ch->local_nentries; nentries > 0; nentries--) { 1092bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson nbytes = nentries * sizeof(struct xpc_send_msg_slot_uv); 1093bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson ch_uv->send_msg_slots = kzalloc(nbytes, GFP_KERNEL); 1094bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (ch_uv->send_msg_slots == NULL) 1095bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson continue; 1096bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1097bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson for (entry = 0; entry < nentries; entry++) { 1098bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg_slot = &ch_uv->send_msg_slots[entry]; 1099bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1100bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg_slot->msg_slot_number = entry; 1101bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_put_fifo_entry_uv(&ch_uv->msg_slot_free_list, 1102bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson &msg_slot->next); 1103bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1104bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1105bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_lock_irqsave(&ch->lock, irq_flags); 1106bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (nentries < ch->local_nentries) 1107bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson ch->local_nentries = nentries; 1108bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_unlock_irqrestore(&ch->lock, irq_flags); 1109bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return xpSuccess; 1110bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1111bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1112bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return xpNoMemory; 1113bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson} 1114bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1115bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonstatic enum xp_retval 1116bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_allocate_recv_msg_slot_uv(struct xpc_channel *ch) 1117bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson{ 1118bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_channel_uv *ch_uv = &ch->sn.uv; 1119bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_notify_mq_msg_uv *msg_slot; 1120bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson unsigned long irq_flags; 1121bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson int nentries; 1122bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson int entry; 1123bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson size_t nbytes; 1124bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1125bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson for (nentries = ch->remote_nentries; nentries > 0; nentries--) { 1126bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson nbytes = nentries * ch->entry_size; 1127bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson ch_uv->recv_msg_slots = kzalloc(nbytes, GFP_KERNEL); 1128bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (ch_uv->recv_msg_slots == NULL) 1129bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson continue; 1130bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1131bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson for (entry = 0; entry < nentries; entry++) { 1132361916a943cd9dbda1c0b00879d0225cc919d868Dean Nelson msg_slot = ch_uv->recv_msg_slots + 1133361916a943cd9dbda1c0b00879d0225cc919d868Dean Nelson entry * ch->entry_size; 1134bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1135bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg_slot->hdr.msg_slot_number = entry; 1136bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1137bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1138bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_lock_irqsave(&ch->lock, irq_flags); 1139bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (nentries < ch->remote_nentries) 1140bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson ch->remote_nentries = nentries; 1141bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_unlock_irqrestore(&ch->lock, irq_flags); 1142bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return xpSuccess; 1143bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1144bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1145bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return xpNoMemory; 1146bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson} 1147bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1148bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson/* 1149bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * Allocate msg_slots associated with the channel. 1150bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson */ 1151bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonstatic enum xp_retval 11525b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_setup_msg_structures_uv(struct xpc_channel *ch) 11535b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 1154bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson static enum xp_retval ret; 1155bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_channel_uv *ch_uv = &ch->sn.uv; 1156bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1157bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson DBUG_ON(ch->flags & XPC_C_SETUP); 1158bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 11596f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner ch_uv->cached_notify_gru_mq_desc = kmalloc(sizeof(struct 11606f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner gru_message_queue_desc), 11616f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner GFP_KERNEL); 11626f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner if (ch_uv->cached_notify_gru_mq_desc == NULL) 11636f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner return xpNoMemory; 11646f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner 1165bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson ret = xpc_allocate_send_msg_slot_uv(ch); 1166bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (ret == xpSuccess) { 1167bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1168bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson ret = xpc_allocate_recv_msg_slot_uv(ch); 1169bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (ret != xpSuccess) { 1170bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson kfree(ch_uv->send_msg_slots); 1171bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_init_fifo_uv(&ch_uv->msg_slot_free_list); 1172bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1173bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1174bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return ret; 11755b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 11765b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 1177bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson/* 1178bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * Free up msg_slots and clear other stuff that were setup for the specified 1179bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * channel. 1180bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson */ 11815b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic void 11825b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_teardown_msg_structures_uv(struct xpc_channel *ch) 11835b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 11845b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson struct xpc_channel_uv *ch_uv = &ch->sn.uv; 11855b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 1186bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson DBUG_ON(!spin_is_locked(&ch->lock)); 1187bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 11886f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner kfree(ch_uv->cached_notify_gru_mq_desc); 11896f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner ch_uv->cached_notify_gru_mq_desc = NULL; 11905b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 1191bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (ch->flags & XPC_C_SETUP) { 1192bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_init_fifo_uv(&ch_uv->msg_slot_free_list); 1193bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson kfree(ch_uv->send_msg_slots); 1194bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_init_fifo_uv(&ch_uv->recv_msg_list); 1195bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson kfree(ch_uv->recv_msg_slots); 1196bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 11975b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 11985b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 11995b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic void 12005b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_send_chctl_closerequest_uv(struct xpc_channel *ch, unsigned long *irq_flags) 12015b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 12025b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson struct xpc_activate_mq_msg_chctl_closerequest_uv msg; 12035b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 12045b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson msg.ch_number = ch->number; 12055b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson msg.reason = ch->reason; 12065b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg), 12075b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson XPC_ACTIVATE_MQ_MSG_CHCTL_CLOSEREQUEST_UV); 12085b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 12095b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 12105b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic void 12115b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_send_chctl_closereply_uv(struct xpc_channel *ch, unsigned long *irq_flags) 12125b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 12135b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson struct xpc_activate_mq_msg_chctl_closereply_uv msg; 12145b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 12155b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson msg.ch_number = ch->number; 12165b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg), 12175b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson XPC_ACTIVATE_MQ_MSG_CHCTL_CLOSEREPLY_UV); 12185b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 12195b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 12205b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic void 12215b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_send_chctl_openrequest_uv(struct xpc_channel *ch, unsigned long *irq_flags) 12225b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 12235b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson struct xpc_activate_mq_msg_chctl_openrequest_uv msg; 12245b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 12255b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson msg.ch_number = ch->number; 1226bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg.entry_size = ch->entry_size; 12275b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson msg.local_nentries = ch->local_nentries; 12285b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg), 12295b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson XPC_ACTIVATE_MQ_MSG_CHCTL_OPENREQUEST_UV); 12305b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 12315b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 12325b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic void 12335b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_send_chctl_openreply_uv(struct xpc_channel *ch, unsigned long *irq_flags) 12345b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 12355b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson struct xpc_activate_mq_msg_chctl_openreply_uv msg; 12365b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 12375b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson msg.ch_number = ch->number; 12385b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson msg.local_nentries = ch->local_nentries; 12395b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson msg.remote_nentries = ch->remote_nentries; 12406f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner msg.notify_gru_mq_desc_gpa = uv_gpa(xpc_notify_mq_uv->gru_mq_desc); 12415b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg), 12425b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson XPC_ACTIVATE_MQ_MSG_CHCTL_OPENREPLY_UV); 12435b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 12445b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 12455b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic void 1246efdd06ed181a88a11e612238c1ac04668e665395Robin Holtxpc_send_chctl_opencomplete_uv(struct xpc_channel *ch, unsigned long *irq_flags) 1247efdd06ed181a88a11e612238c1ac04668e665395Robin Holt{ 1248efdd06ed181a88a11e612238c1ac04668e665395Robin Holt struct xpc_activate_mq_msg_chctl_opencomplete_uv msg; 1249efdd06ed181a88a11e612238c1ac04668e665395Robin Holt 1250efdd06ed181a88a11e612238c1ac04668e665395Robin Holt msg.ch_number = ch->number; 1251efdd06ed181a88a11e612238c1ac04668e665395Robin Holt xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg), 1252efdd06ed181a88a11e612238c1ac04668e665395Robin Holt XPC_ACTIVATE_MQ_MSG_CHCTL_OPENCOMPLETE_UV); 1253efdd06ed181a88a11e612238c1ac04668e665395Robin Holt} 1254efdd06ed181a88a11e612238c1ac04668e665395Robin Holt 1255efdd06ed181a88a11e612238c1ac04668e665395Robin Holtstatic void 1256bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_send_chctl_local_msgrequest_uv(struct xpc_partition *part, int ch_number) 1257bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson{ 1258bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson unsigned long irq_flags; 1259bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1260bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_lock_irqsave(&part->chctl_lock, irq_flags); 1261bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson part->chctl.flags[ch_number] |= XPC_CHCTL_MSGREQUEST; 1262bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_unlock_irqrestore(&part->chctl_lock, irq_flags); 1263bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1264bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_wakeup_channel_mgr(part); 1265bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson} 1266bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 12676f2584f47474d29ce829604bfc8b56c10b352fdbJack Steinerstatic enum xp_retval 12685b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_save_remote_msgqueue_pa_uv(struct xpc_channel *ch, 12696f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner unsigned long gru_mq_desc_gpa) 12705b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 12716f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner struct xpc_channel_uv *ch_uv = &ch->sn.uv; 12726f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner 12736f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner DBUG_ON(ch_uv->cached_notify_gru_mq_desc == NULL); 12746f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner return xpc_cache_remote_gru_mq_desc_uv(ch_uv->cached_notify_gru_mq_desc, 12756f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner gru_mq_desc_gpa); 12765b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 12775b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 12785b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic void 12795b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_indicate_partition_engaged_uv(struct xpc_partition *part) 12805b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 12815b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson struct xpc_activate_mq_msg_uv msg; 12825b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 12835b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg), 12845b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson XPC_ACTIVATE_MQ_MSG_MARK_ENGAGED_UV); 12855b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 12865b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 12875b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic void 12885b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_indicate_partition_disengaged_uv(struct xpc_partition *part) 12895b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 12905b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson struct xpc_activate_mq_msg_uv msg; 12915b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 12925b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg), 12935b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson XPC_ACTIVATE_MQ_MSG_MARK_DISENGAGED_UV); 12945b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 12955b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 12965b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic void 12975b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_assume_partition_disengaged_uv(short partid) 12985b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 12995b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson struct xpc_partition_uv *part_uv = &xpc_partitions[partid].sn.uv; 13005b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson unsigned long irq_flags; 13015b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 13025b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson spin_lock_irqsave(&part_uv->flags_lock, irq_flags); 13035b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson part_uv->flags &= ~XPC_P_ENGAGED_UV; 13045b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags); 13055b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 13065b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 13075b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic int 13085b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_partition_engaged_uv(short partid) 13095b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 13105b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson return (xpc_partitions[partid].sn.uv.flags & XPC_P_ENGAGED_UV) != 0; 13115b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson} 13125b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 13135b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonstatic int 13145b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonxpc_any_partition_engaged_uv(void) 13155b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson{ 13165b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson struct xpc_partition_uv *part_uv; 13175b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson short partid; 13185b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 13195b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson for (partid = 0; partid < XP_MAX_NPARTITIONS_UV; partid++) { 13205b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson part_uv = &xpc_partitions[partid].sn.uv; 13215b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson if ((part_uv->flags & XPC_P_ENGAGED_UV) != 0) 13225b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson return 1; 13235b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson } 13245b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson return 0; 1325e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelson} 1326e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelson 1327bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonstatic enum xp_retval 1328bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_allocate_msg_slot_uv(struct xpc_channel *ch, u32 flags, 1329bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_send_msg_slot_uv **address_of_msg_slot) 1330bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson{ 1331bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson enum xp_retval ret; 1332bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_send_msg_slot_uv *msg_slot; 1333bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_fifo_entry_uv *entry; 1334bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1335bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson while (1) { 1336bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson entry = xpc_get_fifo_entry_uv(&ch->sn.uv.msg_slot_free_list); 1337bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (entry != NULL) 1338bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson break; 1339bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1340bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (flags & XPC_NOWAIT) 1341bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return xpNoWait; 1342bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1343bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson ret = xpc_allocate_msg_wait(ch); 1344bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (ret != xpInterrupted && ret != xpTimeout) 1345bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return ret; 1346bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1347bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1348bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg_slot = container_of(entry, struct xpc_send_msg_slot_uv, next); 1349bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson *address_of_msg_slot = msg_slot; 1350bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return xpSuccess; 1351bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson} 1352bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1353bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonstatic void 1354bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_free_msg_slot_uv(struct xpc_channel *ch, 1355bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_send_msg_slot_uv *msg_slot) 1356bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson{ 1357bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_put_fifo_entry_uv(&ch->sn.uv.msg_slot_free_list, &msg_slot->next); 1358bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1359bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson /* wakeup anyone waiting for a free msg slot */ 1360bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (atomic_read(&ch->n_on_msg_allocate_wq) > 0) 1361bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson wake_up(&ch->msg_allocate_wq); 1362bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson} 1363bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1364bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonstatic void 1365bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_notify_sender_uv(struct xpc_channel *ch, 1366bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_send_msg_slot_uv *msg_slot, 1367bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson enum xp_retval reason) 1368bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson{ 1369bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_notify_func func = msg_slot->func; 1370bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1371bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (func != NULL && cmpxchg(&msg_slot->func, func, NULL) == func) { 1372bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1373bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson atomic_dec(&ch->n_to_notify); 1374bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1375bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson dev_dbg(xpc_chan, "msg_slot->func() called, msg_slot=0x%p " 1376bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson "msg_slot_number=%d partid=%d channel=%d\n", msg_slot, 1377bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg_slot->msg_slot_number, ch->partid, ch->number); 1378bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1379bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson func(reason, ch->partid, ch->number, msg_slot->key); 1380bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1381bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson dev_dbg(xpc_chan, "msg_slot->func() returned, msg_slot=0x%p " 1382bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson "msg_slot_number=%d partid=%d channel=%d\n", msg_slot, 1383bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg_slot->msg_slot_number, ch->partid, ch->number); 1384bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1385bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson} 1386bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1387bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonstatic void 1388bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_handle_notify_mq_ack_uv(struct xpc_channel *ch, 1389bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_notify_mq_msg_uv *msg) 1390bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson{ 1391bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_send_msg_slot_uv *msg_slot; 1392bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson int entry = msg->hdr.msg_slot_number % ch->local_nentries; 1393bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1394bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg_slot = &ch->sn.uv.send_msg_slots[entry]; 1395bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1396bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson BUG_ON(msg_slot->msg_slot_number != msg->hdr.msg_slot_number); 1397bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg_slot->msg_slot_number += ch->local_nentries; 1398bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1399bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (msg_slot->func != NULL) 1400bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_notify_sender_uv(ch, msg_slot, xpMsgDelivered); 1401bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1402bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_free_msg_slot_uv(ch, msg_slot); 1403bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson} 1404bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1405bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonstatic void 1406bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_handle_notify_mq_msg_uv(struct xpc_partition *part, 1407bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_notify_mq_msg_uv *msg) 1408bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson{ 1409bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_partition_uv *part_uv = &part->sn.uv; 1410bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_channel *ch; 1411bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_channel_uv *ch_uv; 1412bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_notify_mq_msg_uv *msg_slot; 1413bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson unsigned long irq_flags; 1414bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson int ch_number = msg->hdr.ch_number; 1415bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1416bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (unlikely(ch_number >= part->nchannels)) { 1417bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson dev_err(xpc_part, "xpc_handle_notify_IRQ_uv() received invalid " 1418bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson "channel number=0x%x in message from partid=%d\n", 1419bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson ch_number, XPC_PARTID(part)); 1420bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1421bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson /* get hb checker to deactivate from the remote partition */ 1422bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); 1423bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (part_uv->act_state_req == 0) 1424bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_activate_IRQ_rcvd++; 1425bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson part_uv->act_state_req = XPC_P_ASR_DEACTIVATE_UV; 1426bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson part_uv->reason = xpBadChannelNumber; 1427bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); 1428bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1429bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson wake_up_interruptible(&xpc_activate_IRQ_wq); 1430bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return; 1431bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1432bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1433bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson ch = &part->channels[ch_number]; 1434bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_msgqueue_ref(ch); 1435bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1436bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (!(ch->flags & XPC_C_CONNECTED)) { 1437bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_msgqueue_deref(ch); 1438bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return; 1439bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1440bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1441bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson /* see if we're really dealing with an ACK for a previously sent msg */ 1442bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (msg->hdr.size == 0) { 1443bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_handle_notify_mq_ack_uv(ch, msg); 1444bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_msgqueue_deref(ch); 1445bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return; 1446bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1447bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1448bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson /* we're dealing with a normal message sent via the notify_mq */ 1449bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson ch_uv = &ch->sn.uv; 1450bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1451361916a943cd9dbda1c0b00879d0225cc919d868Dean Nelson msg_slot = ch_uv->recv_msg_slots + 1452361916a943cd9dbda1c0b00879d0225cc919d868Dean Nelson (msg->hdr.msg_slot_number % ch->remote_nentries) * ch->entry_size; 1453bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1454bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson BUG_ON(msg_slot->hdr.size != 0); 1455bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1456bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson memcpy(msg_slot, msg, msg->hdr.size); 1457bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1458bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_put_fifo_entry_uv(&ch_uv->recv_msg_list, &msg_slot->hdr.u.next); 1459bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1460bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE) { 1461bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson /* 1462bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * If there is an existing idle kthread get it to deliver 1463bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * the payload, otherwise we'll have to get the channel mgr 1464bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * for this partition to create a kthread to do the delivery. 1465bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson */ 1466bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (atomic_read(&ch->kthreads_idle) > 0) 1467bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson wake_up_nr(&ch->idle_wq, 1); 1468bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson else 1469bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_send_chctl_local_msgrequest_uv(part, ch->number); 1470bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1471bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_msgqueue_deref(ch); 1472bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson} 1473bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1474bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonstatic irqreturn_t 1475bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_handle_notify_IRQ_uv(int irq, void *dev_id) 1476bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson{ 1477bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_notify_mq_msg_uv *msg; 1478bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson short partid; 1479bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_partition *part; 1480bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 14816f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner while ((msg = gru_get_next_message(xpc_notify_mq_uv->gru_mq_desc)) != 14826f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner NULL) { 1483bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1484bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson partid = msg->hdr.partid; 1485bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (partid < 0 || partid >= XP_MAX_NPARTITIONS_UV) { 1486bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson dev_err(xpc_part, "xpc_handle_notify_IRQ_uv() received " 1487bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson "invalid partid=0x%x in message\n", partid); 1488bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } else { 1489bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson part = &xpc_partitions[partid]; 1490bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1491bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (xpc_part_ref(part)) { 1492bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_handle_notify_mq_msg_uv(part, msg); 1493bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_part_deref(part); 1494bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1495bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1496bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 14976f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner gru_free_message(xpc_notify_mq_uv->gru_mq_desc, msg); 1498bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1499bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1500bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return IRQ_HANDLED; 1501bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson} 1502bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1503bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonstatic int 1504bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_n_of_deliverable_payloads_uv(struct xpc_channel *ch) 1505bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson{ 1506bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return xpc_n_of_fifo_entries_uv(&ch->sn.uv.recv_msg_list); 1507bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson} 1508bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1509bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonstatic void 1510bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_process_msg_chctl_flags_uv(struct xpc_partition *part, int ch_number) 1511bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson{ 1512bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_channel *ch = &part->channels[ch_number]; 1513bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson int ndeliverable_payloads; 1514bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1515bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_msgqueue_ref(ch); 1516bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1517bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson ndeliverable_payloads = xpc_n_of_deliverable_payloads_uv(ch); 1518bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1519bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (ndeliverable_payloads > 0 && 1520bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson (ch->flags & XPC_C_CONNECTED) && 1521bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE)) { 1522bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1523bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_activate_kthreads(ch, ndeliverable_payloads); 1524bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1525bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1526bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_msgqueue_deref(ch); 1527bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson} 1528bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1529bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonstatic enum xp_retval 1530bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_send_payload_uv(struct xpc_channel *ch, u32 flags, void *payload, 1531bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson u16 payload_size, u8 notify_type, xpc_notify_func func, 1532bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson void *key) 1533bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson{ 1534bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson enum xp_retval ret = xpSuccess; 1535bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_send_msg_slot_uv *msg_slot = NULL; 1536bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_notify_mq_msg_uv *msg; 1537bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson u8 msg_buffer[XPC_NOTIFY_MSG_SIZE_UV]; 1538bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson size_t msg_size; 1539bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1540bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson DBUG_ON(notify_type != XPC_N_CALL); 1541bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1542bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg_size = sizeof(struct xpc_notify_mq_msghdr_uv) + payload_size; 1543bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (msg_size > ch->entry_size) 1544bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return xpPayloadTooBig; 1545bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1546bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_msgqueue_ref(ch); 1547bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1548bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (ch->flags & XPC_C_DISCONNECTING) { 1549bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson ret = ch->reason; 1550bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson goto out_1; 1551bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1552bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (!(ch->flags & XPC_C_CONNECTED)) { 1553bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson ret = xpNotConnected; 1554bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson goto out_1; 1555bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1556bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1557bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson ret = xpc_allocate_msg_slot_uv(ch, flags, &msg_slot); 1558bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (ret != xpSuccess) 1559bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson goto out_1; 1560bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1561bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (func != NULL) { 1562bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson atomic_inc(&ch->n_to_notify); 1563bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1564bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg_slot->key = key; 156569b3bb65fa97a1e8563518dbbc35cd57beefb2d4Robin Holt smp_wmb(); /* a non-NULL func must hit memory after the key */ 1566bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg_slot->func = func; 1567bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1568bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (ch->flags & XPC_C_DISCONNECTING) { 1569bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson ret = ch->reason; 1570bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson goto out_2; 1571bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1572bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1573bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1574bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg = (struct xpc_notify_mq_msg_uv *)&msg_buffer; 1575bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg->hdr.partid = xp_partition_id; 1576bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg->hdr.ch_number = ch->number; 1577bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg->hdr.size = msg_size; 1578bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg->hdr.msg_slot_number = msg_slot->msg_slot_number; 1579bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson memcpy(&msg->payload, payload, payload_size); 1580bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 15816f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner ret = xpc_send_gru_msg(ch->sn.uv.cached_notify_gru_mq_desc, msg, 15826f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner msg_size); 1583bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (ret == xpSuccess) 1584bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson goto out_1; 1585bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1586bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson XPC_DEACTIVATE_PARTITION(&xpc_partitions[ch->partid], ret); 1587bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonout_2: 1588bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (func != NULL) { 1589bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson /* 1590bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * Try to NULL the msg_slot's func field. If we fail, then 1591bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * xpc_notify_senders_of_disconnect_uv() beat us to it, in which 1592bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * case we need to pretend we succeeded to send the message 1593bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * since the user will get a callout for the disconnect error 1594bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * by xpc_notify_senders_of_disconnect_uv(), and to also get an 1595bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * error returned here will confuse them. Additionally, since 1596bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * in this case the channel is being disconnected we don't need 1597bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * to put the the msg_slot back on the free list. 1598bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson */ 1599bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (cmpxchg(&msg_slot->func, func, NULL) != func) { 1600bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson ret = xpSuccess; 1601bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson goto out_1; 1602bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1603bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1604bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg_slot->key = NULL; 1605bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson atomic_dec(&ch->n_to_notify); 1606bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1607bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_free_msg_slot_uv(ch, msg_slot); 1608bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonout_1: 1609bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_msgqueue_deref(ch); 1610bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return ret; 1611bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson} 1612bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1613bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson/* 1614bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * Tell the callers of xpc_send_notify() that the status of their payloads 1615bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * is unknown because the channel is now disconnecting. 1616bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * 1617bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * We don't worry about putting these msg_slots on the free list since the 1618bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * msg_slots themselves are about to be kfree'd. 1619bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson */ 1620bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonstatic void 1621bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_notify_senders_of_disconnect_uv(struct xpc_channel *ch) 1622bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson{ 1623bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_send_msg_slot_uv *msg_slot; 1624bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson int entry; 1625bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1626bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson DBUG_ON(!(ch->flags & XPC_C_DISCONNECTING)); 1627bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1628bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson for (entry = 0; entry < ch->local_nentries; entry++) { 1629bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1630bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (atomic_read(&ch->n_to_notify) == 0) 1631bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson break; 1632bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1633bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg_slot = &ch->sn.uv.send_msg_slots[entry]; 1634bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (msg_slot->func != NULL) 1635bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson xpc_notify_sender_uv(ch, msg_slot, ch->reason); 1636bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1637bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson} 1638bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1639bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson/* 1640bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson * Get the next deliverable message's payload. 1641bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson */ 1642bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonstatic void * 1643bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_get_deliverable_payload_uv(struct xpc_channel *ch) 1644bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson{ 1645bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_fifo_entry_uv *entry; 1646bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_notify_mq_msg_uv *msg; 1647bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson void *payload = NULL; 1648bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1649bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (!(ch->flags & XPC_C_DISCONNECTING)) { 1650bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson entry = xpc_get_fifo_entry_uv(&ch->sn.uv.recv_msg_list); 1651bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (entry != NULL) { 1652bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg = container_of(entry, struct xpc_notify_mq_msg_uv, 1653bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson hdr.u.next); 1654bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson payload = &msg->payload; 1655bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1656bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 1657bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return payload; 1658bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson} 1659bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1660bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonstatic void 1661bd3e64c1759e4930315ebf022611468ee9621486Dean Nelsonxpc_received_payload_uv(struct xpc_channel *ch, void *payload) 1662e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelson{ 1663bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson struct xpc_notify_mq_msg_uv *msg; 1664bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson enum xp_retval ret; 1665bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1666bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg = container_of(payload, struct xpc_notify_mq_msg_uv, payload); 1667bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1668bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson /* return an ACK to the sender of this message */ 1669bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1670bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg->hdr.partid = xp_partition_id; 1671bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson msg->hdr.size = 0; /* size of zero indicates this is an ACK */ 1672bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 16736f2584f47474d29ce829604bfc8b56c10b352fdbJack Steiner ret = xpc_send_gru_msg(ch->sn.uv.cached_notify_gru_mq_desc, msg, 1674bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson sizeof(struct xpc_notify_mq_msghdr_uv)); 1675bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (ret != xpSuccess) 1676bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson XPC_DEACTIVATE_PARTITION(&xpc_partitions[ch->partid], ret); 1677e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelson} 1678e17d416b1bc947df68499863f13b401fb42b48f6Dean Nelson 1679a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holtstatic struct xpc_arch_operations xpc_arch_ops_uv = { 1680a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .setup_partitions = xpc_setup_partitions_uv, 1681a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .teardown_partitions = xpc_teardown_partitions_uv, 1682a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_uv, 1683a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_uv, 1684a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .setup_rsvd_page = xpc_setup_rsvd_page_uv, 1685a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt 1686a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .allow_hb = xpc_allow_hb_uv, 1687a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .disallow_hb = xpc_disallow_hb_uv, 1688a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .disallow_all_hbs = xpc_disallow_all_hbs_uv, 1689a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .increment_heartbeat = xpc_increment_heartbeat_uv, 1690a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .offline_heartbeat = xpc_offline_heartbeat_uv, 1691a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .online_heartbeat = xpc_online_heartbeat_uv, 1692a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .heartbeat_init = xpc_heartbeat_init_uv, 1693a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .heartbeat_exit = xpc_heartbeat_exit_uv, 1694a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .get_remote_heartbeat = xpc_get_remote_heartbeat_uv, 1695a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt 1696a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .request_partition_activation = 1697a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt xpc_request_partition_activation_uv, 1698a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .request_partition_reactivation = 1699a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt xpc_request_partition_reactivation_uv, 1700a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .request_partition_deactivation = 1701a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt xpc_request_partition_deactivation_uv, 1702a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .cancel_partition_deactivation_request = 1703a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt xpc_cancel_partition_deactivation_request_uv, 1704a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt 1705a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .setup_ch_structures = xpc_setup_ch_structures_uv, 1706a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .teardown_ch_structures = xpc_teardown_ch_structures_uv, 1707a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt 1708a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .make_first_contact = xpc_make_first_contact_uv, 1709a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt 1710a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .get_chctl_all_flags = xpc_get_chctl_all_flags_uv, 1711a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .send_chctl_closerequest = xpc_send_chctl_closerequest_uv, 1712a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .send_chctl_closereply = xpc_send_chctl_closereply_uv, 1713a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .send_chctl_openrequest = xpc_send_chctl_openrequest_uv, 1714a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .send_chctl_openreply = xpc_send_chctl_openreply_uv, 1715a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .send_chctl_opencomplete = xpc_send_chctl_opencomplete_uv, 1716a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .process_msg_chctl_flags = xpc_process_msg_chctl_flags_uv, 1717a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt 1718a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .save_remote_msgqueue_pa = xpc_save_remote_msgqueue_pa_uv, 1719a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt 1720a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .setup_msg_structures = xpc_setup_msg_structures_uv, 1721a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .teardown_msg_structures = xpc_teardown_msg_structures_uv, 1722a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt 1723a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .indicate_partition_engaged = xpc_indicate_partition_engaged_uv, 1724a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .indicate_partition_disengaged = xpc_indicate_partition_disengaged_uv, 1725a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .assume_partition_disengaged = xpc_assume_partition_disengaged_uv, 1726a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .partition_engaged = xpc_partition_engaged_uv, 1727a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .any_partition_engaged = xpc_any_partition_engaged_uv, 1728a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt 1729a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .n_of_deliverable_payloads = xpc_n_of_deliverable_payloads_uv, 1730a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .send_payload = xpc_send_payload_uv, 1731a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .get_deliverable_payload = xpc_get_deliverable_payload_uv, 1732a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .received_payload = xpc_received_payload_uv, 1733a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt .notify_senders_of_disconnect = xpc_notify_senders_of_disconnect_uv, 1734a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt}; 1735a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt 17367838f994b4fceff24c343f4e26a6cf4393869579Robin Holtstatic int 17377838f994b4fceff24c343f4e26a6cf4393869579Robin Holtxpc_init_mq_node(int nid) 17387838f994b4fceff24c343f4e26a6cf4393869579Robin Holt{ 17397838f994b4fceff24c343f4e26a6cf4393869579Robin Holt int cpu; 17407838f994b4fceff24c343f4e26a6cf4393869579Robin Holt 17417838f994b4fceff24c343f4e26a6cf4393869579Robin Holt get_online_cpus(); 17427838f994b4fceff24c343f4e26a6cf4393869579Robin Holt 17437838f994b4fceff24c343f4e26a6cf4393869579Robin Holt for_each_cpu(cpu, cpumask_of_node(nid)) { 17447838f994b4fceff24c343f4e26a6cf4393869579Robin Holt xpc_activate_mq_uv = 17457838f994b4fceff24c343f4e26a6cf4393869579Robin Holt xpc_create_gru_mq_uv(XPC_ACTIVATE_MQ_SIZE_UV, nid, 17467838f994b4fceff24c343f4e26a6cf4393869579Robin Holt XPC_ACTIVATE_IRQ_NAME, 17477838f994b4fceff24c343f4e26a6cf4393869579Robin Holt xpc_handle_activate_IRQ_uv); 17487838f994b4fceff24c343f4e26a6cf4393869579Robin Holt if (!IS_ERR(xpc_activate_mq_uv)) 17497838f994b4fceff24c343f4e26a6cf4393869579Robin Holt break; 17507838f994b4fceff24c343f4e26a6cf4393869579Robin Holt } 17517838f994b4fceff24c343f4e26a6cf4393869579Robin Holt if (IS_ERR(xpc_activate_mq_uv)) { 17527838f994b4fceff24c343f4e26a6cf4393869579Robin Holt put_online_cpus(); 17537838f994b4fceff24c343f4e26a6cf4393869579Robin Holt return PTR_ERR(xpc_activate_mq_uv); 17547838f994b4fceff24c343f4e26a6cf4393869579Robin Holt } 17557838f994b4fceff24c343f4e26a6cf4393869579Robin Holt 17567838f994b4fceff24c343f4e26a6cf4393869579Robin Holt for_each_cpu(cpu, cpumask_of_node(nid)) { 17577838f994b4fceff24c343f4e26a6cf4393869579Robin Holt xpc_notify_mq_uv = 17587838f994b4fceff24c343f4e26a6cf4393869579Robin Holt xpc_create_gru_mq_uv(XPC_NOTIFY_MQ_SIZE_UV, nid, 17597838f994b4fceff24c343f4e26a6cf4393869579Robin Holt XPC_NOTIFY_IRQ_NAME, 17607838f994b4fceff24c343f4e26a6cf4393869579Robin Holt xpc_handle_notify_IRQ_uv); 17617838f994b4fceff24c343f4e26a6cf4393869579Robin Holt if (!IS_ERR(xpc_notify_mq_uv)) 17627838f994b4fceff24c343f4e26a6cf4393869579Robin Holt break; 17637838f994b4fceff24c343f4e26a6cf4393869579Robin Holt } 17647838f994b4fceff24c343f4e26a6cf4393869579Robin Holt if (IS_ERR(xpc_notify_mq_uv)) { 17657838f994b4fceff24c343f4e26a6cf4393869579Robin Holt xpc_destroy_gru_mq_uv(xpc_activate_mq_uv); 17667838f994b4fceff24c343f4e26a6cf4393869579Robin Holt put_online_cpus(); 17677838f994b4fceff24c343f4e26a6cf4393869579Robin Holt return PTR_ERR(xpc_notify_mq_uv); 17687838f994b4fceff24c343f4e26a6cf4393869579Robin Holt } 17697838f994b4fceff24c343f4e26a6cf4393869579Robin Holt 17707838f994b4fceff24c343f4e26a6cf4393869579Robin Holt put_online_cpus(); 17717838f994b4fceff24c343f4e26a6cf4393869579Robin Holt return 0; 17727838f994b4fceff24c343f4e26a6cf4393869579Robin Holt} 17737838f994b4fceff24c343f4e26a6cf4393869579Robin Holt 17745b8669dfd110a62a74eea525a009342f73987ea0Dean Nelsonint 177594bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelsonxpc_init_uv(void) 177694bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson{ 17777838f994b4fceff24c343f4e26a6cf4393869579Robin Holt int nid; 17787838f994b4fceff24c343f4e26a6cf4393869579Robin Holt int ret = 0; 17797838f994b4fceff24c343f4e26a6cf4393869579Robin Holt 1780a7665b0a380585fbd70a2275f3120c6086e0c92dRobin Holt xpc_arch_ops = xpc_arch_ops_uv; 1781bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 1782bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson if (sizeof(struct xpc_notify_mq_msghdr_uv) > XPC_MSG_HDR_MAX_SIZE) { 1783bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson dev_err(xpc_part, "xpc_notify_mq_msghdr_uv is larger than %d\n", 1784bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson XPC_MSG_HDR_MAX_SIZE); 1785bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson return -E2BIG; 1786bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson } 17875b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 17887838f994b4fceff24c343f4e26a6cf4393869579Robin Holt if (xpc_mq_node < 0) 17897838f994b4fceff24c343f4e26a6cf4393869579Robin Holt for_each_online_node(nid) { 17907838f994b4fceff24c343f4e26a6cf4393869579Robin Holt ret = xpc_init_mq_node(nid); 17915b8669dfd110a62a74eea525a009342f73987ea0Dean Nelson 17927838f994b4fceff24c343f4e26a6cf4393869579Robin Holt if (!ret) 17937838f994b4fceff24c343f4e26a6cf4393869579Robin Holt break; 17947838f994b4fceff24c343f4e26a6cf4393869579Robin Holt } 17957838f994b4fceff24c343f4e26a6cf4393869579Robin Holt else 17967838f994b4fceff24c343f4e26a6cf4393869579Robin Holt ret = xpc_init_mq_node(xpc_mq_node); 1797bd3e64c1759e4930315ebf022611468ee9621486Dean Nelson 17987838f994b4fceff24c343f4e26a6cf4393869579Robin Holt if (ret < 0) 17997838f994b4fceff24c343f4e26a6cf4393869579Robin Holt dev_err(xpc_part, "xpc_init_mq_node() returned error=%d\n", 18007838f994b4fceff24c343f4e26a6cf4393869579Robin Holt -ret); 18017838f994b4fceff24c343f4e26a6cf4393869579Robin Holt 18027838f994b4fceff24c343f4e26a6cf4393869579Robin Holt return ret; 180394bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson} 180494bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson 180594bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelsonvoid 180694bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelsonxpc_exit_uv(void) 180794bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson{ 18082525789b4694d78df4f001063f042b2b74227d26Dean Nelson xpc_destroy_gru_mq_uv(xpc_notify_mq_uv); 18092525789b4694d78df4f001063f042b2b74227d26Dean Nelson xpc_destroy_gru_mq_uv(xpc_activate_mq_uv); 181094bd2708d4a95d7da5a1c7c28a063eccd127fb69Dean Nelson} 18117838f994b4fceff24c343f4e26a6cf4393869579Robin Holt 18127838f994b4fceff24c343f4e26a6cf4393869579Robin Holtmodule_param(xpc_mq_node, int, 0); 18137838f994b4fceff24c343f4e26a6cf4393869579Robin HoltMODULE_PARM_DESC(xpc_mq_node, "Node number on which to allocate message queues."); 1814