16e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins/* 2391e43da797a96aeb65410281891f6d0b0e9611cPeter Zijlstra * kernel/sched/cpupri.c 36e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * 46e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * CPU priority management 56e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * 66e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * Copyright (C) 2007-2008 Novell 76e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * 86e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * Author: Gregory Haskins <ghaskins@novell.com> 96e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * 106e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * This code tracks the priority of each CPU so that global migration 116e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * decisions are easy to calculate. Each CPU can be in a state as follows: 126e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * 136e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * (INVALID), IDLE, NORMAL, RT1, ... RT99 146e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * 156e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * going from the lowest priority to the highest. CPUs in the INVALID state 166e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * are not eligible for routing. The system maintains this state with 176e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * a 2 dimensional bitmap (the first for priority class, the second for cpus 186e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * in that class). Therefore a typical application without affinity 196e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * restrictions can find a suitable CPU with O(1) complexity (e.g. two bit 206e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * searches). For tasks with affinity restrictions, the algorithm has a 216e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * worst case complexity of O(min(102, nr_domcpus)), though the scenario that 226e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * yields the worst case search is fairly contrived. 236e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * 246e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * This program is free software; you can redistribute it and/or 256e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * modify it under the terms of the GNU General Public License 266e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * as published by the Free Software Foundation; version 2 276e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * of the License. 286e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins */ 296e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 305a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/gfp.h> 318bd75c77b7c6a3954140dd2e20346aef3efe4a35Clark Williams#include <linux/sched.h> 328bd75c77b7c6a3954140dd2e20346aef3efe4a35Clark Williams#include <linux/sched/rt.h> 334dac0b638310d2e92f6e19958b73d4c97c9734bbPeter Zijlstra#include <linux/slab.h> 34391e43da797a96aeb65410281891f6d0b0e9611cPeter Zijlstra#include "cpupri.h" 356e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 366e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins/* Convert between a 140 based task->prio, and our 102 based cpupri */ 376e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskinsstatic int convert_prio(int prio) 386e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins{ 396e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins int cpupri; 406e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 416e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins if (prio == CPUPRI_INVALID) 426e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins cpupri = CPUPRI_INVALID; 436e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins else if (prio == MAX_PRIO) 446e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins cpupri = CPUPRI_IDLE; 456e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins else if (prio >= MAX_RT_PRIO) 466e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins cpupri = CPUPRI_NORMAL; 476e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins else 486e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins cpupri = MAX_RT_PRIO - prio + 1; 496e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 506e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins return cpupri; 516e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins} 526e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 536e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins/** 546e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * cpupri_find - find the best (lowest-pri) CPU in the system 556e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * @cp: The cpupri context 566e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * @p: The task 5713b8bd0a5713bdf05659019badd7c0407984ece1Rusty Russell * @lowest_mask: A mask to fill in with selected CPUs (or NULL) 586e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * 596e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * Note: This function returns the recommended CPUs as calculated during the 602a61aa401638529cd4231f6106980d307fba98faAdam Buchbinder * current invocation. By the time the call returns, the CPUs may have in 616e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * fact changed priorities any number of times. While not ideal, it is not 626e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * an issue of correctness since the normal rebalancer logic will correct 636e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * any discrepancies created by racing against the uncertainty of the current 646e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * priority configuration. 656e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * 66e69f61862ab833e9b8d3c15b6ce07fd69f3bfeccYacine Belkadi * Return: (int)bool - CPUs were found 676e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins */ 686e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskinsint cpupri_find(struct cpupri *cp, struct task_struct *p, 6968e74568fbe5854952355e942acca51f138096d9Rusty Russell struct cpumask *lowest_mask) 706e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins{ 71014acbf0d5c8445e0ff88ae60edd676dd9cc461cYing Xue int idx = 0; 72014acbf0d5c8445e0ff88ae60edd676dd9cc461cYing Xue int task_pri = convert_prio(p->prio); 736e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 746227cb00cc120f9a43ce8313bb0475ddabcb7d01Steven Rostedt (Red Hat) BUG_ON(task_pri >= CPUPRI_NR_PRIORITIES); 75c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt 76c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt for (idx = 0; idx < task_pri; idx++) { 776e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins struct cpupri_vec *vec = &cp->pri_to_cpu[idx]; 78d473750b4073f16f23f46f30dc1bd3de45c35754Steven Rostedt int skip = 0; 796e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 80c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt if (!atomic_read(&(vec)->count)) 81d473750b4073f16f23f46f30dc1bd3de45c35754Steven Rostedt skip = 1; 82c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt /* 83c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt * When looking at the vector, we need to read the counter, 84c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt * do a memory barrier, then read the mask. 85c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt * 86c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt * Note: This is still all racey, but we can deal with it. 87c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt * Ideally, we only want to look at masks that are set. 88c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt * 89c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt * If a mask is not set, then the only thing wrong is that we 90c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt * did a little more work than necessary. 91c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt * 92c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt * If we read a zero count but the mask is set, because of the 93c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt * memory barriers, that can only happen when the highest prio 94c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt * task for a run queue has left the run queue, in which case, 95c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt * it will be followed by a pull. If the task we are processing 96c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt * fails to find a proper place to go, that pull request will 97c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt * pull this task if the run queue is running at a lower 98c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt * priority. 99c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt */ 100c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt smp_rmb(); 1016e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 102d473750b4073f16f23f46f30dc1bd3de45c35754Steven Rostedt /* Need to do the rmb for every iteration */ 103d473750b4073f16f23f46f30dc1bd3de45c35754Steven Rostedt if (skip) 104d473750b4073f16f23f46f30dc1bd3de45c35754Steven Rostedt continue; 105d473750b4073f16f23f46f30dc1bd3de45c35754Steven Rostedt 10668e74568fbe5854952355e942acca51f138096d9Rusty Russell if (cpumask_any_and(&p->cpus_allowed, vec->mask) >= nr_cpu_ids) 1076e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins continue; 1086e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 10907903af152b0597d94e9b0030746b63c4664e787Gregory Haskins if (lowest_mask) { 11013b8bd0a5713bdf05659019badd7c0407984ece1Rusty Russell cpumask_and(lowest_mask, &p->cpus_allowed, vec->mask); 11107903af152b0597d94e9b0030746b63c4664e787Gregory Haskins 11207903af152b0597d94e9b0030746b63c4664e787Gregory Haskins /* 11307903af152b0597d94e9b0030746b63c4664e787Gregory Haskins * We have to ensure that we have at least one bit 11407903af152b0597d94e9b0030746b63c4664e787Gregory Haskins * still set in the array, since the map could have 11507903af152b0597d94e9b0030746b63c4664e787Gregory Haskins * been concurrently emptied between the first and 11607903af152b0597d94e9b0030746b63c4664e787Gregory Haskins * second reads of vec->mask. If we hit this 11707903af152b0597d94e9b0030746b63c4664e787Gregory Haskins * condition, simply act as though we never hit this 11807903af152b0597d94e9b0030746b63c4664e787Gregory Haskins * priority level and continue on. 11907903af152b0597d94e9b0030746b63c4664e787Gregory Haskins */ 12007903af152b0597d94e9b0030746b63c4664e787Gregory Haskins if (cpumask_any(lowest_mask) >= nr_cpu_ids) 12107903af152b0597d94e9b0030746b63c4664e787Gregory Haskins continue; 12207903af152b0597d94e9b0030746b63c4664e787Gregory Haskins } 12307903af152b0597d94e9b0030746b63c4664e787Gregory Haskins 1246e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins return 1; 1256e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins } 1266e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 1276e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins return 0; 1286e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins} 1296e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 1306e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins/** 1316e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * cpupri_set - update the cpu priority setting 1326e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * @cp: The cpupri context 1336e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * @cpu: The target cpu 134fa757281a08799fd6c0f7ec6f111d1cd66afc97bRandy Dunlap * @newpri: The priority (INVALID-RT99) to assign to this CPU 1356e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * 1366e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * Note: Assumes cpu_rq(cpu)->lock is locked 1376e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * 1386e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * Returns: (void) 1396e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins */ 1406e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskinsvoid cpupri_set(struct cpupri *cp, int cpu, int newpri) 1416e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins{ 142014acbf0d5c8445e0ff88ae60edd676dd9cc461cYing Xue int *currpri = &cp->cpu_to_pri[cpu]; 143014acbf0d5c8445e0ff88ae60edd676dd9cc461cYing Xue int oldpri = *currpri; 144014acbf0d5c8445e0ff88ae60edd676dd9cc461cYing Xue int do_mb = 0; 1456e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 1466e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins newpri = convert_prio(newpri); 1476e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 1486e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins BUG_ON(newpri >= CPUPRI_NR_PRIORITIES); 1496e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 1506e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins if (newpri == oldpri) 1516e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins return; 1526e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 1536e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins /* 1546e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * If the cpu was currently mapped to a different value, we 155c3a2ae3d93c0f10d29c071f599764d00b8de00cbSteven Rostedt * need to map it to the new value then remove the old value. 156c3a2ae3d93c0f10d29c071f599764d00b8de00cbSteven Rostedt * Note, we must add the new value first, otherwise we risk the 1575710f15b52664ae0bfa60a66d75464769d297b2bYong Zhang * cpu being missed by the priority loop in cpupri_find. 1586e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins */ 1596e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins if (likely(newpri != CPUPRI_INVALID)) { 1606e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins struct cpupri_vec *vec = &cp->pri_to_cpu[newpri]; 1616e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 16268e74568fbe5854952355e942acca51f138096d9Rusty Russell cpumask_set_cpu(cpu, vec->mask); 163c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt /* 164c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt * When adding a new vector, we update the mask first, 165c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt * do a write memory barrier, and then update the count, to 166c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt * make sure the vector is visible when count is set. 167c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt */ 1684e857c58efeb99393cba5a5d0d8ec7117183137cPeter Zijlstra smp_mb__before_atomic(); 169c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt atomic_inc(&(vec)->count); 170d473750b4073f16f23f46f30dc1bd3de45c35754Steven Rostedt do_mb = 1; 1716e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins } 172c3a2ae3d93c0f10d29c071f599764d00b8de00cbSteven Rostedt if (likely(oldpri != CPUPRI_INVALID)) { 173c3a2ae3d93c0f10d29c071f599764d00b8de00cbSteven Rostedt struct cpupri_vec *vec = &cp->pri_to_cpu[oldpri]; 174c3a2ae3d93c0f10d29c071f599764d00b8de00cbSteven Rostedt 175c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt /* 176d473750b4073f16f23f46f30dc1bd3de45c35754Steven Rostedt * Because the order of modification of the vec->count 177d473750b4073f16f23f46f30dc1bd3de45c35754Steven Rostedt * is important, we must make sure that the update 178d473750b4073f16f23f46f30dc1bd3de45c35754Steven Rostedt * of the new prio is seen before we decrement the 179d473750b4073f16f23f46f30dc1bd3de45c35754Steven Rostedt * old prio. This makes sure that the loop sees 180d473750b4073f16f23f46f30dc1bd3de45c35754Steven Rostedt * one or the other when we raise the priority of 181d473750b4073f16f23f46f30dc1bd3de45c35754Steven Rostedt * the run queue. We don't care about when we lower the 182d473750b4073f16f23f46f30dc1bd3de45c35754Steven Rostedt * priority, as that will trigger an rt pull anyway. 183d473750b4073f16f23f46f30dc1bd3de45c35754Steven Rostedt * 184d473750b4073f16f23f46f30dc1bd3de45c35754Steven Rostedt * We only need to do a memory barrier if we updated 185d473750b4073f16f23f46f30dc1bd3de45c35754Steven Rostedt * the new priority vec. 186d473750b4073f16f23f46f30dc1bd3de45c35754Steven Rostedt */ 187d473750b4073f16f23f46f30dc1bd3de45c35754Steven Rostedt if (do_mb) 1884e857c58efeb99393cba5a5d0d8ec7117183137cPeter Zijlstra smp_mb__after_atomic(); 189d473750b4073f16f23f46f30dc1bd3de45c35754Steven Rostedt 190d473750b4073f16f23f46f30dc1bd3de45c35754Steven Rostedt /* 191c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt * When removing from the vector, we decrement the counter first 192c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt * do a memory barrier and then clear the mask. 193c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt */ 194c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt atomic_dec(&(vec)->count); 1954e857c58efeb99393cba5a5d0d8ec7117183137cPeter Zijlstra smp_mb__after_atomic(); 196c3a2ae3d93c0f10d29c071f599764d00b8de00cbSteven Rostedt cpumask_clear_cpu(cpu, vec->mask); 197c3a2ae3d93c0f10d29c071f599764d00b8de00cbSteven Rostedt } 1986e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 1996e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins *currpri = newpri; 2006e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins} 2016e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 2026e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins/** 2036e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * cpupri_init - initialize the cpupri structure 2046e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * @cp: The cpupri context 2056e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins * 206e69f61862ab833e9b8d3c15b6ce07fd69f3bfeccYacine Belkadi * Return: -ENOMEM on memory allocation failure. 2076e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins */ 20868c38fc3cb4e5a60f502ee9c45f3dfe70e5165adPekka Enbergint cpupri_init(struct cpupri *cp) 2096e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins{ 2106e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins int i; 2116e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 2126e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins memset(cp, 0, sizeof(*cp)); 2136e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 2146e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins for (i = 0; i < CPUPRI_NR_PRIORITIES; i++) { 2156e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins struct cpupri_vec *vec = &cp->pri_to_cpu[i]; 2166e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 217c92211d9b772792a9dea530c042efb4ab5562f50Steven Rostedt atomic_set(&vec->count, 0); 21868c38fc3cb4e5a60f502ee9c45f3dfe70e5165adPekka Enberg if (!zalloc_cpumask_var(&vec->mask, GFP_KERNEL)) 21968e74568fbe5854952355e942acca51f138096d9Rusty Russell goto cleanup; 2206e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins } 2216e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 2224dac0b638310d2e92f6e19958b73d4c97c9734bbPeter Zijlstra cp->cpu_to_pri = kcalloc(nr_cpu_ids, sizeof(int), GFP_KERNEL); 2234dac0b638310d2e92f6e19958b73d4c97c9734bbPeter Zijlstra if (!cp->cpu_to_pri) 2244dac0b638310d2e92f6e19958b73d4c97c9734bbPeter Zijlstra goto cleanup; 2254dac0b638310d2e92f6e19958b73d4c97c9734bbPeter Zijlstra 2266e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins for_each_possible_cpu(i) 2276e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins cp->cpu_to_pri[i] = CPUPRI_INVALID; 2284dac0b638310d2e92f6e19958b73d4c97c9734bbPeter Zijlstra 22968e74568fbe5854952355e942acca51f138096d9Rusty Russell return 0; 23068e74568fbe5854952355e942acca51f138096d9Rusty Russell 23168e74568fbe5854952355e942acca51f138096d9Rusty Russellcleanup: 23268e74568fbe5854952355e942acca51f138096d9Rusty Russell for (i--; i >= 0; i--) 23368e74568fbe5854952355e942acca51f138096d9Rusty Russell free_cpumask_var(cp->pri_to_cpu[i].mask); 23468e74568fbe5854952355e942acca51f138096d9Rusty Russell return -ENOMEM; 2356e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins} 2366e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 23768e74568fbe5854952355e942acca51f138096d9Rusty Russell/** 23868e74568fbe5854952355e942acca51f138096d9Rusty Russell * cpupri_cleanup - clean up the cpupri structure 23968e74568fbe5854952355e942acca51f138096d9Rusty Russell * @cp: The cpupri context 24068e74568fbe5854952355e942acca51f138096d9Rusty Russell */ 24168e74568fbe5854952355e942acca51f138096d9Rusty Russellvoid cpupri_cleanup(struct cpupri *cp) 24268e74568fbe5854952355e942acca51f138096d9Rusty Russell{ 24368e74568fbe5854952355e942acca51f138096d9Rusty Russell int i; 2446e0534f278199f1e3dd1049b9bc19a7a5b87ada1Gregory Haskins 2454dac0b638310d2e92f6e19958b73d4c97c9734bbPeter Zijlstra kfree(cp->cpu_to_pri); 24668e74568fbe5854952355e942acca51f138096d9Rusty Russell for (i = 0; i < CPUPRI_NR_PRIORITIES; i++) 24768e74568fbe5854952355e942acca51f138096d9Rusty Russell free_cpumask_var(cp->pri_to_cpu[i].mask); 24868e74568fbe5854952355e942acca51f138096d9Rusty Russell} 249