1/*
2 *    Copyright IBM Corp. 2007,2011
3 *    Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
4 */
5
6#define KMSG_COMPONENT "cpu"
7#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
8
9#include <linux/workqueue.h>
10#include <linux/bootmem.h>
11#include <linux/cpuset.h>
12#include <linux/device.h>
13#include <linux/kernel.h>
14#include <linux/sched.h>
15#include <linux/init.h>
16#include <linux/delay.h>
17#include <linux/cpu.h>
18#include <linux/smp.h>
19#include <linux/mm.h>
20
21#define PTF_HORIZONTAL	(0UL)
22#define PTF_VERTICAL	(1UL)
23#define PTF_CHECK	(2UL)
24
25struct mask_info {
26	struct mask_info *next;
27	unsigned char id;
28	cpumask_t mask;
29};
30
31static int topology_enabled = 1;
32static void topology_work_fn(struct work_struct *work);
33static struct sysinfo_15_1_x *tl_info;
34static void set_topology_timer(void);
35static DECLARE_WORK(topology_work, topology_work_fn);
36/* topology_lock protects the core linked list */
37static DEFINE_SPINLOCK(topology_lock);
38
39static struct mask_info core_info;
40cpumask_t cpu_core_map[NR_CPUS];
41unsigned char cpu_core_id[NR_CPUS];
42
43static struct mask_info book_info;
44cpumask_t cpu_book_map[NR_CPUS];
45unsigned char cpu_book_id[NR_CPUS];
46
47/* smp_cpu_state_mutex must be held when accessing this array */
48int cpu_polarization[NR_CPUS];
49
50static cpumask_t cpu_group_map(struct mask_info *info, unsigned int cpu)
51{
52	cpumask_t mask;
53
54	cpumask_clear(&mask);
55	if (!topology_enabled || !MACHINE_HAS_TOPOLOGY) {
56		cpumask_copy(&mask, cpumask_of(cpu));
57		return mask;
58	}
59	while (info) {
60		if (cpumask_test_cpu(cpu, &info->mask)) {
61			mask = info->mask;
62			break;
63		}
64		info = info->next;
65	}
66	if (cpumask_empty(&mask))
67		cpumask_copy(&mask, cpumask_of(cpu));
68	return mask;
69}
70
71static struct mask_info *add_cpus_to_mask(struct topology_cpu *tl_cpu,
72					  struct mask_info *book,
73					  struct mask_info *core,
74					  int one_core_per_cpu)
75{
76	unsigned int cpu;
77
78	for (cpu = find_first_bit(&tl_cpu->mask[0], TOPOLOGY_CPU_BITS);
79	     cpu < TOPOLOGY_CPU_BITS;
80	     cpu = find_next_bit(&tl_cpu->mask[0], TOPOLOGY_CPU_BITS, cpu + 1))
81	{
82		unsigned int rcpu;
83		int lcpu;
84
85		rcpu = TOPOLOGY_CPU_BITS - 1 - cpu + tl_cpu->origin;
86		lcpu = smp_find_processor_id(rcpu);
87		if (lcpu >= 0) {
88			cpumask_set_cpu(lcpu, &book->mask);
89			cpu_book_id[lcpu] = book->id;
90			cpumask_set_cpu(lcpu, &core->mask);
91			if (one_core_per_cpu) {
92				cpu_core_id[lcpu] = rcpu;
93				core = core->next;
94			} else {
95				cpu_core_id[lcpu] = core->id;
96			}
97			cpu_set_polarization(lcpu, tl_cpu->pp);
98		}
99	}
100	return core;
101}
102
103static void clear_masks(void)
104{
105	struct mask_info *info;
106
107	info = &core_info;
108	while (info) {
109		cpumask_clear(&info->mask);
110		info = info->next;
111	}
112	info = &book_info;
113	while (info) {
114		cpumask_clear(&info->mask);
115		info = info->next;
116	}
117}
118
119static union topology_entry *next_tle(union topology_entry *tle)
120{
121	if (!tle->nl)
122		return (union topology_entry *)((struct topology_cpu *)tle + 1);
123	return (union topology_entry *)((struct topology_container *)tle + 1);
124}
125
126static void __tl_to_cores_generic(struct sysinfo_15_1_x *info)
127{
128	struct mask_info *core = &core_info;
129	struct mask_info *book = &book_info;
130	union topology_entry *tle, *end;
131
132	tle = info->tle;
133	end = (union topology_entry *)((unsigned long)info + info->length);
134	while (tle < end) {
135		switch (tle->nl) {
136		case 2:
137			book = book->next;
138			book->id = tle->container.id;
139			break;
140		case 1:
141			core = core->next;
142			core->id = tle->container.id;
143			break;
144		case 0:
145			add_cpus_to_mask(&tle->cpu, book, core, 0);
146			break;
147		default:
148			clear_masks();
149			return;
150		}
151		tle = next_tle(tle);
152	}
153}
154
155static void __tl_to_cores_z10(struct sysinfo_15_1_x *info)
156{
157	struct mask_info *core = &core_info;
158	struct mask_info *book = &book_info;
159	union topology_entry *tle, *end;
160
161	tle = info->tle;
162	end = (union topology_entry *)((unsigned long)info + info->length);
163	while (tle < end) {
164		switch (tle->nl) {
165		case 1:
166			book = book->next;
167			book->id = tle->container.id;
168			break;
169		case 0:
170			core = add_cpus_to_mask(&tle->cpu, book, core, 1);
171			break;
172		default:
173			clear_masks();
174			return;
175		}
176		tle = next_tle(tle);
177	}
178}
179
180static void tl_to_cores(struct sysinfo_15_1_x *info)
181{
182	struct cpuid cpu_id;
183
184	get_cpu_id(&cpu_id);
185	spin_lock_irq(&topology_lock);
186	clear_masks();
187	switch (cpu_id.machine) {
188	case 0x2097:
189	case 0x2098:
190		__tl_to_cores_z10(info);
191		break;
192	default:
193		__tl_to_cores_generic(info);
194	}
195	spin_unlock_irq(&topology_lock);
196}
197
198static void topology_update_polarization_simple(void)
199{
200	int cpu;
201
202	mutex_lock(&smp_cpu_state_mutex);
203	for_each_possible_cpu(cpu)
204		cpu_set_polarization(cpu, POLARIZATION_HRZ);
205	mutex_unlock(&smp_cpu_state_mutex);
206}
207
208static int ptf(unsigned long fc)
209{
210	int rc;
211
212	asm volatile(
213		"	.insn	rre,0xb9a20000,%1,%1\n"
214		"	ipm	%0\n"
215		"	srl	%0,28\n"
216		: "=d" (rc)
217		: "d" (fc)  : "cc");
218	return rc;
219}
220
221int topology_set_cpu_management(int fc)
222{
223	int cpu, rc;
224
225	if (!MACHINE_HAS_TOPOLOGY)
226		return -EOPNOTSUPP;
227	if (fc)
228		rc = ptf(PTF_VERTICAL);
229	else
230		rc = ptf(PTF_HORIZONTAL);
231	if (rc)
232		return -EBUSY;
233	for_each_possible_cpu(cpu)
234		cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
235	return rc;
236}
237
238static void update_cpu_core_map(void)
239{
240	unsigned long flags;
241	int cpu;
242
243	spin_lock_irqsave(&topology_lock, flags);
244	for_each_possible_cpu(cpu) {
245		cpu_core_map[cpu] = cpu_group_map(&core_info, cpu);
246		cpu_book_map[cpu] = cpu_group_map(&book_info, cpu);
247	}
248	spin_unlock_irqrestore(&topology_lock, flags);
249}
250
251void store_topology(struct sysinfo_15_1_x *info)
252{
253	int rc;
254
255	rc = stsi(info, 15, 1, 3);
256	if (rc != -ENOSYS)
257		return;
258	stsi(info, 15, 1, 2);
259}
260
261int arch_update_cpu_topology(void)
262{
263	struct sysinfo_15_1_x *info = tl_info;
264	struct device *dev;
265	int cpu;
266
267	if (!MACHINE_HAS_TOPOLOGY) {
268		update_cpu_core_map();
269		topology_update_polarization_simple();
270		return 0;
271	}
272	store_topology(info);
273	tl_to_cores(info);
274	update_cpu_core_map();
275	for_each_online_cpu(cpu) {
276		dev = get_cpu_device(cpu);
277		kobject_uevent(&dev->kobj, KOBJ_CHANGE);
278	}
279	return 1;
280}
281
282static void topology_work_fn(struct work_struct *work)
283{
284	rebuild_sched_domains();
285}
286
287void topology_schedule_update(void)
288{
289	schedule_work(&topology_work);
290}
291
292static void topology_timer_fn(unsigned long ignored)
293{
294	if (ptf(PTF_CHECK))
295		topology_schedule_update();
296	set_topology_timer();
297}
298
299static struct timer_list topology_timer =
300	TIMER_DEFERRED_INITIALIZER(topology_timer_fn, 0, 0);
301
302static atomic_t topology_poll = ATOMIC_INIT(0);
303
304static void set_topology_timer(void)
305{
306	if (atomic_add_unless(&topology_poll, -1, 0))
307		mod_timer(&topology_timer, jiffies + HZ / 10);
308	else
309		mod_timer(&topology_timer, jiffies + HZ * 60);
310}
311
312void topology_expect_change(void)
313{
314	if (!MACHINE_HAS_TOPOLOGY)
315		return;
316	/* This is racy, but it doesn't matter since it is just a heuristic.
317	 * Worst case is that we poll in a higher frequency for a bit longer.
318	 */
319	if (atomic_read(&topology_poll) > 60)
320		return;
321	atomic_add(60, &topology_poll);
322	set_topology_timer();
323}
324
325static int __init early_parse_topology(char *p)
326{
327	if (strncmp(p, "off", 3))
328		return 0;
329	topology_enabled = 0;
330	return 0;
331}
332early_param("topology", early_parse_topology);
333
334static void __init alloc_masks(struct sysinfo_15_1_x *info,
335			       struct mask_info *mask, int offset)
336{
337	int i, nr_masks;
338
339	nr_masks = info->mag[TOPOLOGY_NR_MAG - offset];
340	for (i = 0; i < info->mnest - offset; i++)
341		nr_masks *= info->mag[TOPOLOGY_NR_MAG - offset - 1 - i];
342	nr_masks = max(nr_masks, 1);
343	for (i = 0; i < nr_masks; i++) {
344		mask->next = alloc_bootmem(sizeof(struct mask_info));
345		mask = mask->next;
346	}
347}
348
349void __init s390_init_cpu_topology(void)
350{
351	struct sysinfo_15_1_x *info;
352	int i;
353
354	if (!MACHINE_HAS_TOPOLOGY)
355		return;
356	tl_info = alloc_bootmem_pages(PAGE_SIZE);
357	info = tl_info;
358	store_topology(info);
359	pr_info("The CPU configuration topology of the machine is:");
360	for (i = 0; i < TOPOLOGY_NR_MAG; i++)
361		printk(KERN_CONT " %d", info->mag[i]);
362	printk(KERN_CONT " / %d\n", info->mnest);
363	alloc_masks(info, &core_info, 1);
364	alloc_masks(info, &book_info, 2);
365}
366
367static int cpu_management;
368
369static ssize_t dispatching_show(struct device *dev,
370				struct device_attribute *attr,
371				char *buf)
372{
373	ssize_t count;
374
375	mutex_lock(&smp_cpu_state_mutex);
376	count = sprintf(buf, "%d\n", cpu_management);
377	mutex_unlock(&smp_cpu_state_mutex);
378	return count;
379}
380
381static ssize_t dispatching_store(struct device *dev,
382				 struct device_attribute *attr,
383				 const char *buf,
384				 size_t count)
385{
386	int val, rc;
387	char delim;
388
389	if (sscanf(buf, "%d %c", &val, &delim) != 1)
390		return -EINVAL;
391	if (val != 0 && val != 1)
392		return -EINVAL;
393	rc = 0;
394	get_online_cpus();
395	mutex_lock(&smp_cpu_state_mutex);
396	if (cpu_management == val)
397		goto out;
398	rc = topology_set_cpu_management(val);
399	if (rc)
400		goto out;
401	cpu_management = val;
402	topology_expect_change();
403out:
404	mutex_unlock(&smp_cpu_state_mutex);
405	put_online_cpus();
406	return rc ? rc : count;
407}
408static DEVICE_ATTR(dispatching, 0644, dispatching_show,
409			 dispatching_store);
410
411static ssize_t cpu_polarization_show(struct device *dev,
412				     struct device_attribute *attr, char *buf)
413{
414	int cpu = dev->id;
415	ssize_t count;
416
417	mutex_lock(&smp_cpu_state_mutex);
418	switch (cpu_read_polarization(cpu)) {
419	case POLARIZATION_HRZ:
420		count = sprintf(buf, "horizontal\n");
421		break;
422	case POLARIZATION_VL:
423		count = sprintf(buf, "vertical:low\n");
424		break;
425	case POLARIZATION_VM:
426		count = sprintf(buf, "vertical:medium\n");
427		break;
428	case POLARIZATION_VH:
429		count = sprintf(buf, "vertical:high\n");
430		break;
431	default:
432		count = sprintf(buf, "unknown\n");
433		break;
434	}
435	mutex_unlock(&smp_cpu_state_mutex);
436	return count;
437}
438static DEVICE_ATTR(polarization, 0444, cpu_polarization_show, NULL);
439
440static struct attribute *topology_cpu_attrs[] = {
441	&dev_attr_polarization.attr,
442	NULL,
443};
444
445static struct attribute_group topology_cpu_attr_group = {
446	.attrs = topology_cpu_attrs,
447};
448
449int topology_cpu_init(struct cpu *cpu)
450{
451	return sysfs_create_group(&cpu->dev.kobj, &topology_cpu_attr_group);
452}
453
454static int __init topology_init(void)
455{
456	if (!MACHINE_HAS_TOPOLOGY) {
457		topology_update_polarization_simple();
458		goto out;
459	}
460	set_topology_timer();
461out:
462	update_cpu_core_map();
463	return device_create_file(cpu_subsys.dev_root, &dev_attr_dispatching);
464}
465device_initcall(topology_init);
466