116295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert/* 216295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert * padata.h - header for the padata parallelization interface 316295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert * 416295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert * Copyright (C) 2008, 2009 secunet Security Networks AG 516295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com> 616295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert * 716295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert * This program is free software; you can redistribute it and/or modify it 816295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert * under the terms and conditions of the GNU General Public License, 916295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert * version 2, as published by the Free Software Foundation. 1016295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert * 1116295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert * This program is distributed in the hope it will be useful, but WITHOUT 1216295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 1316295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 1416295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert * more details. 1516295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert * 1616295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert * You should have received a copy of the GNU General Public License along with 1716295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert * this program; if not, write to the Free Software Foundation, Inc., 1816295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 1916295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert */ 2016295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert 2116295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert#ifndef PADATA_H 2216295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert#define PADATA_H 2316295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert 2416295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert#include <linux/workqueue.h> 2516295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert#include <linux/spinlock.h> 2616295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert#include <linux/list.h> 27d46a5ac7a7e2045e33c6ad6ffb8cf18a7e86a15aSteffen Klassert#include <linux/timer.h> 28e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin#include <linux/notifier.h> 295e017dc3f8bc9e4a28983666e6bc00114a2018bbDan Kruchinin#include <linux/kobject.h> 30e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin 31e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin#define PADATA_CPU_SERIAL 0x01 32e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin#define PADATA_CPU_PARALLEL 0x02 3316295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert 340198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert/** 350198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * struct padata_priv - Embedded to the users data structure. 360198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * 370198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @list: List entry, to attach to the padata lists. 380198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @pd: Pointer to the internal control structure. 390198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @cb_cpu: Callback cpu for serializatioon. 400198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @seq_nr: Sequence number of the parallelized data object. 410198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @info: Used to pass information from the parallel to the serial function. 420198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @parallel: Parallel execution function. 430198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @serial: Serial complete function. 440198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert */ 4516295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassertstruct padata_priv { 4616295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert struct list_head list; 4716295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert struct parallel_data *pd; 4816295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert int cb_cpu; 4916295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert int info; 5016295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert void (*parallel)(struct padata_priv *padata); 5116295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert void (*serial)(struct padata_priv *padata); 5216295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert}; 5316295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert 540198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert/** 550198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * struct padata_list 560198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * 570198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @list: List head. 580198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @lock: List lock. 590198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert */ 6016295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassertstruct padata_list { 6116295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert struct list_head list; 6216295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert spinlock_t lock; 6316295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert}; 6416295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert 650198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert/** 66e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin* struct padata_serial_queue - The percpu padata serial queue 67e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin* 68e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin* @serial: List to wait for serialization after reordering. 69e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin* @work: work struct for serialization. 70e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin* @pd: Backpointer to the internal control structure. 71e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin*/ 72e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchininstruct padata_serial_queue { 73e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin struct padata_list serial; 74e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin struct work_struct work; 75e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin struct parallel_data *pd; 76e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin}; 77e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin 78e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin/** 79e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin * struct padata_parallel_queue - The percpu padata parallel queue 800198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * 810198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @parallel: List to wait for parallelization. 820198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @reorder: List to wait for reordering after parallel processing. 830198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @serial: List to wait for serialization after reordering. 840198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @pwork: work struct for parallelization. 850198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @swork: work struct for serialization. 860198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @pd: Backpointer to the internal control structure. 87e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin * @work: work struct for parallelization. 88e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin * @num_obj: Number of objects that are processed by this cpu. 890198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @cpu_index: Index of the cpu. 900198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert */ 91e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchininstruct padata_parallel_queue { 92e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin struct padata_list parallel; 93e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin struct padata_list reorder; 94e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin struct parallel_data *pd; 95e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin struct work_struct work; 96e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin atomic_t num_obj; 97e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin int cpu_index; 9816295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert}; 9916295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert 100c635696c7c0fbc720698dbec34bb83e53df6a967Steffen Klassert/** 101c635696c7c0fbc720698dbec34bb83e53df6a967Steffen Klassert * struct padata_cpumask - The cpumasks for the parallel/serial workers 102c635696c7c0fbc720698dbec34bb83e53df6a967Steffen Klassert * 103c635696c7c0fbc720698dbec34bb83e53df6a967Steffen Klassert * @pcpu: cpumask for the parallel workers. 104c635696c7c0fbc720698dbec34bb83e53df6a967Steffen Klassert * @cbcpu: cpumask for the serial (callback) workers. 105c635696c7c0fbc720698dbec34bb83e53df6a967Steffen Klassert */ 106c635696c7c0fbc720698dbec34bb83e53df6a967Steffen Klassertstruct padata_cpumask { 107c635696c7c0fbc720698dbec34bb83e53df6a967Steffen Klassert cpumask_var_t pcpu; 108c635696c7c0fbc720698dbec34bb83e53df6a967Steffen Klassert cpumask_var_t cbcpu; 109c635696c7c0fbc720698dbec34bb83e53df6a967Steffen Klassert}; 110e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin 1110198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert/** 1120198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * struct parallel_data - Internal control structure, covers everything 1130198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * that depends on the cpumask in use. 1140198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * 1150198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @pinst: padata instance. 116e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin * @pqueue: percpu padata queues used for parallelization. 117e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin * @squeue: percpu padata queues used for serialuzation. 1180198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @reorder_objects: Number of objects waiting in the reorder queues. 1190198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @refcnt: Number of objects holding a reference on this parallel_data. 1200198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @max_seq_nr: Maximal used sequence number. 121c635696c7c0fbc720698dbec34bb83e53df6a967Steffen Klassert * @cpumask: The cpumasks in use for parallel and serial workers. 1220198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @lock: Reorder lock. 1235f1a8c1bc724498ff32acbd59ed5263275676b9dSteffen Klassert * @processed: Number of already processed objects. 1240198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @timer: Reorder timer. 1250198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert */ 12616295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassertstruct parallel_data { 127e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin struct padata_instance *pinst; 12857a2ce5f54f3120467be760662c6ef3bea3f9579Namhyung Kim struct padata_parallel_queue __percpu *pqueue; 12957a2ce5f54f3120467be760662c6ef3bea3f9579Namhyung Kim struct padata_serial_queue __percpu *squeue; 130c635696c7c0fbc720698dbec34bb83e53df6a967Steffen Klassert atomic_t reorder_objects; 131c635696c7c0fbc720698dbec34bb83e53df6a967Steffen Klassert atomic_t refcnt; 1320b6b098efcddac2bf4e2a895c9b655560bbfcee4Mathias Krause atomic_t seq_nr; 133c635696c7c0fbc720698dbec34bb83e53df6a967Steffen Klassert struct padata_cpumask cpumask; 134c635696c7c0fbc720698dbec34bb83e53df6a967Steffen Klassert spinlock_t lock ____cacheline_aligned; 135c635696c7c0fbc720698dbec34bb83e53df6a967Steffen Klassert unsigned int processed; 136c635696c7c0fbc720698dbec34bb83e53df6a967Steffen Klassert struct timer_list timer; 13716295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert}; 13816295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert 1390198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert/** 1400198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * struct padata_instance - The overall control structure. 1410198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * 1420198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @cpu_notifier: cpu hotplug notifier. 1430198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @wq: The workqueue in use. 1440198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @pd: The internal control structure. 145c635696c7c0fbc720698dbec34bb83e53df6a967Steffen Klassert * @cpumask: User supplied cpumasks for parallel and serial works. 146e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin * @cpumask_change_notifier: Notifiers chain for user-defined notify 147e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin * callbacks that will be called when either @pcpu or @cbcpu 1485e017dc3f8bc9e4a28983666e6bc00114a2018bbDan Kruchinin * or both cpumasks change. 1495e017dc3f8bc9e4a28983666e6bc00114a2018bbDan Kruchinin * @kobj: padata instance kernel object. 1500198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @lock: padata instance lock. 1510198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert * @flags: padata flags. 1520198ffd135f51d4fbb0c50036395716c06632ed9Steffen Klassert */ 15316295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassertstruct padata_instance { 154e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin struct notifier_block cpu_notifier; 155e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin struct workqueue_struct *wq; 156e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin struct parallel_data *pd; 157c635696c7c0fbc720698dbec34bb83e53df6a967Steffen Klassert struct padata_cpumask cpumask; 158e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin struct blocking_notifier_head cpumask_change_notifier; 1595e017dc3f8bc9e4a28983666e6bc00114a2018bbDan Kruchinin struct kobject kobj; 160e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin struct mutex lock; 161e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin u8 flags; 162e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin#define PADATA_INIT 1 163e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin#define PADATA_RESET 2 164e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin#define PADATA_INVALID 4 16516295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert}; 16616295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert 167e6cc11707661770ca2bd4db4b0256d28f48e7541Steffen Klassertextern struct padata_instance *padata_alloc_possible( 168e6cc11707661770ca2bd4db4b0256d28f48e7541Steffen Klassert struct workqueue_struct *wq); 169e6cc11707661770ca2bd4db4b0256d28f48e7541Steffen Klassertextern struct padata_instance *padata_alloc(struct workqueue_struct *wq, 170e6cc11707661770ca2bd4db4b0256d28f48e7541Steffen Klassert const struct cpumask *pcpumask, 171e6cc11707661770ca2bd4db4b0256d28f48e7541Steffen Klassert const struct cpumask *cbcpumask); 17216295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassertextern void padata_free(struct padata_instance *pinst); 17316295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassertextern int padata_do_parallel(struct padata_instance *pinst, 17416295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert struct padata_priv *padata, int cb_cpu); 17516295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassertextern void padata_do_serial(struct padata_priv *padata); 176e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchininextern int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, 17716295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert cpumask_var_t cpumask); 17865ff577e6b6e482ee9de3569e058edebdc02f069Steffen Klassertextern int padata_set_cpumasks(struct padata_instance *pinst, 17965ff577e6b6e482ee9de3569e058edebdc02f069Steffen Klassert cpumask_var_t pcpumask, 18065ff577e6b6e482ee9de3569e058edebdc02f069Steffen Klassert cpumask_var_t cbcpumask); 181e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchininextern int padata_add_cpu(struct padata_instance *pinst, int cpu, int mask); 182e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchininextern int padata_remove_cpu(struct padata_instance *pinst, int cpu, int mask); 1834c879170296174bde05cd1c643dac16594edee77Steffen Klassertextern int padata_start(struct padata_instance *pinst); 18416295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassertextern void padata_stop(struct padata_instance *pinst); 185e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchininextern int padata_register_cpumask_notifier(struct padata_instance *pinst, 186e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin struct notifier_block *nblock); 187e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchininextern int padata_unregister_cpumask_notifier(struct padata_instance *pinst, 188e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8Dan Kruchinin struct notifier_block *nblock); 18916295bec6398a3eedc9377e1af6ff4c71b98c300Steffen Klassert#endif 190