cpumask.h 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872
  1. #ifndef __LINUX_CPUMASK_H
  2. #define __LINUX_CPUMASK_H
  3. /*
  4. * Cpumasks provide a bitmap suitable for representing the
  5. * set of CPU's in a system, one bit position per CPU number. In general,
  6. * only nr_cpu_ids (<= NR_CPUS) bits are valid.
  7. */
  8. #include <linux/kernel.h>
  9. #include <linux/threads.h>
  10. #include <linux/bitmap.h>
  11. #include <linux/bug.h>
  12. /* Don't assign or return these: may not be this big! */
  13. typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
  14. /**
  15. * cpumask_bits - get the bits in a cpumask
  16. * @maskp: the struct cpumask *
  17. *
  18. * You should only assume nr_cpu_ids bits of this mask are valid. This is
  19. * a macro so it's const-correct.
  20. */
  21. #define cpumask_bits(maskp) ((maskp)->bits)
  22. /**
  23. * cpumask_pr_args - printf args to output a cpumask
  24. * @maskp: cpumask to be printed
  25. *
  26. * Can be used to provide arguments for '%*pb[l]' when printing a cpumask.
  27. */
  28. #define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp)
  29. #if NR_CPUS == 1
  30. #define nr_cpu_ids 1
  31. #else
  32. extern int nr_cpu_ids;
  33. #endif
  34. #ifdef CONFIG_CPUMASK_OFFSTACK
  35. /* Assuming NR_CPUS is huge, a runtime limit is more efficient. Also,
  36. * not all bits may be allocated. */
  37. #define nr_cpumask_bits nr_cpu_ids
  38. #else
  39. #define nr_cpumask_bits NR_CPUS
  40. #endif
  41. /*
  42. * The following particular system cpumasks and operations manage
  43. * possible, present, active and online cpus.
  44. *
  45. * cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
  46. * cpu_present_mask - has bit 'cpu' set iff cpu is populated
  47. * cpu_online_mask - has bit 'cpu' set iff cpu available to scheduler
  48. * cpu_active_mask - has bit 'cpu' set iff cpu available to migration
  49. *
  50. * If !CONFIG_HOTPLUG_CPU, present == possible, and active == online.
  51. *
  52. * The cpu_possible_mask is fixed at boot time, as the set of CPU id's
  53. * that it is possible might ever be plugged in at anytime during the
  54. * life of that system boot. The cpu_present_mask is dynamic(*),
  55. * representing which CPUs are currently plugged in. And
  56. * cpu_online_mask is the dynamic subset of cpu_present_mask,
  57. * indicating those CPUs available for scheduling.
  58. *
  59. * If HOTPLUG is enabled, then cpu_possible_mask is forced to have
  60. * all NR_CPUS bits set, otherwise it is just the set of CPUs that
  61. * ACPI reports present at boot.
  62. *
  63. * If HOTPLUG is enabled, then cpu_present_mask varies dynamically,
  64. * depending on what ACPI reports as currently plugged in, otherwise
  65. * cpu_present_mask is just a copy of cpu_possible_mask.
  66. *
  67. * (*) Well, cpu_present_mask is dynamic in the hotplug case. If not
  68. * hotplug, it's a copy of cpu_possible_mask, hence fixed at boot.
  69. *
  70. * Subtleties:
  71. * 1) UP arch's (NR_CPUS == 1, CONFIG_SMP not defined) hardcode
  72. * assumption that their single CPU is online. The UP
  73. * cpu_{online,possible,present}_masks are placebos. Changing them
  74. * will have no useful affect on the following num_*_cpus()
  75. * and cpu_*() macros in the UP case. This ugliness is a UP
  76. * optimization - don't waste any instructions or memory references
  77. * asking if you're online or how many CPUs there are if there is
  78. * only one CPU.
  79. */
  80. extern struct cpumask __cpu_possible_mask;
  81. extern struct cpumask __cpu_online_mask;
  82. extern struct cpumask __cpu_present_mask;
  83. extern struct cpumask __cpu_active_mask;
  84. #define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
  85. #define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask)
  86. #define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask)
  87. #define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask)
  88. #if NR_CPUS > 1
  89. #define num_online_cpus() cpumask_weight(cpu_online_mask)
  90. #define num_possible_cpus() cpumask_weight(cpu_possible_mask)
  91. #define num_present_cpus() cpumask_weight(cpu_present_mask)
  92. #define num_active_cpus() cpumask_weight(cpu_active_mask)
  93. #define cpu_online(cpu) cpumask_test_cpu((cpu), cpu_online_mask)
  94. #define cpu_possible(cpu) cpumask_test_cpu((cpu), cpu_possible_mask)
  95. #define cpu_present(cpu) cpumask_test_cpu((cpu), cpu_present_mask)
  96. #define cpu_active(cpu) cpumask_test_cpu((cpu), cpu_active_mask)
  97. #else
  98. #define num_online_cpus() 1U
  99. #define num_possible_cpus() 1U
  100. #define num_present_cpus() 1U
  101. #define num_active_cpus() 1U
  102. #define cpu_online(cpu) ((cpu) == 0)
  103. #define cpu_possible(cpu) ((cpu) == 0)
  104. #define cpu_present(cpu) ((cpu) == 0)
  105. #define cpu_active(cpu) ((cpu) == 0)
  106. #endif
  107. /* verify cpu argument to cpumask_* operators */
  108. static inline unsigned int cpumask_check(unsigned int cpu)
  109. {
  110. #ifdef CONFIG_DEBUG_PER_CPU_MAPS
  111. WARN_ON_ONCE(cpu >= nr_cpumask_bits);
  112. #endif /* CONFIG_DEBUG_PER_CPU_MAPS */
  113. return cpu;
  114. }
  115. #if NR_CPUS == 1
  116. /* Uniprocessor. Assume all masks are "1". */
  117. static inline unsigned int cpumask_first(const struct cpumask *srcp)
  118. {
  119. return 0;
  120. }
  121. /* Valid inputs for n are -1 and 0. */
  122. static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
  123. {
  124. return n+1;
  125. }
  126. static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
  127. {
  128. return n+1;
  129. }
  130. static inline unsigned int cpumask_next_and(int n,
  131. const struct cpumask *srcp,
  132. const struct cpumask *andp)
  133. {
  134. return n+1;
  135. }
  136. /* cpu must be a valid cpu, ie 0, so there's no other choice. */
  137. static inline unsigned int cpumask_any_but(const struct cpumask *mask,
  138. unsigned int cpu)
  139. {
  140. return 1;
  141. }
  142. static inline unsigned int cpumask_local_spread(unsigned int i, int node)
  143. {
  144. return 0;
  145. }
  146. #define for_each_cpu(cpu, mask) \
  147. for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
  148. #define for_each_cpu_not(cpu, mask) \
  149. for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
  150. #define for_each_cpu_and(cpu, mask, and) \
  151. for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)and)
  152. #else
  153. /**
  154. * cpumask_first - get the first cpu in a cpumask
  155. * @srcp: the cpumask pointer
  156. *
  157. * Returns >= nr_cpu_ids if no cpus set.
  158. */
  159. static inline unsigned int cpumask_first(const struct cpumask *srcp)
  160. {
  161. return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits);
  162. }
  163. /**
  164. * cpumask_next - get the next cpu in a cpumask
  165. * @n: the cpu prior to the place to search (ie. return will be > @n)
  166. * @srcp: the cpumask pointer
  167. *
  168. * Returns >= nr_cpu_ids if no further cpus set.
  169. */
  170. static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
  171. {
  172. /* -1 is a legal arg here. */
  173. if (n != -1)
  174. cpumask_check(n);
  175. return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
  176. }
  177. /**
  178. * cpumask_next_zero - get the next unset cpu in a cpumask
  179. * @n: the cpu prior to the place to search (ie. return will be > @n)
  180. * @srcp: the cpumask pointer
  181. *
  182. * Returns >= nr_cpu_ids if no further cpus unset.
  183. */
  184. static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
  185. {
  186. /* -1 is a legal arg here. */
  187. if (n != -1)
  188. cpumask_check(n);
  189. return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
  190. }
  191. int cpumask_next_and(int n, const struct cpumask *, const struct cpumask *);
  192. int cpumask_any_but(const struct cpumask *mask, unsigned int cpu);
  193. unsigned int cpumask_local_spread(unsigned int i, int node);
  194. /**
  195. * for_each_cpu - iterate over every cpu in a mask
  196. * @cpu: the (optionally unsigned) integer iterator
  197. * @mask: the cpumask pointer
  198. *
  199. * After the loop, cpu is >= nr_cpu_ids.
  200. */
  201. #define for_each_cpu(cpu, mask) \
  202. for ((cpu) = -1; \
  203. (cpu) = cpumask_next((cpu), (mask)), \
  204. (cpu) < nr_cpu_ids;)
  205. /**
  206. * for_each_cpu_not - iterate over every cpu in a complemented mask
  207. * @cpu: the (optionally unsigned) integer iterator
  208. * @mask: the cpumask pointer
  209. *
  210. * After the loop, cpu is >= nr_cpu_ids.
  211. */
  212. #define for_each_cpu_not(cpu, mask) \
  213. for ((cpu) = -1; \
  214. (cpu) = cpumask_next_zero((cpu), (mask)), \
  215. (cpu) < nr_cpu_ids;)
  216. extern int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap);
  217. /**
  218. * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location
  219. * @cpu: the (optionally unsigned) integer iterator
  220. * @mask: the cpumask poiter
  221. * @start: the start location
  222. *
  223. * The implementation does not assume any bit in @mask is set (including @start).
  224. *
  225. * After the loop, cpu is >= nr_cpu_ids.
  226. */
  227. #define for_each_cpu_wrap(cpu, mask, start) \
  228. for ((cpu) = cpumask_next_wrap((start)-1, (mask), (start), false); \
  229. (cpu) < nr_cpumask_bits; \
  230. (cpu) = cpumask_next_wrap((cpu), (mask), (start), true))
  231. /**
  232. * for_each_cpu_and - iterate over every cpu in both masks
  233. * @cpu: the (optionally unsigned) integer iterator
  234. * @mask: the first cpumask pointer
  235. * @and: the second cpumask pointer
  236. *
  237. * This saves a temporary CPU mask in many places. It is equivalent to:
  238. * struct cpumask tmp;
  239. * cpumask_and(&tmp, &mask, &and);
  240. * for_each_cpu(cpu, &tmp)
  241. * ...
  242. *
  243. * After the loop, cpu is >= nr_cpu_ids.
  244. */
  245. #define for_each_cpu_and(cpu, mask, and) \
  246. for ((cpu) = -1; \
  247. (cpu) = cpumask_next_and((cpu), (mask), (and)), \
  248. (cpu) < nr_cpu_ids;)
  249. #endif /* SMP */
  250. #define CPU_BITS_NONE \
  251. { \
  252. [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
  253. }
  254. #define CPU_BITS_CPU0 \
  255. { \
  256. [0] = 1UL \
  257. }
  258. /**
  259. * cpumask_set_cpu - set a cpu in a cpumask
  260. * @cpu: cpu number (< nr_cpu_ids)
  261. * @dstp: the cpumask pointer
  262. */
  263. static inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
  264. {
  265. set_bit(cpumask_check(cpu), cpumask_bits(dstp));
  266. }
  267. /**
  268. * cpumask_clear_cpu - clear a cpu in a cpumask
  269. * @cpu: cpu number (< nr_cpu_ids)
  270. * @dstp: the cpumask pointer
  271. */
  272. static inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
  273. {
  274. clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
  275. }
  276. /**
  277. * cpumask_test_cpu - test for a cpu in a cpumask
  278. * @cpu: cpu number (< nr_cpu_ids)
  279. * @cpumask: the cpumask pointer
  280. *
  281. * Returns 1 if @cpu is set in @cpumask, else returns 0
  282. */
  283. static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
  284. {
  285. return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
  286. }
  287. /**
  288. * cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask
  289. * @cpu: cpu number (< nr_cpu_ids)
  290. * @cpumask: the cpumask pointer
  291. *
  292. * Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0
  293. *
  294. * test_and_set_bit wrapper for cpumasks.
  295. */
  296. static inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
  297. {
  298. return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
  299. }
  300. /**
  301. * cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask
  302. * @cpu: cpu number (< nr_cpu_ids)
  303. * @cpumask: the cpumask pointer
  304. *
  305. * Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0
  306. *
  307. * test_and_clear_bit wrapper for cpumasks.
  308. */
  309. static inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
  310. {
  311. return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
  312. }
  313. /**
  314. * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
  315. * @dstp: the cpumask pointer
  316. */
  317. static inline void cpumask_setall(struct cpumask *dstp)
  318. {
  319. bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits);
  320. }
  321. /**
  322. * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
  323. * @dstp: the cpumask pointer
  324. */
  325. static inline void cpumask_clear(struct cpumask *dstp)
  326. {
  327. bitmap_zero(cpumask_bits(dstp), nr_cpumask_bits);
  328. }
  329. /**
  330. * cpumask_and - *dstp = *src1p & *src2p
  331. * @dstp: the cpumask result
  332. * @src1p: the first input
  333. * @src2p: the second input
  334. *
  335. * If *@dstp is empty, returns 0, else returns 1
  336. */
  337. static inline int cpumask_and(struct cpumask *dstp,
  338. const struct cpumask *src1p,
  339. const struct cpumask *src2p)
  340. {
  341. return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
  342. cpumask_bits(src2p), nr_cpumask_bits);
  343. }
  344. /**
  345. * cpumask_or - *dstp = *src1p | *src2p
  346. * @dstp: the cpumask result
  347. * @src1p: the first input
  348. * @src2p: the second input
  349. */
  350. static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
  351. const struct cpumask *src2p)
  352. {
  353. bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
  354. cpumask_bits(src2p), nr_cpumask_bits);
  355. }
  356. /**
  357. * cpumask_xor - *dstp = *src1p ^ *src2p
  358. * @dstp: the cpumask result
  359. * @src1p: the first input
  360. * @src2p: the second input
  361. */
  362. static inline void cpumask_xor(struct cpumask *dstp,
  363. const struct cpumask *src1p,
  364. const struct cpumask *src2p)
  365. {
  366. bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
  367. cpumask_bits(src2p), nr_cpumask_bits);
  368. }
  369. /**
  370. * cpumask_andnot - *dstp = *src1p & ~*src2p
  371. * @dstp: the cpumask result
  372. * @src1p: the first input
  373. * @src2p: the second input
  374. *
  375. * If *@dstp is empty, returns 0, else returns 1
  376. */
  377. static inline int cpumask_andnot(struct cpumask *dstp,
  378. const struct cpumask *src1p,
  379. const struct cpumask *src2p)
  380. {
  381. return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
  382. cpumask_bits(src2p), nr_cpumask_bits);
  383. }
  384. /**
  385. * cpumask_complement - *dstp = ~*srcp
  386. * @dstp: the cpumask result
  387. * @srcp: the input to invert
  388. */
  389. static inline void cpumask_complement(struct cpumask *dstp,
  390. const struct cpumask *srcp)
  391. {
  392. bitmap_complement(cpumask_bits(dstp), cpumask_bits(srcp),
  393. nr_cpumask_bits);
  394. }
  395. /**
  396. * cpumask_equal - *src1p == *src2p
  397. * @src1p: the first input
  398. * @src2p: the second input
  399. */
  400. static inline bool cpumask_equal(const struct cpumask *src1p,
  401. const struct cpumask *src2p)
  402. {
  403. return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
  404. nr_cpumask_bits);
  405. }
  406. /**
  407. * cpumask_intersects - (*src1p & *src2p) != 0
  408. * @src1p: the first input
  409. * @src2p: the second input
  410. */
  411. static inline bool cpumask_intersects(const struct cpumask *src1p,
  412. const struct cpumask *src2p)
  413. {
  414. return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
  415. nr_cpumask_bits);
  416. }
  417. /**
  418. * cpumask_subset - (*src1p & ~*src2p) == 0
  419. * @src1p: the first input
  420. * @src2p: the second input
  421. *
  422. * Returns 1 if *@src1p is a subset of *@src2p, else returns 0
  423. */
  424. static inline int cpumask_subset(const struct cpumask *src1p,
  425. const struct cpumask *src2p)
  426. {
  427. return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
  428. nr_cpumask_bits);
  429. }
  430. /**
  431. * cpumask_empty - *srcp == 0
  432. * @srcp: the cpumask to that all cpus < nr_cpu_ids are clear.
  433. */
  434. static inline bool cpumask_empty(const struct cpumask *srcp)
  435. {
  436. return bitmap_empty(cpumask_bits(srcp), nr_cpumask_bits);
  437. }
  438. /**
  439. * cpumask_full - *srcp == 0xFFFFFFFF...
  440. * @srcp: the cpumask to that all cpus < nr_cpu_ids are set.
  441. */
  442. static inline bool cpumask_full(const struct cpumask *srcp)
  443. {
  444. return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
  445. }
  446. /**
  447. * cpumask_weight - Count of bits in *srcp
  448. * @srcp: the cpumask to count bits (< nr_cpu_ids) in.
  449. */
  450. static inline unsigned int cpumask_weight(const struct cpumask *srcp)
  451. {
  452. return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits);
  453. }
  454. /**
  455. * cpumask_shift_right - *dstp = *srcp >> n
  456. * @dstp: the cpumask result
  457. * @srcp: the input to shift
  458. * @n: the number of bits to shift by
  459. */
  460. static inline void cpumask_shift_right(struct cpumask *dstp,
  461. const struct cpumask *srcp, int n)
  462. {
  463. bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
  464. nr_cpumask_bits);
  465. }
  466. /**
  467. * cpumask_shift_left - *dstp = *srcp << n
  468. * @dstp: the cpumask result
  469. * @srcp: the input to shift
  470. * @n: the number of bits to shift by
  471. */
  472. static inline void cpumask_shift_left(struct cpumask *dstp,
  473. const struct cpumask *srcp, int n)
  474. {
  475. bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
  476. nr_cpumask_bits);
  477. }
  478. /**
  479. * cpumask_copy - *dstp = *srcp
  480. * @dstp: the result
  481. * @srcp: the input cpumask
  482. */
  483. static inline void cpumask_copy(struct cpumask *dstp,
  484. const struct cpumask *srcp)
  485. {
  486. bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), nr_cpumask_bits);
  487. }
  488. /**
  489. * cpumask_any - pick a "random" cpu from *srcp
  490. * @srcp: the input cpumask
  491. *
  492. * Returns >= nr_cpu_ids if no cpus set.
  493. */
  494. #define cpumask_any(srcp) cpumask_first(srcp)
  495. /**
  496. * cpumask_first_and - return the first cpu from *srcp1 & *srcp2
  497. * @src1p: the first input
  498. * @src2p: the second input
  499. *
  500. * Returns >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and().
  501. */
  502. #define cpumask_first_and(src1p, src2p) cpumask_next_and(-1, (src1p), (src2p))
  503. /**
  504. * cpumask_any_and - pick a "random" cpu from *mask1 & *mask2
  505. * @mask1: the first input cpumask
  506. * @mask2: the second input cpumask
  507. *
  508. * Returns >= nr_cpu_ids if no cpus set.
  509. */
  510. #define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2))
  511. /**
  512. * cpumask_of - the cpumask containing just a given cpu
  513. * @cpu: the cpu (<= nr_cpu_ids)
  514. */
  515. #define cpumask_of(cpu) (get_cpu_mask(cpu))
  516. /**
  517. * cpumask_parse_user - extract a cpumask from a user string
  518. * @buf: the buffer to extract from
  519. * @len: the length of the buffer
  520. * @dstp: the cpumask to set.
  521. *
  522. * Returns -errno, or 0 for success.
  523. */
  524. static inline int cpumask_parse_user(const char __user *buf, int len,
  525. struct cpumask *dstp)
  526. {
  527. return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
  528. }
  529. /**
  530. * cpumask_parselist_user - extract a cpumask from a user string
  531. * @buf: the buffer to extract from
  532. * @len: the length of the buffer
  533. * @dstp: the cpumask to set.
  534. *
  535. * Returns -errno, or 0 for success.
  536. */
  537. static inline int cpumask_parselist_user(const char __user *buf, int len,
  538. struct cpumask *dstp)
  539. {
  540. return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
  541. nr_cpumask_bits);
  542. }
  543. /**
  544. * cpumask_parse - extract a cpumask from a string
  545. * @buf: the buffer to extract from
  546. * @dstp: the cpumask to set.
  547. *
  548. * Returns -errno, or 0 for success.
  549. */
  550. static inline int cpumask_parse(const char *buf, struct cpumask *dstp)
  551. {
  552. char *nl = strchr(buf, '\n');
  553. unsigned int len = nl ? (unsigned int)(nl - buf) : strlen(buf);
  554. return bitmap_parse(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
  555. }
  556. /**
  557. * cpulist_parse - extract a cpumask from a user string of ranges
  558. * @buf: the buffer to extract from
  559. * @dstp: the cpumask to set.
  560. *
  561. * Returns -errno, or 0 for success.
  562. */
  563. static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
  564. {
  565. return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
  566. }
  567. /**
  568. * cpumask_size - size to allocate for a 'struct cpumask' in bytes
  569. */
  570. static inline size_t cpumask_size(void)
  571. {
  572. return BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long);
  573. }
  574. /*
  575. * cpumask_var_t: struct cpumask for stack usage.
  576. *
  577. * Oh, the wicked games we play! In order to make kernel coding a
  578. * little more difficult, we typedef cpumask_var_t to an array or a
  579. * pointer: doing &mask on an array is a noop, so it still works.
  580. *
  581. * ie.
  582. * cpumask_var_t tmpmask;
  583. * if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
  584. * return -ENOMEM;
  585. *
  586. * ... use 'tmpmask' like a normal struct cpumask * ...
  587. *
  588. * free_cpumask_var(tmpmask);
  589. *
  590. *
  591. * However, one notable exception is there. alloc_cpumask_var() allocates
  592. * only nr_cpumask_bits bits (in the other hand, real cpumask_t always has
  593. * NR_CPUS bits). Therefore you don't have to dereference cpumask_var_t.
  594. *
  595. * cpumask_var_t tmpmask;
  596. * if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
  597. * return -ENOMEM;
  598. *
  599. * var = *tmpmask;
  600. *
  601. * This code makes NR_CPUS length memcopy and brings to a memory corruption.
  602. * cpumask_copy() provide safe copy functionality.
  603. *
  604. * Note that there is another evil here: If you define a cpumask_var_t
  605. * as a percpu variable then the way to obtain the address of the cpumask
  606. * structure differently influences what this_cpu_* operation needs to be
  607. * used. Please use this_cpu_cpumask_var_t in those cases. The direct use
  608. * of this_cpu_ptr() or this_cpu_read() will lead to failures when the
  609. * other type of cpumask_var_t implementation is configured.
  610. */
  611. #ifdef CONFIG_CPUMASK_OFFSTACK
  612. typedef struct cpumask *cpumask_var_t;
  613. #define this_cpu_cpumask_var_ptr(x) this_cpu_read(x)
  614. bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
  615. bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
  616. bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
  617. bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
  618. void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
  619. void free_cpumask_var(cpumask_var_t mask);
  620. void free_bootmem_cpumask_var(cpumask_var_t mask);
  621. #else
  622. typedef struct cpumask cpumask_var_t[1];
  623. #define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
  624. static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
  625. {
  626. return true;
  627. }
  628. static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
  629. int node)
  630. {
  631. return true;
  632. }
  633. static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
  634. {
  635. cpumask_clear(*mask);
  636. return true;
  637. }
  638. static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
  639. int node)
  640. {
  641. cpumask_clear(*mask);
  642. return true;
  643. }
  644. static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
  645. {
  646. }
  647. static inline void free_cpumask_var(cpumask_var_t mask)
  648. {
  649. }
  650. static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
  651. {
  652. }
  653. #endif /* CONFIG_CPUMASK_OFFSTACK */
  654. /* It's common to want to use cpu_all_mask in struct member initializers,
  655. * so it has to refer to an address rather than a pointer. */
  656. extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
  657. #define cpu_all_mask to_cpumask(cpu_all_bits)
  658. /* First bits of cpu_bit_bitmap are in fact unset. */
  659. #define cpu_none_mask to_cpumask(cpu_bit_bitmap[0])
  660. #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
  661. #define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask)
  662. #define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask)
  663. /* Wrappers for arch boot code to manipulate normally-constant masks */
  664. void init_cpu_present(const struct cpumask *src);
  665. void init_cpu_possible(const struct cpumask *src);
  666. void init_cpu_online(const struct cpumask *src);
  667. static inline void
  668. set_cpu_possible(unsigned int cpu, bool possible)
  669. {
  670. if (possible)
  671. cpumask_set_cpu(cpu, &__cpu_possible_mask);
  672. else
  673. cpumask_clear_cpu(cpu, &__cpu_possible_mask);
  674. }
  675. static inline void
  676. set_cpu_present(unsigned int cpu, bool present)
  677. {
  678. if (present)
  679. cpumask_set_cpu(cpu, &__cpu_present_mask);
  680. else
  681. cpumask_clear_cpu(cpu, &__cpu_present_mask);
  682. }
  683. static inline void
  684. set_cpu_online(unsigned int cpu, bool online)
  685. {
  686. if (online)
  687. cpumask_set_cpu(cpu, &__cpu_online_mask);
  688. else
  689. cpumask_clear_cpu(cpu, &__cpu_online_mask);
  690. }
  691. static inline void
  692. set_cpu_active(unsigned int cpu, bool active)
  693. {
  694. if (active)
  695. cpumask_set_cpu(cpu, &__cpu_active_mask);
  696. else
  697. cpumask_clear_cpu(cpu, &__cpu_active_mask);
  698. }
  699. /**
  700. * to_cpumask - convert an NR_CPUS bitmap to a struct cpumask *
  701. * @bitmap: the bitmap
  702. *
  703. * There are a few places where cpumask_var_t isn't appropriate and
  704. * static cpumasks must be used (eg. very early boot), yet we don't
  705. * expose the definition of 'struct cpumask'.
  706. *
  707. * This does the conversion, and can be used as a constant initializer.
  708. */
  709. #define to_cpumask(bitmap) \
  710. ((struct cpumask *)(1 ? (bitmap) \
  711. : (void *)sizeof(__check_is_bitmap(bitmap))))
  712. static inline int __check_is_bitmap(const unsigned long *bitmap)
  713. {
  714. return 1;
  715. }
  716. /*
  717. * Special-case data structure for "single bit set only" constant CPU masks.
  718. *
  719. * We pre-generate all the 64 (or 32) possible bit positions, with enough
  720. * padding to the left and the right, and return the constant pointer
  721. * appropriately offset.
  722. */
  723. extern const unsigned long
  724. cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
  725. static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
  726. {
  727. const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
  728. p -= cpu / BITS_PER_LONG;
  729. return to_cpumask(p);
  730. }
  731. #define cpu_is_offline(cpu) unlikely(!cpu_online(cpu))
  732. #if NR_CPUS <= BITS_PER_LONG
  733. #define CPU_BITS_ALL \
  734. { \
  735. [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
  736. }
  737. #else /* NR_CPUS > BITS_PER_LONG */
  738. #define CPU_BITS_ALL \
  739. { \
  740. [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
  741. [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
  742. }
  743. #endif /* NR_CPUS > BITS_PER_LONG */
  744. /**
  745. * cpumap_print_to_pagebuf - copies the cpumask into the buffer either
  746. * as comma-separated list of cpus or hex values of cpumask
  747. * @list: indicates whether the cpumap must be list
  748. * @mask: the cpumask to copy
  749. * @buf: the buffer to copy into
  750. *
  751. * Returns the length of the (null-terminated) @buf string, zero if
  752. * nothing is copied.
  753. */
  754. static inline ssize_t
  755. cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
  756. {
  757. return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),
  758. nr_cpu_ids);
  759. }
  760. #if NR_CPUS <= BITS_PER_LONG
  761. #define CPU_MASK_ALL \
  762. (cpumask_t) { { \
  763. [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
  764. } }
  765. #else
  766. #define CPU_MASK_ALL \
  767. (cpumask_t) { { \
  768. [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
  769. [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
  770. } }
  771. #endif /* NR_CPUS > BITS_PER_LONG */
  772. #define CPU_MASK_NONE \
  773. (cpumask_t) { { \
  774. [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
  775. } }
  776. #define CPU_MASK_CPU0 \
  777. (cpumask_t) { { \
  778. [0] = 1UL \
  779. } }
  780. #endif /* __LINUX_CPUMASK_H */