idle_book3s.S 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692
  1. /*
  2. * This file contains idle entry/exit functions for POWER7,
  3. * POWER8 and POWER9 CPUs.
  4. *
  5. * This program is free software; you can redistribute it and/or
  6. * modify it under the terms of the GNU General Public License
  7. * as published by the Free Software Foundation; either version
  8. * 2 of the License, or (at your option) any later version.
  9. */
  10. #include <linux/threads.h>
  11. #include <asm/processor.h>
  12. #include <asm/page.h>
  13. #include <asm/cputable.h>
  14. #include <asm/thread_info.h>
  15. #include <asm/ppc_asm.h>
  16. #include <asm/asm-offsets.h>
  17. #include <asm/ppc-opcode.h>
  18. #include <asm/hw_irq.h>
  19. #include <asm/kvm_book3s_asm.h>
  20. #include <asm/opal.h>
  21. #include <asm/cpuidle.h>
  22. #include <asm/book3s/64/mmu-hash.h>
  23. #include <asm/mmu.h>
  24. #undef DEBUG
  25. /*
  26. * Use unused space in the interrupt stack to save and restore
  27. * registers for winkle support.
  28. */
  29. #define _SDR1 GPR3
  30. #define _RPR GPR4
  31. #define _SPURR GPR5
  32. #define _PURR GPR6
  33. #define _TSCR GPR7
  34. #define _DSCR GPR8
  35. #define _AMOR GPR9
  36. #define _WORT GPR10
  37. #define _WORC GPR11
  38. #define _PTCR GPR12
  39. #define PSSCR_HV_TEMPLATE PSSCR_ESL | PSSCR_EC | \
  40. PSSCR_PSLL_MASK | PSSCR_TR_MASK | \
  41. PSSCR_MTL_MASK
  42. .text
  43. /*
  44. * Used by threads before entering deep idle states. Saves SPRs
  45. * in interrupt stack frame
  46. */
  47. save_sprs_to_stack:
  48. /*
  49. * Note all register i.e per-core, per-subcore or per-thread is saved
  50. * here since any thread in the core might wake up first
  51. */
  52. BEGIN_FTR_SECTION
  53. mfspr r3,SPRN_PTCR
  54. std r3,_PTCR(r1)
  55. /*
  56. * Note - SDR1 is dropped in Power ISA v3. Hence not restoring
  57. * SDR1 here
  58. */
  59. FTR_SECTION_ELSE
  60. mfspr r3,SPRN_SDR1
  61. std r3,_SDR1(r1)
  62. ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300)
  63. mfspr r3,SPRN_RPR
  64. std r3,_RPR(r1)
  65. mfspr r3,SPRN_SPURR
  66. std r3,_SPURR(r1)
  67. mfspr r3,SPRN_PURR
  68. std r3,_PURR(r1)
  69. mfspr r3,SPRN_TSCR
  70. std r3,_TSCR(r1)
  71. mfspr r3,SPRN_DSCR
  72. std r3,_DSCR(r1)
  73. mfspr r3,SPRN_AMOR
  74. std r3,_AMOR(r1)
  75. mfspr r3,SPRN_WORT
  76. std r3,_WORT(r1)
  77. mfspr r3,SPRN_WORC
  78. std r3,_WORC(r1)
  79. blr
  80. /*
  81. * Used by threads when the lock bit of core_idle_state is set.
  82. * Threads will spin in HMT_LOW until the lock bit is cleared.
  83. * r14 - pointer to core_idle_state
  84. * r15 - used to load contents of core_idle_state
  85. * r9 - used as a temporary variable
  86. */
  87. core_idle_lock_held:
  88. HMT_LOW
  89. 3: lwz r15,0(r14)
  90. andi. r15,r15,PNV_CORE_IDLE_LOCK_BIT
  91. bne 3b
  92. HMT_MEDIUM
  93. lwarx r15,0,r14
  94. andi. r9,r15,PNV_CORE_IDLE_LOCK_BIT
  95. bne core_idle_lock_held
  96. blr
  97. /*
  98. * Pass requested state in r3:
  99. * r3 - PNV_THREAD_NAP/SLEEP/WINKLE in POWER8
  100. * - Requested STOP state in POWER9
  101. *
  102. * To check IRQ_HAPPENED in r4
  103. * 0 - don't check
  104. * 1 - check
  105. *
  106. * Address to 'rfid' to in r5
  107. */
  108. _GLOBAL(pnv_powersave_common)
  109. /* Use r3 to pass state nap/sleep/winkle */
  110. /* NAP is a state loss, we create a regs frame on the
  111. * stack, fill it up with the state we care about and
  112. * stick a pointer to it in PACAR1. We really only
  113. * need to save PC, some CR bits and the NV GPRs,
  114. * but for now an interrupt frame will do.
  115. */
  116. mflr r0
  117. std r0,16(r1)
  118. stdu r1,-INT_FRAME_SIZE(r1)
  119. std r0,_LINK(r1)
  120. std r0,_NIP(r1)
  121. /* Hard disable interrupts */
  122. mfmsr r9
  123. rldicl r9,r9,48,1
  124. rotldi r9,r9,16
  125. mtmsrd r9,1 /* hard-disable interrupts */
  126. /* Check if something happened while soft-disabled */
  127. lbz r0,PACAIRQHAPPENED(r13)
  128. andi. r0,r0,~PACA_IRQ_HARD_DIS@l
  129. beq 1f
  130. cmpwi cr0,r4,0
  131. beq 1f
  132. addi r1,r1,INT_FRAME_SIZE
  133. ld r0,16(r1)
  134. li r3,0 /* Return 0 (no nap) */
  135. mtlr r0
  136. blr
  137. 1: /* We mark irqs hard disabled as this is the state we'll
  138. * be in when returning and we need to tell arch_local_irq_restore()
  139. * about it
  140. */
  141. li r0,PACA_IRQ_HARD_DIS
  142. stb r0,PACAIRQHAPPENED(r13)
  143. /* We haven't lost state ... yet */
  144. li r0,0
  145. stb r0,PACA_NAPSTATELOST(r13)
  146. /* Continue saving state */
  147. SAVE_GPR(2, r1)
  148. SAVE_NVGPRS(r1)
  149. mfcr r4
  150. std r4,_CCR(r1)
  151. std r9,_MSR(r1)
  152. std r1,PACAR1(r13)
  153. /*
  154. * Go to real mode to do the nap, as required by the architecture.
  155. * Also, we need to be in real mode before setting hwthread_state,
  156. * because as soon as we do that, another thread can switch
  157. * the MMU context to the guest.
  158. */
  159. LOAD_REG_IMMEDIATE(r7, MSR_IDLE)
  160. li r6, MSR_RI
  161. andc r6, r9, r6
  162. mtmsrd r6, 1 /* clear RI before setting SRR0/1 */
  163. mtspr SPRN_SRR0, r5
  164. mtspr SPRN_SRR1, r7
  165. rfid
  166. .globl pnv_enter_arch207_idle_mode
  167. pnv_enter_arch207_idle_mode:
  168. #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
  169. /* Tell KVM we're entering idle */
  170. li r4,KVM_HWTHREAD_IN_IDLE
  171. /******************************************************/
  172. /* N O T E W E L L ! ! ! N O T E W E L L */
  173. /* The following store to HSTATE_HWTHREAD_STATE(r13) */
  174. /* MUST occur in real mode, i.e. with the MMU off, */
  175. /* and the MMU must stay off until we clear this flag */
  176. /* and test HSTATE_HWTHREAD_REQ(r13) in the system */
  177. /* reset interrupt vector in exceptions-64s.S. */
  178. /* The reason is that another thread can switch the */
  179. /* MMU to a guest context whenever this flag is set */
  180. /* to KVM_HWTHREAD_IN_IDLE, and if the MMU was on, */
  181. /* that would potentially cause this thread to start */
  182. /* executing instructions from guest memory in */
  183. /* hypervisor mode, leading to a host crash or data */
  184. /* corruption, or worse. */
  185. /******************************************************/
  186. stb r4,HSTATE_HWTHREAD_STATE(r13)
  187. #endif
  188. stb r3,PACA_THREAD_IDLE_STATE(r13)
  189. cmpwi cr3,r3,PNV_THREAD_SLEEP
  190. bge cr3,2f
  191. IDLE_STATE_ENTER_SEQ(PPC_NAP)
  192. /* No return */
  193. 2:
  194. /* Sleep or winkle */
  195. lbz r7,PACA_THREAD_MASK(r13)
  196. ld r14,PACA_CORE_IDLE_STATE_PTR(r13)
  197. lwarx_loop1:
  198. lwarx r15,0,r14
  199. andi. r9,r15,PNV_CORE_IDLE_LOCK_BIT
  200. bnel core_idle_lock_held
  201. andc r15,r15,r7 /* Clear thread bit */
  202. andi. r15,r15,PNV_CORE_IDLE_THREAD_BITS
  203. /*
  204. * If cr0 = 0, then current thread is the last thread of the core entering
  205. * sleep. Last thread needs to execute the hardware bug workaround code if
  206. * required by the platform.
  207. * Make the workaround call unconditionally here. The below branch call is
  208. * patched out when the idle states are discovered if the platform does not
  209. * require it.
  210. */
  211. .global pnv_fastsleep_workaround_at_entry
  212. pnv_fastsleep_workaround_at_entry:
  213. beq fastsleep_workaround_at_entry
  214. stwcx. r15,0,r14
  215. bne- lwarx_loop1
  216. isync
  217. common_enter: /* common code for all the threads entering sleep or winkle */
  218. bgt cr3,enter_winkle
  219. IDLE_STATE_ENTER_SEQ(PPC_SLEEP)
  220. fastsleep_workaround_at_entry:
  221. ori r15,r15,PNV_CORE_IDLE_LOCK_BIT
  222. stwcx. r15,0,r14
  223. bne- lwarx_loop1
  224. isync
  225. /* Fast sleep workaround */
  226. li r3,1
  227. li r4,1
  228. bl opal_rm_config_cpu_idle_state
  229. /* Clear Lock bit */
  230. li r0,0
  231. lwsync
  232. stw r0,0(r14)
  233. b common_enter
  234. enter_winkle:
  235. bl save_sprs_to_stack
  236. IDLE_STATE_ENTER_SEQ(PPC_WINKLE)
  237. /*
  238. * r3 - requested stop state
  239. */
  240. power_enter_stop:
  241. #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
  242. /* Tell KVM we're entering idle */
  243. li r4,KVM_HWTHREAD_IN_IDLE
  244. /* DO THIS IN REAL MODE! See comment above. */
  245. stb r4,HSTATE_HWTHREAD_STATE(r13)
  246. #endif
  247. /*
  248. * Check if the requested state is a deep idle state.
  249. */
  250. LOAD_REG_ADDRBASE(r5,pnv_first_deep_stop_state)
  251. ld r4,ADDROFF(pnv_first_deep_stop_state)(r5)
  252. cmpd r3,r4
  253. bge 2f
  254. IDLE_STATE_ENTER_SEQ(PPC_STOP)
  255. 2:
  256. /*
  257. * Entering deep idle state.
  258. * Clear thread bit in PACA_CORE_IDLE_STATE, save SPRs to
  259. * stack and enter stop
  260. */
  261. lbz r7,PACA_THREAD_MASK(r13)
  262. ld r14,PACA_CORE_IDLE_STATE_PTR(r13)
  263. lwarx_loop_stop:
  264. lwarx r15,0,r14
  265. andi. r9,r15,PNV_CORE_IDLE_LOCK_BIT
  266. bnel core_idle_lock_held
  267. andc r15,r15,r7 /* Clear thread bit */
  268. stwcx. r15,0,r14
  269. bne- lwarx_loop_stop
  270. isync
  271. bl save_sprs_to_stack
  272. IDLE_STATE_ENTER_SEQ(PPC_STOP)
  273. _GLOBAL(power7_idle)
  274. /* Now check if user or arch enabled NAP mode */
  275. LOAD_REG_ADDRBASE(r3,powersave_nap)
  276. lwz r4,ADDROFF(powersave_nap)(r3)
  277. cmpwi 0,r4,0
  278. beqlr
  279. li r3, 1
  280. /* fall through */
  281. _GLOBAL(power7_nap)
  282. mr r4,r3
  283. li r3,PNV_THREAD_NAP
  284. LOAD_REG_ADDR(r5, pnv_enter_arch207_idle_mode)
  285. b pnv_powersave_common
  286. /* No return */
  287. _GLOBAL(power7_sleep)
  288. li r3,PNV_THREAD_SLEEP
  289. li r4,1
  290. LOAD_REG_ADDR(r5, pnv_enter_arch207_idle_mode)
  291. b pnv_powersave_common
  292. /* No return */
  293. _GLOBAL(power7_winkle)
  294. li r3,PNV_THREAD_WINKLE
  295. li r4,1
  296. LOAD_REG_ADDR(r5, pnv_enter_arch207_idle_mode)
  297. b pnv_powersave_common
  298. /* No return */
  299. #define CHECK_HMI_INTERRUPT \
  300. mfspr r0,SPRN_SRR1; \
  301. BEGIN_FTR_SECTION_NESTED(66); \
  302. rlwinm r0,r0,45-31,0xf; /* extract wake reason field (P8) */ \
  303. FTR_SECTION_ELSE_NESTED(66); \
  304. rlwinm r0,r0,45-31,0xe; /* P7 wake reason field is 3 bits */ \
  305. ALT_FTR_SECTION_END_NESTED_IFSET(CPU_FTR_ARCH_207S, 66); \
  306. cmpwi r0,0xa; /* Hypervisor maintenance ? */ \
  307. bne 20f; \
  308. /* Invoke opal call to handle hmi */ \
  309. ld r2,PACATOC(r13); \
  310. ld r1,PACAR1(r13); \
  311. std r3,ORIG_GPR3(r1); /* Save original r3 */ \
  312. li r3,0; /* NULL argument */ \
  313. bl hmi_exception_realmode; \
  314. nop; \
  315. ld r3,ORIG_GPR3(r1); /* Restore original r3 */ \
  316. 20: nop;
  317. /*
  318. * r3 - requested stop state
  319. */
  320. _GLOBAL(power9_idle_stop)
  321. LOAD_REG_IMMEDIATE(r4, PSSCR_HV_TEMPLATE)
  322. or r4,r4,r3
  323. mtspr SPRN_PSSCR, r4
  324. li r4, 1
  325. LOAD_REG_ADDR(r5,power_enter_stop)
  326. b pnv_powersave_common
  327. /* No return */
  328. /*
  329. * Called from reset vector. Check whether we have woken up with
  330. * hypervisor state loss. If yes, restore hypervisor state and return
  331. * back to reset vector.
  332. *
  333. * r13 - Contents of HSPRG0
  334. * cr3 - set to gt if waking up with partial/complete hypervisor state loss
  335. */
  336. _GLOBAL(pnv_restore_hyp_resource)
  337. BEGIN_FTR_SECTION
  338. ld r2,PACATOC(r13);
  339. /*
  340. * POWER ISA 3. Use PSSCR to determine if we
  341. * are waking up from deep idle state
  342. */
  343. LOAD_REG_ADDRBASE(r5,pnv_first_deep_stop_state)
  344. ld r4,ADDROFF(pnv_first_deep_stop_state)(r5)
  345. mfspr r5,SPRN_PSSCR
  346. /*
  347. * 0-3 bits correspond to Power-Saving Level Status
  348. * which indicates the idle state we are waking up from
  349. */
  350. rldicl r5,r5,4,60
  351. cmpd cr4,r5,r4
  352. bge cr4,pnv_wakeup_tb_loss
  353. /*
  354. * Waking up without hypervisor state loss. Return to
  355. * reset vector
  356. */
  357. blr
  358. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
  359. /*
  360. * POWER ISA 2.07 or less.
  361. * Check if last bit of HSPGR0 is set. This indicates whether we are
  362. * waking up from winkle.
  363. */
  364. clrldi r5,r13,63
  365. clrrdi r13,r13,1
  366. /* Now that we are sure r13 is corrected, load TOC */
  367. ld r2,PACATOC(r13);
  368. cmpwi cr4,r5,1
  369. mtspr SPRN_HSPRG0,r13
  370. lbz r0,PACA_THREAD_IDLE_STATE(r13)
  371. cmpwi cr2,r0,PNV_THREAD_NAP
  372. bgt cr2,pnv_wakeup_tb_loss /* Either sleep or Winkle */
  373. /*
  374. * We fall through here if PACA_THREAD_IDLE_STATE shows we are waking
  375. * up from nap. At this stage CR3 shouldn't contains 'gt' since that
  376. * indicates we are waking with hypervisor state loss from nap.
  377. */
  378. bgt cr3,.
  379. blr /* Return back to System Reset vector from where
  380. pnv_restore_hyp_resource was invoked */
  381. /*
  382. * Called if waking up from idle state which can cause either partial or
  383. * complete hyp state loss.
  384. * In POWER8, called if waking up from fastsleep or winkle
  385. * In POWER9, called if waking up from stop state >= pnv_first_deep_stop_state
  386. *
  387. * r13 - PACA
  388. * cr3 - gt if waking up with partial/complete hypervisor state loss
  389. * cr4 - gt or eq if waking up from complete hypervisor state loss.
  390. */
  391. _GLOBAL(pnv_wakeup_tb_loss)
  392. ld r1,PACAR1(r13)
  393. /*
  394. * Before entering any idle state, the NVGPRs are saved in the stack.
  395. * If there was a state loss, or PACA_NAPSTATELOST was set, then the
  396. * NVGPRs are restored. If we are here, it is likely that state is lost,
  397. * but not guaranteed -- neither ISA207 nor ISA300 tests to reach
  398. * here are the same as the test to restore NVGPRS:
  399. * PACA_THREAD_IDLE_STATE test for ISA207, PSSCR test for ISA300,
  400. * and SRR1 test for restoring NVGPRs.
  401. *
  402. * We are about to clobber NVGPRs now, so set NAPSTATELOST to
  403. * guarantee they will always be restored. This might be tightened
  404. * with careful reading of specs (particularly for ISA300) but this
  405. * is already a slow wakeup path and it's simpler to be safe.
  406. */
  407. li r0,1
  408. stb r0,PACA_NAPSTATELOST(r13)
  409. /*
  410. *
  411. * Save SRR1 and LR in NVGPRs as they might be clobbered in
  412. * opal_call() (called in CHECK_HMI_INTERRUPT). SRR1 is required
  413. * to determine the wakeup reason if we branch to kvm_start_guest. LR
  414. * is required to return back to reset vector after hypervisor state
  415. * restore is complete.
  416. */
  417. mflr r17
  418. mfspr r16,SPRN_SRR1
  419. BEGIN_FTR_SECTION
  420. CHECK_HMI_INTERRUPT
  421. END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
  422. lbz r7,PACA_THREAD_MASK(r13)
  423. ld r14,PACA_CORE_IDLE_STATE_PTR(r13)
  424. lwarx_loop2:
  425. lwarx r15,0,r14
  426. andi. r9,r15,PNV_CORE_IDLE_LOCK_BIT
  427. /*
  428. * Lock bit is set in one of the 2 cases-
  429. * a. In the sleep/winkle enter path, the last thread is executing
  430. * fastsleep workaround code.
  431. * b. In the wake up path, another thread is executing fastsleep
  432. * workaround undo code or resyncing timebase or restoring context
  433. * In either case loop until the lock bit is cleared.
  434. */
  435. bnel core_idle_lock_held
  436. cmpwi cr2,r15,0
  437. /*
  438. * At this stage
  439. * cr2 - eq if first thread to wakeup in core
  440. * cr3- gt if waking up with partial/complete hypervisor state loss
  441. * cr4 - gt or eq if waking up from complete hypervisor state loss.
  442. */
  443. ori r15,r15,PNV_CORE_IDLE_LOCK_BIT
  444. stwcx. r15,0,r14
  445. bne- lwarx_loop2
  446. isync
  447. BEGIN_FTR_SECTION
  448. lbz r4,PACA_SUBCORE_SIBLING_MASK(r13)
  449. and r4,r4,r15
  450. cmpwi r4,0 /* Check if first in subcore */
  451. or r15,r15,r7 /* Set thread bit */
  452. beq first_thread_in_subcore
  453. END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
  454. or r15,r15,r7 /* Set thread bit */
  455. beq cr2,first_thread_in_core
  456. /* Not first thread in core or subcore to wake up */
  457. b clear_lock
  458. first_thread_in_subcore:
  459. /*
  460. * If waking up from sleep, subcore state is not lost. Hence
  461. * skip subcore state restore
  462. */
  463. blt cr4,subcore_state_restored
  464. /* Restore per-subcore state */
  465. ld r4,_SDR1(r1)
  466. mtspr SPRN_SDR1,r4
  467. ld r4,_RPR(r1)
  468. mtspr SPRN_RPR,r4
  469. ld r4,_AMOR(r1)
  470. mtspr SPRN_AMOR,r4
  471. subcore_state_restored:
  472. /*
  473. * Check if the thread is also the first thread in the core. If not,
  474. * skip to clear_lock.
  475. */
  476. bne cr2,clear_lock
  477. first_thread_in_core:
  478. /*
  479. * First thread in the core waking up from any state which can cause
  480. * partial or complete hypervisor state loss. It needs to
  481. * call the fastsleep workaround code if the platform requires it.
  482. * Call it unconditionally here. The below branch instruction will
  483. * be patched out if the platform does not have fastsleep or does not
  484. * require the workaround. Patching will be performed during the
  485. * discovery of idle-states.
  486. */
  487. .global pnv_fastsleep_workaround_at_exit
  488. pnv_fastsleep_workaround_at_exit:
  489. b fastsleep_workaround_at_exit
  490. timebase_resync:
  491. /*
  492. * Use cr3 which indicates that we are waking up with atleast partial
  493. * hypervisor state loss to determine if TIMEBASE RESYNC is needed.
  494. */
  495. ble cr3,clear_lock
  496. /* Time base re-sync */
  497. bl opal_rm_resync_timebase;
  498. /*
  499. * If waking up from sleep, per core state is not lost, skip to
  500. * clear_lock.
  501. */
  502. blt cr4,clear_lock
  503. /*
  504. * First thread in the core to wake up and its waking up with
  505. * complete hypervisor state loss. Restore per core hypervisor
  506. * state.
  507. */
  508. BEGIN_FTR_SECTION
  509. ld r4,_PTCR(r1)
  510. mtspr SPRN_PTCR,r4
  511. ld r4,_RPR(r1)
  512. mtspr SPRN_RPR,r4
  513. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
  514. ld r4,_TSCR(r1)
  515. mtspr SPRN_TSCR,r4
  516. ld r4,_WORC(r1)
  517. mtspr SPRN_WORC,r4
  518. clear_lock:
  519. andi. r15,r15,PNV_CORE_IDLE_THREAD_BITS
  520. lwsync
  521. stw r15,0(r14)
  522. common_exit:
  523. /*
  524. * Common to all threads.
  525. *
  526. * If waking up from sleep, hypervisor state is not lost. Hence
  527. * skip hypervisor state restore.
  528. */
  529. blt cr4,hypervisor_state_restored
  530. /* Waking up from winkle */
  531. BEGIN_MMU_FTR_SECTION
  532. b no_segments
  533. END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
  534. /* Restore SLB from PACA */
  535. ld r8,PACA_SLBSHADOWPTR(r13)
  536. .rept SLB_NUM_BOLTED
  537. li r3, SLBSHADOW_SAVEAREA
  538. LDX_BE r5, r8, r3
  539. addi r3, r3, 8
  540. LDX_BE r6, r8, r3
  541. andis. r7,r5,SLB_ESID_V@h
  542. beq 1f
  543. slbmte r6,r5
  544. 1: addi r8,r8,16
  545. .endr
  546. no_segments:
  547. /* Restore per thread state */
  548. ld r4,_SPURR(r1)
  549. mtspr SPRN_SPURR,r4
  550. ld r4,_PURR(r1)
  551. mtspr SPRN_PURR,r4
  552. ld r4,_DSCR(r1)
  553. mtspr SPRN_DSCR,r4
  554. ld r4,_WORT(r1)
  555. mtspr SPRN_WORT,r4
  556. /* Call cur_cpu_spec->cpu_restore() */
  557. LOAD_REG_ADDR(r4, cur_cpu_spec)
  558. ld r4,0(r4)
  559. ld r12,CPU_SPEC_RESTORE(r4)
  560. #ifdef PPC64_ELF_ABI_v1
  561. ld r12,0(r12)
  562. #endif
  563. mtctr r12
  564. bctrl
  565. hypervisor_state_restored:
  566. mtspr SPRN_SRR1,r16
  567. mtlr r17
  568. blr /* Return back to System Reset vector from where
  569. pnv_restore_hyp_resource was invoked */
  570. fastsleep_workaround_at_exit:
  571. li r3,1
  572. li r4,0
  573. bl opal_rm_config_cpu_idle_state
  574. b timebase_resync
  575. /*
  576. * R3 here contains the value that will be returned to the caller
  577. * of power7_nap.
  578. */
  579. _GLOBAL(pnv_wakeup_loss)
  580. ld r1,PACAR1(r13)
  581. BEGIN_FTR_SECTION
  582. CHECK_HMI_INTERRUPT
  583. END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
  584. REST_NVGPRS(r1)
  585. REST_GPR(2, r1)
  586. ld r6,_CCR(r1)
  587. ld r4,_MSR(r1)
  588. ld r5,_NIP(r1)
  589. addi r1,r1,INT_FRAME_SIZE
  590. mtcr r6
  591. mtspr SPRN_SRR1,r4
  592. mtspr SPRN_SRR0,r5
  593. rfid
  594. /*
  595. * R3 here contains the value that will be returned to the caller
  596. * of power7_nap.
  597. */
  598. _GLOBAL(pnv_wakeup_noloss)
  599. lbz r0,PACA_NAPSTATELOST(r13)
  600. cmpwi r0,0
  601. bne pnv_wakeup_loss
  602. BEGIN_FTR_SECTION
  603. CHECK_HMI_INTERRUPT
  604. END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
  605. ld r1,PACAR1(r13)
  606. ld r6,_CCR(r1)
  607. ld r4,_MSR(r1)
  608. ld r5,_NIP(r1)
  609. addi r1,r1,INT_FRAME_SIZE
  610. mtcr r6
  611. mtspr SPRN_SRR1,r4
  612. mtspr SPRN_SRR0,r5
  613. rfid