filemap.c 78 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933
  1. /*
  2. * linux/mm/filemap.c
  3. *
  4. * Copyright (C) 1994-1999 Linus Torvalds
  5. */
  6. /*
  7. * This file handles the generic file mmap semantics used by
  8. * most "normal" filesystems (but you don't /have/ to use this:
  9. * the NFS filesystem used to do this differently, for example)
  10. */
  11. #include <linux/export.h>
  12. #include <linux/compiler.h>
  13. #include <linux/dax.h>
  14. #include <linux/fs.h>
  15. #include <linux/uaccess.h>
  16. #include <linux/capability.h>
  17. #include <linux/kernel_stat.h>
  18. #include <linux/gfp.h>
  19. #include <linux/mm.h>
  20. #include <linux/swap.h>
  21. #include <linux/mman.h>
  22. #include <linux/pagemap.h>
  23. #include <linux/file.h>
  24. #include <linux/uio.h>
  25. #include <linux/hash.h>
  26. #include <linux/writeback.h>
  27. #include <linux/backing-dev.h>
  28. #include <linux/pagevec.h>
  29. #include <linux/blkdev.h>
  30. #include <linux/security.h>
  31. #include <linux/cpuset.h>
  32. #include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */
  33. #include <linux/hugetlb.h>
  34. #include <linux/memcontrol.h>
  35. #include <linux/cleancache.h>
  36. #include <linux/rmap.h>
  37. #include "internal.h"
  38. #define CREATE_TRACE_POINTS
  39. #include <trace/events/filemap.h>
  40. /*
  41. * FIXME: remove all knowledge of the buffer layer from the core VM
  42. */
  43. #include <linux/buffer_head.h> /* for try_to_free_buffers */
  44. #include <asm/mman.h>
  45. /*
  46. * Shared mappings implemented 30.11.1994. It's not fully working yet,
  47. * though.
  48. *
  49. * Shared mappings now work. 15.8.1995 Bruno.
  50. *
  51. * finished 'unifying' the page and buffer cache and SMP-threaded the
  52. * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
  53. *
  54. * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
  55. */
  56. /*
  57. * Lock ordering:
  58. *
  59. * ->i_mmap_rwsem (truncate_pagecache)
  60. * ->private_lock (__free_pte->__set_page_dirty_buffers)
  61. * ->swap_lock (exclusive_swap_page, others)
  62. * ->mapping->tree_lock
  63. *
  64. * ->i_mutex
  65. * ->i_mmap_rwsem (truncate->unmap_mapping_range)
  66. *
  67. * ->mmap_sem
  68. * ->i_mmap_rwsem
  69. * ->page_table_lock or pte_lock (various, mainly in memory.c)
  70. * ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock)
  71. *
  72. * ->mmap_sem
  73. * ->lock_page (access_process_vm)
  74. *
  75. * ->i_mutex (generic_perform_write)
  76. * ->mmap_sem (fault_in_pages_readable->do_page_fault)
  77. *
  78. * bdi->wb.list_lock
  79. * sb_lock (fs/fs-writeback.c)
  80. * ->mapping->tree_lock (__sync_single_inode)
  81. *
  82. * ->i_mmap_rwsem
  83. * ->anon_vma.lock (vma_adjust)
  84. *
  85. * ->anon_vma.lock
  86. * ->page_table_lock or pte_lock (anon_vma_prepare and various)
  87. *
  88. * ->page_table_lock or pte_lock
  89. * ->swap_lock (try_to_unmap_one)
  90. * ->private_lock (try_to_unmap_one)
  91. * ->tree_lock (try_to_unmap_one)
  92. * ->zone_lru_lock(zone) (follow_page->mark_page_accessed)
  93. * ->zone_lru_lock(zone) (check_pte_range->isolate_lru_page)
  94. * ->private_lock (page_remove_rmap->set_page_dirty)
  95. * ->tree_lock (page_remove_rmap->set_page_dirty)
  96. * bdi.wb->list_lock (page_remove_rmap->set_page_dirty)
  97. * ->inode->i_lock (page_remove_rmap->set_page_dirty)
  98. * ->memcg->move_lock (page_remove_rmap->lock_page_memcg)
  99. * bdi.wb->list_lock (zap_pte_range->set_page_dirty)
  100. * ->inode->i_lock (zap_pte_range->set_page_dirty)
  101. * ->private_lock (zap_pte_range->__set_page_dirty_buffers)
  102. *
  103. * ->i_mmap_rwsem
  104. * ->tasklist_lock (memory_failure, collect_procs_ao)
  105. */
  106. static int page_cache_tree_insert(struct address_space *mapping,
  107. struct page *page, void **shadowp)
  108. {
  109. struct radix_tree_node *node;
  110. void **slot;
  111. int error;
  112. error = __radix_tree_create(&mapping->page_tree, page->index, 0,
  113. &node, &slot);
  114. if (error)
  115. return error;
  116. if (*slot) {
  117. void *p;
  118. p = radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
  119. if (!radix_tree_exceptional_entry(p))
  120. return -EEXIST;
  121. mapping->nrexceptional--;
  122. if (!dax_mapping(mapping)) {
  123. if (shadowp)
  124. *shadowp = p;
  125. if (node)
  126. workingset_node_shadows_dec(node);
  127. } else {
  128. /* DAX can replace empty locked entry with a hole */
  129. WARN_ON_ONCE(p !=
  130. (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY |
  131. RADIX_DAX_ENTRY_LOCK));
  132. /* DAX accounts exceptional entries as normal pages */
  133. if (node)
  134. workingset_node_pages_dec(node);
  135. /* Wakeup waiters for exceptional entry lock */
  136. dax_wake_mapping_entry_waiter(mapping, page->index,
  137. true);
  138. }
  139. }
  140. radix_tree_replace_slot(slot, page);
  141. mapping->nrpages++;
  142. if (node) {
  143. workingset_node_pages_inc(node);
  144. /*
  145. * Don't track node that contains actual pages.
  146. *
  147. * Avoid acquiring the list_lru lock if already
  148. * untracked. The list_empty() test is safe as
  149. * node->private_list is protected by
  150. * mapping->tree_lock.
  151. */
  152. if (!list_empty(&node->private_list))
  153. list_lru_del(&workingset_shadow_nodes,
  154. &node->private_list);
  155. }
  156. return 0;
  157. }
  158. static void page_cache_tree_delete(struct address_space *mapping,
  159. struct page *page, void *shadow)
  160. {
  161. int i, nr = PageHuge(page) ? 1 : hpage_nr_pages(page);
  162. VM_BUG_ON_PAGE(!PageLocked(page), page);
  163. VM_BUG_ON_PAGE(PageTail(page), page);
  164. VM_BUG_ON_PAGE(nr != 1 && shadow, page);
  165. for (i = 0; i < nr; i++) {
  166. struct radix_tree_node *node;
  167. void **slot;
  168. __radix_tree_lookup(&mapping->page_tree, page->index + i,
  169. &node, &slot);
  170. radix_tree_clear_tags(&mapping->page_tree, node, slot);
  171. if (!node) {
  172. VM_BUG_ON_PAGE(nr != 1, page);
  173. /*
  174. * We need a node to properly account shadow
  175. * entries. Don't plant any without. XXX
  176. */
  177. shadow = NULL;
  178. }
  179. radix_tree_replace_slot(slot, shadow);
  180. if (!node)
  181. break;
  182. workingset_node_pages_dec(node);
  183. if (shadow)
  184. workingset_node_shadows_inc(node);
  185. else
  186. if (__radix_tree_delete_node(&mapping->page_tree, node))
  187. continue;
  188. /*
  189. * Track node that only contains shadow entries. DAX mappings
  190. * contain no shadow entries and may contain other exceptional
  191. * entries so skip those.
  192. *
  193. * Avoid acquiring the list_lru lock if already tracked.
  194. * The list_empty() test is safe as node->private_list is
  195. * protected by mapping->tree_lock.
  196. */
  197. if (!dax_mapping(mapping) && !workingset_node_pages(node) &&
  198. list_empty(&node->private_list)) {
  199. node->private_data = mapping;
  200. list_lru_add(&workingset_shadow_nodes,
  201. &node->private_list);
  202. }
  203. }
  204. if (shadow) {
  205. mapping->nrexceptional += nr;
  206. /*
  207. * Make sure the nrexceptional update is committed before
  208. * the nrpages update so that final truncate racing
  209. * with reclaim does not see both counters 0 at the
  210. * same time and miss a shadow entry.
  211. */
  212. smp_wmb();
  213. }
  214. mapping->nrpages -= nr;
  215. }
  216. /*
  217. * Delete a page from the page cache and free it. Caller has to make
  218. * sure the page is locked and that nobody else uses it - or that usage
  219. * is safe. The caller must hold the mapping's tree_lock.
  220. */
  221. void __delete_from_page_cache(struct page *page, void *shadow)
  222. {
  223. struct address_space *mapping = page->mapping;
  224. int nr = hpage_nr_pages(page);
  225. trace_mm_filemap_delete_from_page_cache(page);
  226. /*
  227. * if we're uptodate, flush out into the cleancache, otherwise
  228. * invalidate any existing cleancache entries. We can't leave
  229. * stale data around in the cleancache once our page is gone
  230. */
  231. if (PageUptodate(page) && PageMappedToDisk(page))
  232. cleancache_put_page(page);
  233. else
  234. cleancache_invalidate_page(mapping, page);
  235. VM_BUG_ON_PAGE(PageTail(page), page);
  236. VM_BUG_ON_PAGE(page_mapped(page), page);
  237. if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(page_mapped(page))) {
  238. int mapcount;
  239. pr_alert("BUG: Bad page cache in process %s pfn:%05lx\n",
  240. current->comm, page_to_pfn(page));
  241. dump_page(page, "still mapped when deleted");
  242. dump_stack();
  243. add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
  244. mapcount = page_mapcount(page);
  245. if (mapping_exiting(mapping) &&
  246. page_count(page) >= mapcount + 2) {
  247. /*
  248. * All vmas have already been torn down, so it's
  249. * a good bet that actually the page is unmapped,
  250. * and we'd prefer not to leak it: if we're wrong,
  251. * some other bad page check should catch it later.
  252. */
  253. page_mapcount_reset(page);
  254. page_ref_sub(page, mapcount);
  255. }
  256. }
  257. page_cache_tree_delete(mapping, page, shadow);
  258. page->mapping = NULL;
  259. /* Leave page->index set: truncation lookup relies upon it */
  260. /* hugetlb pages do not participate in page cache accounting. */
  261. if (!PageHuge(page))
  262. __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, -nr);
  263. if (PageSwapBacked(page)) {
  264. __mod_node_page_state(page_pgdat(page), NR_SHMEM, -nr);
  265. if (PageTransHuge(page))
  266. __dec_node_page_state(page, NR_SHMEM_THPS);
  267. } else {
  268. VM_BUG_ON_PAGE(PageTransHuge(page) && !PageHuge(page), page);
  269. }
  270. /*
  271. * At this point page must be either written or cleaned by truncate.
  272. * Dirty page here signals a bug and loss of unwritten data.
  273. *
  274. * This fixes dirty accounting after removing the page entirely but
  275. * leaves PageDirty set: it has no effect for truncated page and
  276. * anyway will be cleared before returning page into buddy allocator.
  277. */
  278. if (WARN_ON_ONCE(PageDirty(page)))
  279. account_page_cleaned(page, mapping, inode_to_wb(mapping->host));
  280. }
  281. /**
  282. * delete_from_page_cache - delete page from page cache
  283. * @page: the page which the kernel is trying to remove from page cache
  284. *
  285. * This must be called only on pages that have been verified to be in the page
  286. * cache and locked. It will never put the page into the free list, the caller
  287. * has a reference on the page.
  288. */
  289. void delete_from_page_cache(struct page *page)
  290. {
  291. struct address_space *mapping = page_mapping(page);
  292. unsigned long flags;
  293. void (*freepage)(struct page *);
  294. BUG_ON(!PageLocked(page));
  295. freepage = mapping->a_ops->freepage;
  296. spin_lock_irqsave(&mapping->tree_lock, flags);
  297. __delete_from_page_cache(page, NULL);
  298. spin_unlock_irqrestore(&mapping->tree_lock, flags);
  299. if (freepage)
  300. freepage(page);
  301. if (PageTransHuge(page) && !PageHuge(page)) {
  302. page_ref_sub(page, HPAGE_PMD_NR);
  303. VM_BUG_ON_PAGE(page_count(page) <= 0, page);
  304. } else {
  305. put_page(page);
  306. }
  307. }
  308. EXPORT_SYMBOL(delete_from_page_cache);
  309. int filemap_check_errors(struct address_space *mapping)
  310. {
  311. int ret = 0;
  312. /* Check for outstanding write errors */
  313. if (test_bit(AS_ENOSPC, &mapping->flags) &&
  314. test_and_clear_bit(AS_ENOSPC, &mapping->flags))
  315. ret = -ENOSPC;
  316. if (test_bit(AS_EIO, &mapping->flags) &&
  317. test_and_clear_bit(AS_EIO, &mapping->flags))
  318. ret = -EIO;
  319. return ret;
  320. }
  321. EXPORT_SYMBOL(filemap_check_errors);
  322. /**
  323. * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
  324. * @mapping: address space structure to write
  325. * @start: offset in bytes where the range starts
  326. * @end: offset in bytes where the range ends (inclusive)
  327. * @sync_mode: enable synchronous operation
  328. *
  329. * Start writeback against all of a mapping's dirty pages that lie
  330. * within the byte offsets <start, end> inclusive.
  331. *
  332. * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
  333. * opposed to a regular memory cleansing writeback. The difference between
  334. * these two operations is that if a dirty page/buffer is encountered, it must
  335. * be waited upon, and not just skipped over.
  336. */
  337. int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
  338. loff_t end, int sync_mode)
  339. {
  340. int ret;
  341. struct writeback_control wbc = {
  342. .sync_mode = sync_mode,
  343. .nr_to_write = LONG_MAX,
  344. .range_start = start,
  345. .range_end = end,
  346. };
  347. if (!mapping_cap_writeback_dirty(mapping))
  348. return 0;
  349. wbc_attach_fdatawrite_inode(&wbc, mapping->host);
  350. ret = do_writepages(mapping, &wbc);
  351. wbc_detach_inode(&wbc);
  352. return ret;
  353. }
  354. static inline int __filemap_fdatawrite(struct address_space *mapping,
  355. int sync_mode)
  356. {
  357. return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
  358. }
  359. int filemap_fdatawrite(struct address_space *mapping)
  360. {
  361. return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
  362. }
  363. EXPORT_SYMBOL(filemap_fdatawrite);
  364. int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
  365. loff_t end)
  366. {
  367. return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
  368. }
  369. EXPORT_SYMBOL(filemap_fdatawrite_range);
  370. /**
  371. * filemap_flush - mostly a non-blocking flush
  372. * @mapping: target address_space
  373. *
  374. * This is a mostly non-blocking flush. Not suitable for data-integrity
  375. * purposes - I/O may not be started against all dirty pages.
  376. */
  377. int filemap_flush(struct address_space *mapping)
  378. {
  379. return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
  380. }
  381. EXPORT_SYMBOL(filemap_flush);
  382. static int __filemap_fdatawait_range(struct address_space *mapping,
  383. loff_t start_byte, loff_t end_byte)
  384. {
  385. pgoff_t index = start_byte >> PAGE_SHIFT;
  386. pgoff_t end = end_byte >> PAGE_SHIFT;
  387. struct pagevec pvec;
  388. int nr_pages;
  389. int ret = 0;
  390. if (end_byte < start_byte)
  391. goto out;
  392. pagevec_init(&pvec, 0);
  393. while ((index <= end) &&
  394. (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
  395. PAGECACHE_TAG_WRITEBACK,
  396. min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1)) != 0) {
  397. unsigned i;
  398. for (i = 0; i < nr_pages; i++) {
  399. struct page *page = pvec.pages[i];
  400. /* until radix tree lookup accepts end_index */
  401. if (page->index > end)
  402. continue;
  403. wait_on_page_writeback(page);
  404. if (TestClearPageError(page))
  405. ret = -EIO;
  406. }
  407. pagevec_release(&pvec);
  408. cond_resched();
  409. }
  410. out:
  411. return ret;
  412. }
  413. /**
  414. * filemap_fdatawait_range - wait for writeback to complete
  415. * @mapping: address space structure to wait for
  416. * @start_byte: offset in bytes where the range starts
  417. * @end_byte: offset in bytes where the range ends (inclusive)
  418. *
  419. * Walk the list of under-writeback pages of the given address space
  420. * in the given range and wait for all of them. Check error status of
  421. * the address space and return it.
  422. *
  423. * Since the error status of the address space is cleared by this function,
  424. * callers are responsible for checking the return value and handling and/or
  425. * reporting the error.
  426. */
  427. int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte,
  428. loff_t end_byte)
  429. {
  430. int ret, ret2;
  431. ret = __filemap_fdatawait_range(mapping, start_byte, end_byte);
  432. ret2 = filemap_check_errors(mapping);
  433. if (!ret)
  434. ret = ret2;
  435. return ret;
  436. }
  437. EXPORT_SYMBOL(filemap_fdatawait_range);
  438. /**
  439. * filemap_fdatawait_keep_errors - wait for writeback without clearing errors
  440. * @mapping: address space structure to wait for
  441. *
  442. * Walk the list of under-writeback pages of the given address space
  443. * and wait for all of them. Unlike filemap_fdatawait(), this function
  444. * does not clear error status of the address space.
  445. *
  446. * Use this function if callers don't handle errors themselves. Expected
  447. * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
  448. * fsfreeze(8)
  449. */
  450. void filemap_fdatawait_keep_errors(struct address_space *mapping)
  451. {
  452. loff_t i_size = i_size_read(mapping->host);
  453. if (i_size == 0)
  454. return;
  455. __filemap_fdatawait_range(mapping, 0, i_size - 1);
  456. }
  457. /**
  458. * filemap_fdatawait - wait for all under-writeback pages to complete
  459. * @mapping: address space structure to wait for
  460. *
  461. * Walk the list of under-writeback pages of the given address space
  462. * and wait for all of them. Check error status of the address space
  463. * and return it.
  464. *
  465. * Since the error status of the address space is cleared by this function,
  466. * callers are responsible for checking the return value and handling and/or
  467. * reporting the error.
  468. */
  469. int filemap_fdatawait(struct address_space *mapping)
  470. {
  471. loff_t i_size = i_size_read(mapping->host);
  472. if (i_size == 0)
  473. return 0;
  474. return filemap_fdatawait_range(mapping, 0, i_size - 1);
  475. }
  476. EXPORT_SYMBOL(filemap_fdatawait);
  477. int filemap_write_and_wait(struct address_space *mapping)
  478. {
  479. int err = 0;
  480. if ((!dax_mapping(mapping) && mapping->nrpages) ||
  481. (dax_mapping(mapping) && mapping->nrexceptional)) {
  482. err = filemap_fdatawrite(mapping);
  483. /*
  484. * Even if the above returned error, the pages may be
  485. * written partially (e.g. -ENOSPC), so we wait for it.
  486. * But the -EIO is special case, it may indicate the worst
  487. * thing (e.g. bug) happened, so we avoid waiting for it.
  488. */
  489. if (err != -EIO) {
  490. int err2 = filemap_fdatawait(mapping);
  491. if (!err)
  492. err = err2;
  493. }
  494. } else {
  495. err = filemap_check_errors(mapping);
  496. }
  497. return err;
  498. }
  499. EXPORT_SYMBOL(filemap_write_and_wait);
  500. /**
  501. * filemap_write_and_wait_range - write out & wait on a file range
  502. * @mapping: the address_space for the pages
  503. * @lstart: offset in bytes where the range starts
  504. * @lend: offset in bytes where the range ends (inclusive)
  505. *
  506. * Write out and wait upon file offsets lstart->lend, inclusive.
  507. *
  508. * Note that `lend' is inclusive (describes the last byte to be written) so
  509. * that this function can be used to write to the very end-of-file (end = -1).
  510. */
  511. int filemap_write_and_wait_range(struct address_space *mapping,
  512. loff_t lstart, loff_t lend)
  513. {
  514. int err = 0;
  515. if ((!dax_mapping(mapping) && mapping->nrpages) ||
  516. (dax_mapping(mapping) && mapping->nrexceptional)) {
  517. err = __filemap_fdatawrite_range(mapping, lstart, lend,
  518. WB_SYNC_ALL);
  519. /* See comment of filemap_write_and_wait() */
  520. if (err != -EIO) {
  521. int err2 = filemap_fdatawait_range(mapping,
  522. lstart, lend);
  523. if (!err)
  524. err = err2;
  525. }
  526. } else {
  527. err = filemap_check_errors(mapping);
  528. }
  529. return err;
  530. }
  531. EXPORT_SYMBOL(filemap_write_and_wait_range);
  532. /**
  533. * replace_page_cache_page - replace a pagecache page with a new one
  534. * @old: page to be replaced
  535. * @new: page to replace with
  536. * @gfp_mask: allocation mode
  537. *
  538. * This function replaces a page in the pagecache with a new one. On
  539. * success it acquires the pagecache reference for the new page and
  540. * drops it for the old page. Both the old and new pages must be
  541. * locked. This function does not add the new page to the LRU, the
  542. * caller must do that.
  543. *
  544. * The remove + add is atomic. The only way this function can fail is
  545. * memory allocation failure.
  546. */
  547. int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
  548. {
  549. int error;
  550. VM_BUG_ON_PAGE(!PageLocked(old), old);
  551. VM_BUG_ON_PAGE(!PageLocked(new), new);
  552. VM_BUG_ON_PAGE(new->mapping, new);
  553. error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM);
  554. if (!error) {
  555. struct address_space *mapping = old->mapping;
  556. void (*freepage)(struct page *);
  557. unsigned long flags;
  558. pgoff_t offset = old->index;
  559. freepage = mapping->a_ops->freepage;
  560. get_page(new);
  561. new->mapping = mapping;
  562. new->index = offset;
  563. spin_lock_irqsave(&mapping->tree_lock, flags);
  564. __delete_from_page_cache(old, NULL);
  565. error = page_cache_tree_insert(mapping, new, NULL);
  566. BUG_ON(error);
  567. /*
  568. * hugetlb pages do not participate in page cache accounting.
  569. */
  570. if (!PageHuge(new))
  571. __inc_node_page_state(new, NR_FILE_PAGES);
  572. if (PageSwapBacked(new))
  573. __inc_node_page_state(new, NR_SHMEM);
  574. spin_unlock_irqrestore(&mapping->tree_lock, flags);
  575. mem_cgroup_migrate(old, new);
  576. radix_tree_preload_end();
  577. if (freepage)
  578. freepage(old);
  579. put_page(old);
  580. }
  581. return error;
  582. }
  583. EXPORT_SYMBOL_GPL(replace_page_cache_page);
  584. static int __add_to_page_cache_locked(struct page *page,
  585. struct address_space *mapping,
  586. pgoff_t offset, gfp_t gfp_mask,
  587. void **shadowp)
  588. {
  589. int huge = PageHuge(page);
  590. struct mem_cgroup *memcg;
  591. int error;
  592. VM_BUG_ON_PAGE(!PageLocked(page), page);
  593. VM_BUG_ON_PAGE(PageSwapBacked(page), page);
  594. if (!huge) {
  595. error = mem_cgroup_try_charge(page, current->mm,
  596. gfp_mask, &memcg, false);
  597. if (error)
  598. return error;
  599. }
  600. error = radix_tree_maybe_preload(gfp_mask & ~__GFP_HIGHMEM);
  601. if (error) {
  602. if (!huge)
  603. mem_cgroup_cancel_charge(page, memcg, false);
  604. return error;
  605. }
  606. get_page(page);
  607. page->mapping = mapping;
  608. page->index = offset;
  609. spin_lock_irq(&mapping->tree_lock);
  610. error = page_cache_tree_insert(mapping, page, shadowp);
  611. radix_tree_preload_end();
  612. if (unlikely(error))
  613. goto err_insert;
  614. /* hugetlb pages do not participate in page cache accounting. */
  615. if (!huge)
  616. __inc_node_page_state(page, NR_FILE_PAGES);
  617. spin_unlock_irq(&mapping->tree_lock);
  618. if (!huge)
  619. mem_cgroup_commit_charge(page, memcg, false, false);
  620. trace_mm_filemap_add_to_page_cache(page);
  621. return 0;
  622. err_insert:
  623. page->mapping = NULL;
  624. /* Leave page->index set: truncation relies upon it */
  625. spin_unlock_irq(&mapping->tree_lock);
  626. if (!huge)
  627. mem_cgroup_cancel_charge(page, memcg, false);
  628. put_page(page);
  629. return error;
  630. }
  631. /**
  632. * add_to_page_cache_locked - add a locked page to the pagecache
  633. * @page: page to add
  634. * @mapping: the page's address_space
  635. * @offset: page index
  636. * @gfp_mask: page allocation mode
  637. *
  638. * This function is used to add a page to the pagecache. It must be locked.
  639. * This function does not add the page to the LRU. The caller must do that.
  640. */
  641. int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
  642. pgoff_t offset, gfp_t gfp_mask)
  643. {
  644. return __add_to_page_cache_locked(page, mapping, offset,
  645. gfp_mask, NULL);
  646. }
  647. EXPORT_SYMBOL(add_to_page_cache_locked);
  648. int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
  649. pgoff_t offset, gfp_t gfp_mask)
  650. {
  651. void *shadow = NULL;
  652. int ret;
  653. __SetPageLocked(page);
  654. ret = __add_to_page_cache_locked(page, mapping, offset,
  655. gfp_mask, &shadow);
  656. if (unlikely(ret))
  657. __ClearPageLocked(page);
  658. else {
  659. /*
  660. * The page might have been evicted from cache only
  661. * recently, in which case it should be activated like
  662. * any other repeatedly accessed page.
  663. * The exception is pages getting rewritten; evicting other
  664. * data from the working set, only to cache data that will
  665. * get overwritten with something else, is a waste of memory.
  666. */
  667. if (!(gfp_mask & __GFP_WRITE) &&
  668. shadow && workingset_refault(shadow)) {
  669. SetPageActive(page);
  670. workingset_activation(page);
  671. } else
  672. ClearPageActive(page);
  673. lru_cache_add(page);
  674. }
  675. return ret;
  676. }
  677. EXPORT_SYMBOL_GPL(add_to_page_cache_lru);
  678. #ifdef CONFIG_NUMA
  679. struct page *__page_cache_alloc(gfp_t gfp)
  680. {
  681. int n;
  682. struct page *page;
  683. if (cpuset_do_page_mem_spread()) {
  684. unsigned int cpuset_mems_cookie;
  685. do {
  686. cpuset_mems_cookie = read_mems_allowed_begin();
  687. n = cpuset_mem_spread_node();
  688. page = __alloc_pages_node(n, gfp, 0);
  689. } while (!page && read_mems_allowed_retry(cpuset_mems_cookie));
  690. return page;
  691. }
  692. return alloc_pages(gfp, 0);
  693. }
  694. EXPORT_SYMBOL(__page_cache_alloc);
  695. #endif
  696. /*
  697. * In order to wait for pages to become available there must be
  698. * waitqueues associated with pages. By using a hash table of
  699. * waitqueues where the bucket discipline is to maintain all
  700. * waiters on the same queue and wake all when any of the pages
  701. * become available, and for the woken contexts to check to be
  702. * sure the appropriate page became available, this saves space
  703. * at a cost of "thundering herd" phenomena during rare hash
  704. * collisions.
  705. */
  706. wait_queue_head_t *page_waitqueue(struct page *page)
  707. {
  708. return bit_waitqueue(page, 0);
  709. }
  710. EXPORT_SYMBOL(page_waitqueue);
  711. void wait_on_page_bit(struct page *page, int bit_nr)
  712. {
  713. DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);
  714. if (test_bit(bit_nr, &page->flags))
  715. __wait_on_bit(page_waitqueue(page), &wait, bit_wait_io,
  716. TASK_UNINTERRUPTIBLE);
  717. }
  718. EXPORT_SYMBOL(wait_on_page_bit);
  719. int wait_on_page_bit_killable(struct page *page, int bit_nr)
  720. {
  721. DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);
  722. if (!test_bit(bit_nr, &page->flags))
  723. return 0;
  724. return __wait_on_bit(page_waitqueue(page), &wait,
  725. bit_wait_io, TASK_KILLABLE);
  726. }
  727. int wait_on_page_bit_killable_timeout(struct page *page,
  728. int bit_nr, unsigned long timeout)
  729. {
  730. DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);
  731. wait.key.timeout = jiffies + timeout;
  732. if (!test_bit(bit_nr, &page->flags))
  733. return 0;
  734. return __wait_on_bit(page_waitqueue(page), &wait,
  735. bit_wait_io_timeout, TASK_KILLABLE);
  736. }
  737. EXPORT_SYMBOL_GPL(wait_on_page_bit_killable_timeout);
  738. /**
  739. * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue
  740. * @page: Page defining the wait queue of interest
  741. * @waiter: Waiter to add to the queue
  742. *
  743. * Add an arbitrary @waiter to the wait queue for the nominated @page.
  744. */
  745. void add_page_wait_queue(struct page *page, wait_queue_t *waiter)
  746. {
  747. wait_queue_head_t *q = page_waitqueue(page);
  748. unsigned long flags;
  749. spin_lock_irqsave(&q->lock, flags);
  750. __add_wait_queue(q, waiter);
  751. spin_unlock_irqrestore(&q->lock, flags);
  752. }
  753. EXPORT_SYMBOL_GPL(add_page_wait_queue);
  754. /**
  755. * unlock_page - unlock a locked page
  756. * @page: the page
  757. *
  758. * Unlocks the page and wakes up sleepers in ___wait_on_page_locked().
  759. * Also wakes sleepers in wait_on_page_writeback() because the wakeup
  760. * mechanism between PageLocked pages and PageWriteback pages is shared.
  761. * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
  762. *
  763. * The mb is necessary to enforce ordering between the clear_bit and the read
  764. * of the waitqueue (to avoid SMP races with a parallel wait_on_page_locked()).
  765. */
  766. void unlock_page(struct page *page)
  767. {
  768. page = compound_head(page);
  769. VM_BUG_ON_PAGE(!PageLocked(page), page);
  770. clear_bit_unlock(PG_locked, &page->flags);
  771. smp_mb__after_atomic();
  772. wake_up_page(page, PG_locked);
  773. }
  774. EXPORT_SYMBOL(unlock_page);
  775. /**
  776. * end_page_writeback - end writeback against a page
  777. * @page: the page
  778. */
  779. void end_page_writeback(struct page *page)
  780. {
  781. /*
  782. * TestClearPageReclaim could be used here but it is an atomic
  783. * operation and overkill in this particular case. Failing to
  784. * shuffle a page marked for immediate reclaim is too mild to
  785. * justify taking an atomic operation penalty at the end of
  786. * ever page writeback.
  787. */
  788. if (PageReclaim(page)) {
  789. ClearPageReclaim(page);
  790. rotate_reclaimable_page(page);
  791. }
  792. if (!test_clear_page_writeback(page))
  793. BUG();
  794. smp_mb__after_atomic();
  795. wake_up_page(page, PG_writeback);
  796. }
  797. EXPORT_SYMBOL(end_page_writeback);
  798. /*
  799. * After completing I/O on a page, call this routine to update the page
  800. * flags appropriately
  801. */
  802. void page_endio(struct page *page, bool is_write, int err)
  803. {
  804. if (!is_write) {
  805. if (!err) {
  806. SetPageUptodate(page);
  807. } else {
  808. ClearPageUptodate(page);
  809. SetPageError(page);
  810. }
  811. unlock_page(page);
  812. } else {
  813. if (err) {
  814. struct address_space *mapping;
  815. SetPageError(page);
  816. mapping = page_mapping(page);
  817. if (mapping)
  818. mapping_set_error(mapping, err);
  819. }
  820. end_page_writeback(page);
  821. }
  822. }
  823. EXPORT_SYMBOL_GPL(page_endio);
  824. /**
  825. * __lock_page - get a lock on the page, assuming we need to sleep to get it
  826. * @page: the page to lock
  827. */
  828. void __lock_page(struct page *page)
  829. {
  830. struct page *page_head = compound_head(page);
  831. DEFINE_WAIT_BIT(wait, &page_head->flags, PG_locked);
  832. __wait_on_bit_lock(page_waitqueue(page_head), &wait, bit_wait_io,
  833. TASK_UNINTERRUPTIBLE);
  834. }
  835. EXPORT_SYMBOL(__lock_page);
  836. int __lock_page_killable(struct page *page)
  837. {
  838. struct page *page_head = compound_head(page);
  839. DEFINE_WAIT_BIT(wait, &page_head->flags, PG_locked);
  840. return __wait_on_bit_lock(page_waitqueue(page_head), &wait,
  841. bit_wait_io, TASK_KILLABLE);
  842. }
  843. EXPORT_SYMBOL_GPL(__lock_page_killable);
  844. /*
  845. * Return values:
  846. * 1 - page is locked; mmap_sem is still held.
  847. * 0 - page is not locked.
  848. * mmap_sem has been released (up_read()), unless flags had both
  849. * FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_RETRY_NOWAIT set, in
  850. * which case mmap_sem is still held.
  851. *
  852. * If neither ALLOW_RETRY nor KILLABLE are set, will always return 1
  853. * with the page locked and the mmap_sem unperturbed.
  854. */
  855. int __lock_page_or_retry(struct page *page, struct mm_struct *mm,
  856. unsigned int flags)
  857. {
  858. if (flags & FAULT_FLAG_ALLOW_RETRY) {
  859. /*
  860. * CAUTION! In this case, mmap_sem is not released
  861. * even though return 0.
  862. */
  863. if (flags & FAULT_FLAG_RETRY_NOWAIT)
  864. return 0;
  865. up_read(&mm->mmap_sem);
  866. if (flags & FAULT_FLAG_KILLABLE)
  867. wait_on_page_locked_killable(page);
  868. else
  869. wait_on_page_locked(page);
  870. return 0;
  871. } else {
  872. if (flags & FAULT_FLAG_KILLABLE) {
  873. int ret;
  874. ret = __lock_page_killable(page);
  875. if (ret) {
  876. up_read(&mm->mmap_sem);
  877. return 0;
  878. }
  879. } else
  880. __lock_page(page);
  881. return 1;
  882. }
  883. }
  884. /**
  885. * page_cache_next_hole - find the next hole (not-present entry)
  886. * @mapping: mapping
  887. * @index: index
  888. * @max_scan: maximum range to search
  889. *
  890. * Search the set [index, min(index+max_scan-1, MAX_INDEX)] for the
  891. * lowest indexed hole.
  892. *
  893. * Returns: the index of the hole if found, otherwise returns an index
  894. * outside of the set specified (in which case 'return - index >=
  895. * max_scan' will be true). In rare cases of index wrap-around, 0 will
  896. * be returned.
  897. *
  898. * page_cache_next_hole may be called under rcu_read_lock. However,
  899. * like radix_tree_gang_lookup, this will not atomically search a
  900. * snapshot of the tree at a single point in time. For example, if a
  901. * hole is created at index 5, then subsequently a hole is created at
  902. * index 10, page_cache_next_hole covering both indexes may return 10
  903. * if called under rcu_read_lock.
  904. */
  905. pgoff_t page_cache_next_hole(struct address_space *mapping,
  906. pgoff_t index, unsigned long max_scan)
  907. {
  908. unsigned long i;
  909. for (i = 0; i < max_scan; i++) {
  910. struct page *page;
  911. page = radix_tree_lookup(&mapping->page_tree, index);
  912. if (!page || radix_tree_exceptional_entry(page))
  913. break;
  914. index++;
  915. if (index == 0)
  916. break;
  917. }
  918. return index;
  919. }
  920. EXPORT_SYMBOL(page_cache_next_hole);
  921. /**
  922. * page_cache_prev_hole - find the prev hole (not-present entry)
  923. * @mapping: mapping
  924. * @index: index
  925. * @max_scan: maximum range to search
  926. *
  927. * Search backwards in the range [max(index-max_scan+1, 0), index] for
  928. * the first hole.
  929. *
  930. * Returns: the index of the hole if found, otherwise returns an index
  931. * outside of the set specified (in which case 'index - return >=
  932. * max_scan' will be true). In rare cases of wrap-around, ULONG_MAX
  933. * will be returned.
  934. *
  935. * page_cache_prev_hole may be called under rcu_read_lock. However,
  936. * like radix_tree_gang_lookup, this will not atomically search a
  937. * snapshot of the tree at a single point in time. For example, if a
  938. * hole is created at index 10, then subsequently a hole is created at
  939. * index 5, page_cache_prev_hole covering both indexes may return 5 if
  940. * called under rcu_read_lock.
  941. */
  942. pgoff_t page_cache_prev_hole(struct address_space *mapping,
  943. pgoff_t index, unsigned long max_scan)
  944. {
  945. unsigned long i;
  946. for (i = 0; i < max_scan; i++) {
  947. struct page *page;
  948. page = radix_tree_lookup(&mapping->page_tree, index);
  949. if (!page || radix_tree_exceptional_entry(page))
  950. break;
  951. index--;
  952. if (index == ULONG_MAX)
  953. break;
  954. }
  955. return index;
  956. }
  957. EXPORT_SYMBOL(page_cache_prev_hole);
  958. /**
  959. * find_get_entry - find and get a page cache entry
  960. * @mapping: the address_space to search
  961. * @offset: the page cache index
  962. *
  963. * Looks up the page cache slot at @mapping & @offset. If there is a
  964. * page cache page, it is returned with an increased refcount.
  965. *
  966. * If the slot holds a shadow entry of a previously evicted page, or a
  967. * swap entry from shmem/tmpfs, it is returned.
  968. *
  969. * Otherwise, %NULL is returned.
  970. */
  971. struct page *find_get_entry(struct address_space *mapping, pgoff_t offset)
  972. {
  973. void **pagep;
  974. struct page *head, *page;
  975. rcu_read_lock();
  976. repeat:
  977. page = NULL;
  978. pagep = radix_tree_lookup_slot(&mapping->page_tree, offset);
  979. if (pagep) {
  980. page = radix_tree_deref_slot(pagep);
  981. if (unlikely(!page))
  982. goto out;
  983. if (radix_tree_exception(page)) {
  984. if (radix_tree_deref_retry(page))
  985. goto repeat;
  986. /*
  987. * A shadow entry of a recently evicted page,
  988. * or a swap entry from shmem/tmpfs. Return
  989. * it without attempting to raise page count.
  990. */
  991. goto out;
  992. }
  993. head = compound_head(page);
  994. if (!page_cache_get_speculative(head))
  995. goto repeat;
  996. /* The page was split under us? */
  997. if (compound_head(page) != head) {
  998. put_page(head);
  999. goto repeat;
  1000. }
  1001. /*
  1002. * Has the page moved?
  1003. * This is part of the lockless pagecache protocol. See
  1004. * include/linux/pagemap.h for details.
  1005. */
  1006. if (unlikely(page != *pagep)) {
  1007. put_page(head);
  1008. goto repeat;
  1009. }
  1010. }
  1011. out:
  1012. rcu_read_unlock();
  1013. return page;
  1014. }
  1015. EXPORT_SYMBOL(find_get_entry);
  1016. /**
  1017. * find_lock_entry - locate, pin and lock a page cache entry
  1018. * @mapping: the address_space to search
  1019. * @offset: the page cache index
  1020. *
  1021. * Looks up the page cache slot at @mapping & @offset. If there is a
  1022. * page cache page, it is returned locked and with an increased
  1023. * refcount.
  1024. *
  1025. * If the slot holds a shadow entry of a previously evicted page, or a
  1026. * swap entry from shmem/tmpfs, it is returned.
  1027. *
  1028. * Otherwise, %NULL is returned.
  1029. *
  1030. * find_lock_entry() may sleep.
  1031. */
  1032. struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset)
  1033. {
  1034. struct page *page;
  1035. repeat:
  1036. page = find_get_entry(mapping, offset);
  1037. if (page && !radix_tree_exception(page)) {
  1038. lock_page(page);
  1039. /* Has the page been truncated? */
  1040. if (unlikely(page_mapping(page) != mapping)) {
  1041. unlock_page(page);
  1042. put_page(page);
  1043. goto repeat;
  1044. }
  1045. VM_BUG_ON_PAGE(page_to_pgoff(page) != offset, page);
  1046. }
  1047. return page;
  1048. }
  1049. EXPORT_SYMBOL(find_lock_entry);
  1050. /**
  1051. * pagecache_get_page - find and get a page reference
  1052. * @mapping: the address_space to search
  1053. * @offset: the page index
  1054. * @fgp_flags: PCG flags
  1055. * @gfp_mask: gfp mask to use for the page cache data page allocation
  1056. *
  1057. * Looks up the page cache slot at @mapping & @offset.
  1058. *
  1059. * PCG flags modify how the page is returned.
  1060. *
  1061. * FGP_ACCESSED: the page will be marked accessed
  1062. * FGP_LOCK: Page is return locked
  1063. * FGP_CREAT: If page is not present then a new page is allocated using
  1064. * @gfp_mask and added to the page cache and the VM's LRU
  1065. * list. The page is returned locked and with an increased
  1066. * refcount. Otherwise, %NULL is returned.
  1067. *
  1068. * If FGP_LOCK or FGP_CREAT are specified then the function may sleep even
  1069. * if the GFP flags specified for FGP_CREAT are atomic.
  1070. *
  1071. * If there is a page cache page, it is returned with an increased refcount.
  1072. */
  1073. struct page *pagecache_get_page(struct address_space *mapping, pgoff_t offset,
  1074. int fgp_flags, gfp_t gfp_mask)
  1075. {
  1076. struct page *page;
  1077. repeat:
  1078. page = find_get_entry(mapping, offset);
  1079. if (radix_tree_exceptional_entry(page))
  1080. page = NULL;
  1081. if (!page)
  1082. goto no_page;
  1083. if (fgp_flags & FGP_LOCK) {
  1084. if (fgp_flags & FGP_NOWAIT) {
  1085. if (!trylock_page(page)) {
  1086. put_page(page);
  1087. return NULL;
  1088. }
  1089. } else {
  1090. lock_page(page);
  1091. }
  1092. /* Has the page been truncated? */
  1093. if (unlikely(page->mapping != mapping)) {
  1094. unlock_page(page);
  1095. put_page(page);
  1096. goto repeat;
  1097. }
  1098. VM_BUG_ON_PAGE(page->index != offset, page);
  1099. }
  1100. if (page && (fgp_flags & FGP_ACCESSED))
  1101. mark_page_accessed(page);
  1102. no_page:
  1103. if (!page && (fgp_flags & FGP_CREAT)) {
  1104. int err;
  1105. if ((fgp_flags & FGP_WRITE) && mapping_cap_account_dirty(mapping))
  1106. gfp_mask |= __GFP_WRITE;
  1107. if (fgp_flags & FGP_NOFS)
  1108. gfp_mask &= ~__GFP_FS;
  1109. page = __page_cache_alloc(gfp_mask);
  1110. if (!page)
  1111. return NULL;
  1112. if (WARN_ON_ONCE(!(fgp_flags & FGP_LOCK)))
  1113. fgp_flags |= FGP_LOCK;
  1114. /* Init accessed so avoid atomic mark_page_accessed later */
  1115. if (fgp_flags & FGP_ACCESSED)
  1116. __SetPageReferenced(page);
  1117. err = add_to_page_cache_lru(page, mapping, offset,
  1118. gfp_mask & GFP_RECLAIM_MASK);
  1119. if (unlikely(err)) {
  1120. put_page(page);
  1121. page = NULL;
  1122. if (err == -EEXIST)
  1123. goto repeat;
  1124. }
  1125. }
  1126. return page;
  1127. }
  1128. EXPORT_SYMBOL(pagecache_get_page);
  1129. /**
  1130. * find_get_entries - gang pagecache lookup
  1131. * @mapping: The address_space to search
  1132. * @start: The starting page cache index
  1133. * @nr_entries: The maximum number of entries
  1134. * @entries: Where the resulting entries are placed
  1135. * @indices: The cache indices corresponding to the entries in @entries
  1136. *
  1137. * find_get_entries() will search for and return a group of up to
  1138. * @nr_entries entries in the mapping. The entries are placed at
  1139. * @entries. find_get_entries() takes a reference against any actual
  1140. * pages it returns.
  1141. *
  1142. * The search returns a group of mapping-contiguous page cache entries
  1143. * with ascending indexes. There may be holes in the indices due to
  1144. * not-present pages.
  1145. *
  1146. * Any shadow entries of evicted pages, or swap entries from
  1147. * shmem/tmpfs, are included in the returned array.
  1148. *
  1149. * find_get_entries() returns the number of pages and shadow entries
  1150. * which were found.
  1151. */
  1152. unsigned find_get_entries(struct address_space *mapping,
  1153. pgoff_t start, unsigned int nr_entries,
  1154. struct page **entries, pgoff_t *indices)
  1155. {
  1156. void **slot;
  1157. unsigned int ret = 0;
  1158. struct radix_tree_iter iter;
  1159. if (!nr_entries)
  1160. return 0;
  1161. rcu_read_lock();
  1162. radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
  1163. struct page *head, *page;
  1164. repeat:
  1165. page = radix_tree_deref_slot(slot);
  1166. if (unlikely(!page))
  1167. continue;
  1168. if (radix_tree_exception(page)) {
  1169. if (radix_tree_deref_retry(page)) {
  1170. slot = radix_tree_iter_retry(&iter);
  1171. continue;
  1172. }
  1173. /*
  1174. * A shadow entry of a recently evicted page, a swap
  1175. * entry from shmem/tmpfs or a DAX entry. Return it
  1176. * without attempting to raise page count.
  1177. */
  1178. goto export;
  1179. }
  1180. head = compound_head(page);
  1181. if (!page_cache_get_speculative(head))
  1182. goto repeat;
  1183. /* The page was split under us? */
  1184. if (compound_head(page) != head) {
  1185. put_page(head);
  1186. goto repeat;
  1187. }
  1188. /* Has the page moved? */
  1189. if (unlikely(page != *slot)) {
  1190. put_page(head);
  1191. goto repeat;
  1192. }
  1193. export:
  1194. indices[ret] = iter.index;
  1195. entries[ret] = page;
  1196. if (++ret == nr_entries)
  1197. break;
  1198. }
  1199. rcu_read_unlock();
  1200. return ret;
  1201. }
  1202. /**
  1203. * find_get_pages - gang pagecache lookup
  1204. * @mapping: The address_space to search
  1205. * @start: The starting page index
  1206. * @nr_pages: The maximum number of pages
  1207. * @pages: Where the resulting pages are placed
  1208. *
  1209. * find_get_pages() will search for and return a group of up to
  1210. * @nr_pages pages in the mapping. The pages are placed at @pages.
  1211. * find_get_pages() takes a reference against the returned pages.
  1212. *
  1213. * The search returns a group of mapping-contiguous pages with ascending
  1214. * indexes. There may be holes in the indices due to not-present pages.
  1215. *
  1216. * find_get_pages() returns the number of pages which were found.
  1217. */
  1218. unsigned find_get_pages(struct address_space *mapping, pgoff_t start,
  1219. unsigned int nr_pages, struct page **pages)
  1220. {
  1221. struct radix_tree_iter iter;
  1222. void **slot;
  1223. unsigned ret = 0;
  1224. if (unlikely(!nr_pages))
  1225. return 0;
  1226. rcu_read_lock();
  1227. radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
  1228. struct page *head, *page;
  1229. repeat:
  1230. page = radix_tree_deref_slot(slot);
  1231. if (unlikely(!page))
  1232. continue;
  1233. if (radix_tree_exception(page)) {
  1234. if (radix_tree_deref_retry(page)) {
  1235. slot = radix_tree_iter_retry(&iter);
  1236. continue;
  1237. }
  1238. /*
  1239. * A shadow entry of a recently evicted page,
  1240. * or a swap entry from shmem/tmpfs. Skip
  1241. * over it.
  1242. */
  1243. continue;
  1244. }
  1245. head = compound_head(page);
  1246. if (!page_cache_get_speculative(head))
  1247. goto repeat;
  1248. /* The page was split under us? */
  1249. if (compound_head(page) != head) {
  1250. put_page(head);
  1251. goto repeat;
  1252. }
  1253. /* Has the page moved? */
  1254. if (unlikely(page != *slot)) {
  1255. put_page(head);
  1256. goto repeat;
  1257. }
  1258. pages[ret] = page;
  1259. if (++ret == nr_pages)
  1260. break;
  1261. }
  1262. rcu_read_unlock();
  1263. return ret;
  1264. }
  1265. /**
  1266. * find_get_pages_contig - gang contiguous pagecache lookup
  1267. * @mapping: The address_space to search
  1268. * @index: The starting page index
  1269. * @nr_pages: The maximum number of pages
  1270. * @pages: Where the resulting pages are placed
  1271. *
  1272. * find_get_pages_contig() works exactly like find_get_pages(), except
  1273. * that the returned number of pages are guaranteed to be contiguous.
  1274. *
  1275. * find_get_pages_contig() returns the number of pages which were found.
  1276. */
  1277. unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
  1278. unsigned int nr_pages, struct page **pages)
  1279. {
  1280. struct radix_tree_iter iter;
  1281. void **slot;
  1282. unsigned int ret = 0;
  1283. if (unlikely(!nr_pages))
  1284. return 0;
  1285. rcu_read_lock();
  1286. radix_tree_for_each_contig(slot, &mapping->page_tree, &iter, index) {
  1287. struct page *head, *page;
  1288. repeat:
  1289. page = radix_tree_deref_slot(slot);
  1290. /* The hole, there no reason to continue */
  1291. if (unlikely(!page))
  1292. break;
  1293. if (radix_tree_exception(page)) {
  1294. if (radix_tree_deref_retry(page)) {
  1295. slot = radix_tree_iter_retry(&iter);
  1296. continue;
  1297. }
  1298. /*
  1299. * A shadow entry of a recently evicted page,
  1300. * or a swap entry from shmem/tmpfs. Stop
  1301. * looking for contiguous pages.
  1302. */
  1303. break;
  1304. }
  1305. head = compound_head(page);
  1306. if (!page_cache_get_speculative(head))
  1307. goto repeat;
  1308. /* The page was split under us? */
  1309. if (compound_head(page) != head) {
  1310. put_page(head);
  1311. goto repeat;
  1312. }
  1313. /* Has the page moved? */
  1314. if (unlikely(page != *slot)) {
  1315. put_page(head);
  1316. goto repeat;
  1317. }
  1318. /*
  1319. * must check mapping and index after taking the ref.
  1320. * otherwise we can get both false positives and false
  1321. * negatives, which is just confusing to the caller.
  1322. */
  1323. if (page->mapping == NULL || page_to_pgoff(page) != iter.index) {
  1324. put_page(page);
  1325. break;
  1326. }
  1327. pages[ret] = page;
  1328. if (++ret == nr_pages)
  1329. break;
  1330. }
  1331. rcu_read_unlock();
  1332. return ret;
  1333. }
  1334. EXPORT_SYMBOL(find_get_pages_contig);
  1335. /**
  1336. * find_get_pages_tag - find and return pages that match @tag
  1337. * @mapping: the address_space to search
  1338. * @index: the starting page index
  1339. * @tag: the tag index
  1340. * @nr_pages: the maximum number of pages
  1341. * @pages: where the resulting pages are placed
  1342. *
  1343. * Like find_get_pages, except we only return pages which are tagged with
  1344. * @tag. We update @index to index the next page for the traversal.
  1345. */
  1346. unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
  1347. int tag, unsigned int nr_pages, struct page **pages)
  1348. {
  1349. struct radix_tree_iter iter;
  1350. void **slot;
  1351. unsigned ret = 0;
  1352. if (unlikely(!nr_pages))
  1353. return 0;
  1354. rcu_read_lock();
  1355. radix_tree_for_each_tagged(slot, &mapping->page_tree,
  1356. &iter, *index, tag) {
  1357. struct page *head, *page;
  1358. repeat:
  1359. page = radix_tree_deref_slot(slot);
  1360. if (unlikely(!page))
  1361. continue;
  1362. if (radix_tree_exception(page)) {
  1363. if (radix_tree_deref_retry(page)) {
  1364. slot = radix_tree_iter_retry(&iter);
  1365. continue;
  1366. }
  1367. /*
  1368. * A shadow entry of a recently evicted page.
  1369. *
  1370. * Those entries should never be tagged, but
  1371. * this tree walk is lockless and the tags are
  1372. * looked up in bulk, one radix tree node at a
  1373. * time, so there is a sizable window for page
  1374. * reclaim to evict a page we saw tagged.
  1375. *
  1376. * Skip over it.
  1377. */
  1378. continue;
  1379. }
  1380. head = compound_head(page);
  1381. if (!page_cache_get_speculative(head))
  1382. goto repeat;
  1383. /* The page was split under us? */
  1384. if (compound_head(page) != head) {
  1385. put_page(head);
  1386. goto repeat;
  1387. }
  1388. /* Has the page moved? */
  1389. if (unlikely(page != *slot)) {
  1390. put_page(head);
  1391. goto repeat;
  1392. }
  1393. pages[ret] = page;
  1394. if (++ret == nr_pages)
  1395. break;
  1396. }
  1397. rcu_read_unlock();
  1398. if (ret)
  1399. *index = pages[ret - 1]->index + 1;
  1400. return ret;
  1401. }
  1402. EXPORT_SYMBOL(find_get_pages_tag);
  1403. /**
  1404. * find_get_entries_tag - find and return entries that match @tag
  1405. * @mapping: the address_space to search
  1406. * @start: the starting page cache index
  1407. * @tag: the tag index
  1408. * @nr_entries: the maximum number of entries
  1409. * @entries: where the resulting entries are placed
  1410. * @indices: the cache indices corresponding to the entries in @entries
  1411. *
  1412. * Like find_get_entries, except we only return entries which are tagged with
  1413. * @tag.
  1414. */
  1415. unsigned find_get_entries_tag(struct address_space *mapping, pgoff_t start,
  1416. int tag, unsigned int nr_entries,
  1417. struct page **entries, pgoff_t *indices)
  1418. {
  1419. void **slot;
  1420. unsigned int ret = 0;
  1421. struct radix_tree_iter iter;
  1422. if (!nr_entries)
  1423. return 0;
  1424. rcu_read_lock();
  1425. radix_tree_for_each_tagged(slot, &mapping->page_tree,
  1426. &iter, start, tag) {
  1427. struct page *head, *page;
  1428. repeat:
  1429. page = radix_tree_deref_slot(slot);
  1430. if (unlikely(!page))
  1431. continue;
  1432. if (radix_tree_exception(page)) {
  1433. if (radix_tree_deref_retry(page)) {
  1434. slot = radix_tree_iter_retry(&iter);
  1435. continue;
  1436. }
  1437. /*
  1438. * A shadow entry of a recently evicted page, a swap
  1439. * entry from shmem/tmpfs or a DAX entry. Return it
  1440. * without attempting to raise page count.
  1441. */
  1442. goto export;
  1443. }
  1444. head = compound_head(page);
  1445. if (!page_cache_get_speculative(head))
  1446. goto repeat;
  1447. /* The page was split under us? */
  1448. if (compound_head(page) != head) {
  1449. put_page(head);
  1450. goto repeat;
  1451. }
  1452. /* Has the page moved? */
  1453. if (unlikely(page != *slot)) {
  1454. put_page(head);
  1455. goto repeat;
  1456. }
  1457. export:
  1458. indices[ret] = iter.index;
  1459. entries[ret] = page;
  1460. if (++ret == nr_entries)
  1461. break;
  1462. }
  1463. rcu_read_unlock();
  1464. return ret;
  1465. }
  1466. EXPORT_SYMBOL(find_get_entries_tag);
  1467. /*
  1468. * CD/DVDs are error prone. When a medium error occurs, the driver may fail
  1469. * a _large_ part of the i/o request. Imagine the worst scenario:
  1470. *
  1471. * ---R__________________________________________B__________
  1472. * ^ reading here ^ bad block(assume 4k)
  1473. *
  1474. * read(R) => miss => readahead(R...B) => media error => frustrating retries
  1475. * => failing the whole request => read(R) => read(R+1) =>
  1476. * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) =>
  1477. * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) =>
  1478. * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ......
  1479. *
  1480. * It is going insane. Fix it by quickly scaling down the readahead size.
  1481. */
  1482. static void shrink_readahead_size_eio(struct file *filp,
  1483. struct file_ra_state *ra)
  1484. {
  1485. ra->ra_pages /= 4;
  1486. }
  1487. /**
  1488. * do_generic_file_read - generic file read routine
  1489. * @filp: the file to read
  1490. * @ppos: current file position
  1491. * @iter: data destination
  1492. * @written: already copied
  1493. *
  1494. * This is a generic file read routine, and uses the
  1495. * mapping->a_ops->readpage() function for the actual low-level stuff.
  1496. *
  1497. * This is really ugly. But the goto's actually try to clarify some
  1498. * of the logic when it comes to error handling etc.
  1499. */
  1500. static ssize_t do_generic_file_read(struct file *filp, loff_t *ppos,
  1501. struct iov_iter *iter, ssize_t written)
  1502. {
  1503. struct address_space *mapping = filp->f_mapping;
  1504. struct inode *inode = mapping->host;
  1505. struct file_ra_state *ra = &filp->f_ra;
  1506. pgoff_t index;
  1507. pgoff_t last_index;
  1508. pgoff_t prev_index;
  1509. unsigned long offset; /* offset into pagecache page */
  1510. unsigned int prev_offset;
  1511. int error = 0;
  1512. if (unlikely(*ppos >= inode->i_sb->s_maxbytes))
  1513. return 0;
  1514. iov_iter_truncate(iter, inode->i_sb->s_maxbytes);
  1515. index = *ppos >> PAGE_SHIFT;
  1516. prev_index = ra->prev_pos >> PAGE_SHIFT;
  1517. prev_offset = ra->prev_pos & (PAGE_SIZE-1);
  1518. last_index = (*ppos + iter->count + PAGE_SIZE-1) >> PAGE_SHIFT;
  1519. offset = *ppos & ~PAGE_MASK;
  1520. for (;;) {
  1521. struct page *page;
  1522. pgoff_t end_index;
  1523. loff_t isize;
  1524. unsigned long nr, ret;
  1525. cond_resched();
  1526. find_page:
  1527. if (fatal_signal_pending(current)) {
  1528. error = -EINTR;
  1529. goto out;
  1530. }
  1531. page = find_get_page(mapping, index);
  1532. if (!page) {
  1533. page_cache_sync_readahead(mapping,
  1534. ra, filp,
  1535. index, last_index - index);
  1536. page = find_get_page(mapping, index);
  1537. if (unlikely(page == NULL))
  1538. goto no_cached_page;
  1539. }
  1540. if (PageReadahead(page)) {
  1541. page_cache_async_readahead(mapping,
  1542. ra, filp, page,
  1543. index, last_index - index);
  1544. }
  1545. if (!PageUptodate(page)) {
  1546. /*
  1547. * See comment in do_read_cache_page on why
  1548. * wait_on_page_locked is used to avoid unnecessarily
  1549. * serialisations and why it's safe.
  1550. */
  1551. error = wait_on_page_locked_killable(page);
  1552. if (unlikely(error))
  1553. goto readpage_error;
  1554. if (PageUptodate(page))
  1555. goto page_ok;
  1556. if (inode->i_blkbits == PAGE_SHIFT ||
  1557. !mapping->a_ops->is_partially_uptodate)
  1558. goto page_not_up_to_date;
  1559. /* pipes can't handle partially uptodate pages */
  1560. if (unlikely(iter->type & ITER_PIPE))
  1561. goto page_not_up_to_date;
  1562. if (!trylock_page(page))
  1563. goto page_not_up_to_date;
  1564. /* Did it get truncated before we got the lock? */
  1565. if (!page->mapping)
  1566. goto page_not_up_to_date_locked;
  1567. if (!mapping->a_ops->is_partially_uptodate(page,
  1568. offset, iter->count))
  1569. goto page_not_up_to_date_locked;
  1570. unlock_page(page);
  1571. }
  1572. page_ok:
  1573. /*
  1574. * i_size must be checked after we know the page is Uptodate.
  1575. *
  1576. * Checking i_size after the check allows us to calculate
  1577. * the correct value for "nr", which means the zero-filled
  1578. * part of the page is not copied back to userspace (unless
  1579. * another truncate extends the file - this is desired though).
  1580. */
  1581. isize = i_size_read(inode);
  1582. end_index = (isize - 1) >> PAGE_SHIFT;
  1583. if (unlikely(!isize || index > end_index)) {
  1584. put_page(page);
  1585. goto out;
  1586. }
  1587. /* nr is the maximum number of bytes to copy from this page */
  1588. nr = PAGE_SIZE;
  1589. if (index == end_index) {
  1590. nr = ((isize - 1) & ~PAGE_MASK) + 1;
  1591. if (nr <= offset) {
  1592. put_page(page);
  1593. goto out;
  1594. }
  1595. }
  1596. nr = nr - offset;
  1597. /* If users can be writing to this page using arbitrary
  1598. * virtual addresses, take care about potential aliasing
  1599. * before reading the page on the kernel side.
  1600. */
  1601. if (mapping_writably_mapped(mapping))
  1602. flush_dcache_page(page);
  1603. /*
  1604. * When a sequential read accesses a page several times,
  1605. * only mark it as accessed the first time.
  1606. */
  1607. if (prev_index != index || offset != prev_offset)
  1608. mark_page_accessed(page);
  1609. prev_index = index;
  1610. /*
  1611. * Ok, we have the page, and it's up-to-date, so
  1612. * now we can copy it to user space...
  1613. */
  1614. ret = copy_page_to_iter(page, offset, nr, iter);
  1615. offset += ret;
  1616. index += offset >> PAGE_SHIFT;
  1617. offset &= ~PAGE_MASK;
  1618. prev_offset = offset;
  1619. put_page(page);
  1620. written += ret;
  1621. if (!iov_iter_count(iter))
  1622. goto out;
  1623. if (ret < nr) {
  1624. error = -EFAULT;
  1625. goto out;
  1626. }
  1627. continue;
  1628. page_not_up_to_date:
  1629. /* Get exclusive access to the page ... */
  1630. error = lock_page_killable(page);
  1631. if (unlikely(error))
  1632. goto readpage_error;
  1633. page_not_up_to_date_locked:
  1634. /* Did it get truncated before we got the lock? */
  1635. if (!page->mapping) {
  1636. unlock_page(page);
  1637. put_page(page);
  1638. continue;
  1639. }
  1640. /* Did somebody else fill it already? */
  1641. if (PageUptodate(page)) {
  1642. unlock_page(page);
  1643. goto page_ok;
  1644. }
  1645. readpage:
  1646. /*
  1647. * A previous I/O error may have been due to temporary
  1648. * failures, eg. multipath errors.
  1649. * PG_error will be set again if readpage fails.
  1650. */
  1651. ClearPageError(page);
  1652. /* Start the actual read. The read will unlock the page. */
  1653. error = mapping->a_ops->readpage(filp, page);
  1654. if (unlikely(error)) {
  1655. if (error == AOP_TRUNCATED_PAGE) {
  1656. put_page(page);
  1657. error = 0;
  1658. goto find_page;
  1659. }
  1660. goto readpage_error;
  1661. }
  1662. if (!PageUptodate(page)) {
  1663. error = lock_page_killable(page);
  1664. if (unlikely(error))
  1665. goto readpage_error;
  1666. if (!PageUptodate(page)) {
  1667. if (page->mapping == NULL) {
  1668. /*
  1669. * invalidate_mapping_pages got it
  1670. */
  1671. unlock_page(page);
  1672. put_page(page);
  1673. goto find_page;
  1674. }
  1675. unlock_page(page);
  1676. shrink_readahead_size_eio(filp, ra);
  1677. error = -EIO;
  1678. goto readpage_error;
  1679. }
  1680. unlock_page(page);
  1681. }
  1682. goto page_ok;
  1683. readpage_error:
  1684. /* UHHUH! A synchronous read error occurred. Report it */
  1685. put_page(page);
  1686. goto out;
  1687. no_cached_page:
  1688. /*
  1689. * Ok, it wasn't cached, so we need to create a new
  1690. * page..
  1691. */
  1692. page = page_cache_alloc_cold(mapping);
  1693. if (!page) {
  1694. error = -ENOMEM;
  1695. goto out;
  1696. }
  1697. error = add_to_page_cache_lru(page, mapping, index,
  1698. mapping_gfp_constraint(mapping, GFP_KERNEL));
  1699. if (error) {
  1700. put_page(page);
  1701. if (error == -EEXIST) {
  1702. error = 0;
  1703. goto find_page;
  1704. }
  1705. goto out;
  1706. }
  1707. goto readpage;
  1708. }
  1709. out:
  1710. ra->prev_pos = prev_index;
  1711. ra->prev_pos <<= PAGE_SHIFT;
  1712. ra->prev_pos |= prev_offset;
  1713. *ppos = ((loff_t)index << PAGE_SHIFT) + offset;
  1714. file_accessed(filp);
  1715. return written ? written : error;
  1716. }
  1717. /**
  1718. * generic_file_read_iter - generic filesystem read routine
  1719. * @iocb: kernel I/O control block
  1720. * @iter: destination for the data read
  1721. *
  1722. * This is the "read_iter()" routine for all filesystems
  1723. * that can use the page cache directly.
  1724. */
  1725. ssize_t
  1726. generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
  1727. {
  1728. struct file *file = iocb->ki_filp;
  1729. ssize_t retval = 0;
  1730. size_t count = iov_iter_count(iter);
  1731. if (!count)
  1732. goto out; /* skip atime */
  1733. if (iocb->ki_flags & IOCB_DIRECT) {
  1734. struct address_space *mapping = file->f_mapping;
  1735. struct inode *inode = mapping->host;
  1736. struct iov_iter data = *iter;
  1737. loff_t size;
  1738. size = i_size_read(inode);
  1739. retval = filemap_write_and_wait_range(mapping, iocb->ki_pos,
  1740. iocb->ki_pos + count - 1);
  1741. if (retval < 0)
  1742. goto out;
  1743. file_accessed(file);
  1744. retval = mapping->a_ops->direct_IO(iocb, &data);
  1745. if (retval >= 0) {
  1746. iocb->ki_pos += retval;
  1747. iov_iter_advance(iter, retval);
  1748. }
  1749. /*
  1750. * Btrfs can have a short DIO read if we encounter
  1751. * compressed extents, so if there was an error, or if
  1752. * we've already read everything we wanted to, or if
  1753. * there was a short read because we hit EOF, go ahead
  1754. * and return. Otherwise fallthrough to buffered io for
  1755. * the rest of the read. Buffered reads will not work for
  1756. * DAX files, so don't bother trying.
  1757. */
  1758. if (retval < 0 || !iov_iter_count(iter) || iocb->ki_pos >= size ||
  1759. IS_DAX(inode))
  1760. goto out;
  1761. }
  1762. retval = do_generic_file_read(file, &iocb->ki_pos, iter, retval);
  1763. out:
  1764. return retval;
  1765. }
  1766. EXPORT_SYMBOL(generic_file_read_iter);
  1767. #ifdef CONFIG_MMU
  1768. /**
  1769. * page_cache_read - adds requested page to the page cache if not already there
  1770. * @file: file to read
  1771. * @offset: page index
  1772. * @gfp_mask: memory allocation flags
  1773. *
  1774. * This adds the requested page to the page cache if it isn't already there,
  1775. * and schedules an I/O to read in its contents from disk.
  1776. */
  1777. static int page_cache_read(struct file *file, pgoff_t offset, gfp_t gfp_mask)
  1778. {
  1779. struct address_space *mapping = file->f_mapping;
  1780. struct page *page;
  1781. int ret;
  1782. do {
  1783. page = __page_cache_alloc(gfp_mask|__GFP_COLD);
  1784. if (!page)
  1785. return -ENOMEM;
  1786. ret = add_to_page_cache_lru(page, mapping, offset, gfp_mask & GFP_KERNEL);
  1787. if (ret == 0)
  1788. ret = mapping->a_ops->readpage(file, page);
  1789. else if (ret == -EEXIST)
  1790. ret = 0; /* losing race to add is OK */
  1791. put_page(page);
  1792. } while (ret == AOP_TRUNCATED_PAGE);
  1793. return ret;
  1794. }
  1795. #define MMAP_LOTSAMISS (100)
  1796. /*
  1797. * Synchronous readahead happens when we don't even find
  1798. * a page in the page cache at all.
  1799. */
  1800. static void do_sync_mmap_readahead(struct vm_area_struct *vma,
  1801. struct file_ra_state *ra,
  1802. struct file *file,
  1803. pgoff_t offset)
  1804. {
  1805. struct address_space *mapping = file->f_mapping;
  1806. /* If we don't want any read-ahead, don't bother */
  1807. if (vma->vm_flags & VM_RAND_READ)
  1808. return;
  1809. if (!ra->ra_pages)
  1810. return;
  1811. if (vma->vm_flags & VM_SEQ_READ) {
  1812. page_cache_sync_readahead(mapping, ra, file, offset,
  1813. ra->ra_pages);
  1814. return;
  1815. }
  1816. /* Avoid banging the cache line if not needed */
  1817. if (ra->mmap_miss < MMAP_LOTSAMISS * 10)
  1818. ra->mmap_miss++;
  1819. /*
  1820. * Do we miss much more than hit in this file? If so,
  1821. * stop bothering with read-ahead. It will only hurt.
  1822. */
  1823. if (ra->mmap_miss > MMAP_LOTSAMISS)
  1824. return;
  1825. /*
  1826. * mmap read-around
  1827. */
  1828. ra->start = max_t(long, 0, offset - ra->ra_pages / 2);
  1829. ra->size = ra->ra_pages;
  1830. ra->async_size = ra->ra_pages / 4;
  1831. ra_submit(ra, mapping, file);
  1832. }
  1833. /*
  1834. * Asynchronous readahead happens when we find the page and PG_readahead,
  1835. * so we want to possibly extend the readahead further..
  1836. */
  1837. static void do_async_mmap_readahead(struct vm_area_struct *vma,
  1838. struct file_ra_state *ra,
  1839. struct file *file,
  1840. struct page *page,
  1841. pgoff_t offset)
  1842. {
  1843. struct address_space *mapping = file->f_mapping;
  1844. /* If we don't want any read-ahead, don't bother */
  1845. if (vma->vm_flags & VM_RAND_READ)
  1846. return;
  1847. if (ra->mmap_miss > 0)
  1848. ra->mmap_miss--;
  1849. if (PageReadahead(page))
  1850. page_cache_async_readahead(mapping, ra, file,
  1851. page, offset, ra->ra_pages);
  1852. }
  1853. /**
  1854. * filemap_fault - read in file data for page fault handling
  1855. * @vma: vma in which the fault was taken
  1856. * @vmf: struct vm_fault containing details of the fault
  1857. *
  1858. * filemap_fault() is invoked via the vma operations vector for a
  1859. * mapped memory region to read in file data during a page fault.
  1860. *
  1861. * The goto's are kind of ugly, but this streamlines the normal case of having
  1862. * it in the page cache, and handles the special cases reasonably without
  1863. * having a lot of duplicated code.
  1864. *
  1865. * vma->vm_mm->mmap_sem must be held on entry.
  1866. *
  1867. * If our return value has VM_FAULT_RETRY set, it's because
  1868. * lock_page_or_retry() returned 0.
  1869. * The mmap_sem has usually been released in this case.
  1870. * See __lock_page_or_retry() for the exception.
  1871. *
  1872. * If our return value does not have VM_FAULT_RETRY set, the mmap_sem
  1873. * has not been released.
  1874. *
  1875. * We never return with VM_FAULT_RETRY and a bit from VM_FAULT_ERROR set.
  1876. */
  1877. int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  1878. {
  1879. int error;
  1880. struct file *file = vma->vm_file;
  1881. struct address_space *mapping = file->f_mapping;
  1882. struct file_ra_state *ra = &file->f_ra;
  1883. struct inode *inode = mapping->host;
  1884. pgoff_t offset = vmf->pgoff;
  1885. struct page *page;
  1886. loff_t size;
  1887. int ret = 0;
  1888. size = round_up(i_size_read(inode), PAGE_SIZE);
  1889. if (offset >= size >> PAGE_SHIFT)
  1890. return VM_FAULT_SIGBUS;
  1891. /*
  1892. * Do we have something in the page cache already?
  1893. */
  1894. page = find_get_page(mapping, offset);
  1895. if (likely(page) && !(vmf->flags & FAULT_FLAG_TRIED)) {
  1896. /*
  1897. * We found the page, so try async readahead before
  1898. * waiting for the lock.
  1899. */
  1900. do_async_mmap_readahead(vma, ra, file, page, offset);
  1901. } else if (!page) {
  1902. /* No page in the page cache at all */
  1903. do_sync_mmap_readahead(vma, ra, file, offset);
  1904. count_vm_event(PGMAJFAULT);
  1905. mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
  1906. ret = VM_FAULT_MAJOR;
  1907. retry_find:
  1908. page = find_get_page(mapping, offset);
  1909. if (!page)
  1910. goto no_cached_page;
  1911. }
  1912. if (!lock_page_or_retry(page, vma->vm_mm, vmf->flags)) {
  1913. put_page(page);
  1914. return ret | VM_FAULT_RETRY;
  1915. }
  1916. /* Did it get truncated? */
  1917. if (unlikely(page->mapping != mapping)) {
  1918. unlock_page(page);
  1919. put_page(page);
  1920. goto retry_find;
  1921. }
  1922. VM_BUG_ON_PAGE(page->index != offset, page);
  1923. /*
  1924. * We have a locked page in the page cache, now we need to check
  1925. * that it's up-to-date. If not, it is going to be due to an error.
  1926. */
  1927. if (unlikely(!PageUptodate(page)))
  1928. goto page_not_uptodate;
  1929. /*
  1930. * Found the page and have a reference on it.
  1931. * We must recheck i_size under page lock.
  1932. */
  1933. size = round_up(i_size_read(inode), PAGE_SIZE);
  1934. if (unlikely(offset >= size >> PAGE_SHIFT)) {
  1935. unlock_page(page);
  1936. put_page(page);
  1937. return VM_FAULT_SIGBUS;
  1938. }
  1939. vmf->page = page;
  1940. return ret | VM_FAULT_LOCKED;
  1941. no_cached_page:
  1942. /*
  1943. * We're only likely to ever get here if MADV_RANDOM is in
  1944. * effect.
  1945. */
  1946. error = page_cache_read(file, offset, vmf->gfp_mask);
  1947. /*
  1948. * The page we want has now been added to the page cache.
  1949. * In the unlikely event that someone removed it in the
  1950. * meantime, we'll just come back here and read it again.
  1951. */
  1952. if (error >= 0)
  1953. goto retry_find;
  1954. /*
  1955. * An error return from page_cache_read can result if the
  1956. * system is low on memory, or a problem occurs while trying
  1957. * to schedule I/O.
  1958. */
  1959. if (error == -ENOMEM)
  1960. return VM_FAULT_OOM;
  1961. return VM_FAULT_SIGBUS;
  1962. page_not_uptodate:
  1963. /*
  1964. * Umm, take care of errors if the page isn't up-to-date.
  1965. * Try to re-read it _once_. We do this synchronously,
  1966. * because there really aren't any performance issues here
  1967. * and we need to check for errors.
  1968. */
  1969. ClearPageError(page);
  1970. error = mapping->a_ops->readpage(file, page);
  1971. if (!error) {
  1972. wait_on_page_locked(page);
  1973. if (!PageUptodate(page))
  1974. error = -EIO;
  1975. }
  1976. put_page(page);
  1977. if (!error || error == AOP_TRUNCATED_PAGE)
  1978. goto retry_find;
  1979. /* Things didn't work out. Return zero to tell the mm layer so. */
  1980. shrink_readahead_size_eio(file, ra);
  1981. return VM_FAULT_SIGBUS;
  1982. }
  1983. EXPORT_SYMBOL(filemap_fault);
  1984. void filemap_map_pages(struct fault_env *fe,
  1985. pgoff_t start_pgoff, pgoff_t end_pgoff)
  1986. {
  1987. struct radix_tree_iter iter;
  1988. void **slot;
  1989. struct file *file = fe->vma->vm_file;
  1990. struct address_space *mapping = file->f_mapping;
  1991. pgoff_t last_pgoff = start_pgoff;
  1992. loff_t size;
  1993. struct page *head, *page;
  1994. rcu_read_lock();
  1995. radix_tree_for_each_slot(slot, &mapping->page_tree, &iter,
  1996. start_pgoff) {
  1997. if (iter.index > end_pgoff)
  1998. break;
  1999. repeat:
  2000. page = radix_tree_deref_slot(slot);
  2001. if (unlikely(!page))
  2002. goto next;
  2003. if (radix_tree_exception(page)) {
  2004. if (radix_tree_deref_retry(page)) {
  2005. slot = radix_tree_iter_retry(&iter);
  2006. continue;
  2007. }
  2008. goto next;
  2009. }
  2010. head = compound_head(page);
  2011. if (!page_cache_get_speculative(head))
  2012. goto repeat;
  2013. /* The page was split under us? */
  2014. if (compound_head(page) != head) {
  2015. put_page(head);
  2016. goto repeat;
  2017. }
  2018. /* Has the page moved? */
  2019. if (unlikely(page != *slot)) {
  2020. put_page(head);
  2021. goto repeat;
  2022. }
  2023. if (!PageUptodate(page) ||
  2024. PageReadahead(page) ||
  2025. PageHWPoison(page))
  2026. goto skip;
  2027. if (!trylock_page(page))
  2028. goto skip;
  2029. if (page->mapping != mapping || !PageUptodate(page))
  2030. goto unlock;
  2031. size = round_up(i_size_read(mapping->host), PAGE_SIZE);
  2032. if (page->index >= size >> PAGE_SHIFT)
  2033. goto unlock;
  2034. if (file->f_ra.mmap_miss > 0)
  2035. file->f_ra.mmap_miss--;
  2036. fe->address += (iter.index - last_pgoff) << PAGE_SHIFT;
  2037. if (fe->pte)
  2038. fe->pte += iter.index - last_pgoff;
  2039. last_pgoff = iter.index;
  2040. if (alloc_set_pte(fe, NULL, page))
  2041. goto unlock;
  2042. unlock_page(page);
  2043. goto next;
  2044. unlock:
  2045. unlock_page(page);
  2046. skip:
  2047. put_page(page);
  2048. next:
  2049. /* Huge page is mapped? No need to proceed. */
  2050. if (pmd_trans_huge(*fe->pmd))
  2051. break;
  2052. if (iter.index == end_pgoff)
  2053. break;
  2054. }
  2055. rcu_read_unlock();
  2056. }
  2057. EXPORT_SYMBOL(filemap_map_pages);
  2058. int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
  2059. {
  2060. struct page *page = vmf->page;
  2061. struct inode *inode = file_inode(vma->vm_file);
  2062. int ret = VM_FAULT_LOCKED;
  2063. sb_start_pagefault(inode->i_sb);
  2064. file_update_time(vma->vm_file);
  2065. lock_page(page);
  2066. if (page->mapping != inode->i_mapping) {
  2067. unlock_page(page);
  2068. ret = VM_FAULT_NOPAGE;
  2069. goto out;
  2070. }
  2071. /*
  2072. * We mark the page dirty already here so that when freeze is in
  2073. * progress, we are guaranteed that writeback during freezing will
  2074. * see the dirty page and writeprotect it again.
  2075. */
  2076. set_page_dirty(page);
  2077. wait_for_stable_page(page);
  2078. out:
  2079. sb_end_pagefault(inode->i_sb);
  2080. return ret;
  2081. }
  2082. EXPORT_SYMBOL(filemap_page_mkwrite);
  2083. const struct vm_operations_struct generic_file_vm_ops = {
  2084. .fault = filemap_fault,
  2085. .map_pages = filemap_map_pages,
  2086. .page_mkwrite = filemap_page_mkwrite,
  2087. };
  2088. /* This is used for a general mmap of a disk file */
  2089. int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
  2090. {
  2091. struct address_space *mapping = file->f_mapping;
  2092. if (!mapping->a_ops->readpage)
  2093. return -ENOEXEC;
  2094. file_accessed(file);
  2095. vma->vm_ops = &generic_file_vm_ops;
  2096. return 0;
  2097. }
  2098. /*
  2099. * This is for filesystems which do not implement ->writepage.
  2100. */
  2101. int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
  2102. {
  2103. if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
  2104. return -EINVAL;
  2105. return generic_file_mmap(file, vma);
  2106. }
  2107. #else
  2108. int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
  2109. {
  2110. return -ENOSYS;
  2111. }
  2112. int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma)
  2113. {
  2114. return -ENOSYS;
  2115. }
  2116. #endif /* CONFIG_MMU */
  2117. EXPORT_SYMBOL(generic_file_mmap);
  2118. EXPORT_SYMBOL(generic_file_readonly_mmap);
  2119. static struct page *wait_on_page_read(struct page *page)
  2120. {
  2121. if (!IS_ERR(page)) {
  2122. wait_on_page_locked(page);
  2123. if (!PageUptodate(page)) {
  2124. put_page(page);
  2125. page = ERR_PTR(-EIO);
  2126. }
  2127. }
  2128. return page;
  2129. }
  2130. static struct page *do_read_cache_page(struct address_space *mapping,
  2131. pgoff_t index,
  2132. int (*filler)(void *, struct page *),
  2133. void *data,
  2134. gfp_t gfp)
  2135. {
  2136. struct page *page;
  2137. int err;
  2138. repeat:
  2139. page = find_get_page(mapping, index);
  2140. if (!page) {
  2141. page = __page_cache_alloc(gfp | __GFP_COLD);
  2142. if (!page)
  2143. return ERR_PTR(-ENOMEM);
  2144. err = add_to_page_cache_lru(page, mapping, index, gfp);
  2145. if (unlikely(err)) {
  2146. put_page(page);
  2147. if (err == -EEXIST)
  2148. goto repeat;
  2149. /* Presumably ENOMEM for radix tree node */
  2150. return ERR_PTR(err);
  2151. }
  2152. filler:
  2153. err = filler(data, page);
  2154. if (err < 0) {
  2155. put_page(page);
  2156. return ERR_PTR(err);
  2157. }
  2158. page = wait_on_page_read(page);
  2159. if (IS_ERR(page))
  2160. return page;
  2161. goto out;
  2162. }
  2163. if (PageUptodate(page))
  2164. goto out;
  2165. /*
  2166. * Page is not up to date and may be locked due one of the following
  2167. * case a: Page is being filled and the page lock is held
  2168. * case b: Read/write error clearing the page uptodate status
  2169. * case c: Truncation in progress (page locked)
  2170. * case d: Reclaim in progress
  2171. *
  2172. * Case a, the page will be up to date when the page is unlocked.
  2173. * There is no need to serialise on the page lock here as the page
  2174. * is pinned so the lock gives no additional protection. Even if the
  2175. * the page is truncated, the data is still valid if PageUptodate as
  2176. * it's a race vs truncate race.
  2177. * Case b, the page will not be up to date
  2178. * Case c, the page may be truncated but in itself, the data may still
  2179. * be valid after IO completes as it's a read vs truncate race. The
  2180. * operation must restart if the page is not uptodate on unlock but
  2181. * otherwise serialising on page lock to stabilise the mapping gives
  2182. * no additional guarantees to the caller as the page lock is
  2183. * released before return.
  2184. * Case d, similar to truncation. If reclaim holds the page lock, it
  2185. * will be a race with remove_mapping that determines if the mapping
  2186. * is valid on unlock but otherwise the data is valid and there is
  2187. * no need to serialise with page lock.
  2188. *
  2189. * As the page lock gives no additional guarantee, we optimistically
  2190. * wait on the page to be unlocked and check if it's up to date and
  2191. * use the page if it is. Otherwise, the page lock is required to
  2192. * distinguish between the different cases. The motivation is that we
  2193. * avoid spurious serialisations and wakeups when multiple processes
  2194. * wait on the same page for IO to complete.
  2195. */
  2196. wait_on_page_locked(page);
  2197. if (PageUptodate(page))
  2198. goto out;
  2199. /* Distinguish between all the cases under the safety of the lock */
  2200. lock_page(page);
  2201. /* Case c or d, restart the operation */
  2202. if (!page->mapping) {
  2203. unlock_page(page);
  2204. put_page(page);
  2205. goto repeat;
  2206. }
  2207. /* Someone else locked and filled the page in a very small window */
  2208. if (PageUptodate(page)) {
  2209. unlock_page(page);
  2210. goto out;
  2211. }
  2212. goto filler;
  2213. out:
  2214. mark_page_accessed(page);
  2215. return page;
  2216. }
  2217. /**
  2218. * read_cache_page - read into page cache, fill it if needed
  2219. * @mapping: the page's address_space
  2220. * @index: the page index
  2221. * @filler: function to perform the read
  2222. * @data: first arg to filler(data, page) function, often left as NULL
  2223. *
  2224. * Read into the page cache. If a page already exists, and PageUptodate() is
  2225. * not set, try to fill the page and wait for it to become unlocked.
  2226. *
  2227. * If the page does not get brought uptodate, return -EIO.
  2228. */
  2229. struct page *read_cache_page(struct address_space *mapping,
  2230. pgoff_t index,
  2231. int (*filler)(void *, struct page *),
  2232. void *data)
  2233. {
  2234. return do_read_cache_page(mapping, index, filler, data, mapping_gfp_mask(mapping));
  2235. }
  2236. EXPORT_SYMBOL(read_cache_page);
  2237. /**
  2238. * read_cache_page_gfp - read into page cache, using specified page allocation flags.
  2239. * @mapping: the page's address_space
  2240. * @index: the page index
  2241. * @gfp: the page allocator flags to use if allocating
  2242. *
  2243. * This is the same as "read_mapping_page(mapping, index, NULL)", but with
  2244. * any new page allocations done using the specified allocation flags.
  2245. *
  2246. * If the page does not get brought uptodate, return -EIO.
  2247. */
  2248. struct page *read_cache_page_gfp(struct address_space *mapping,
  2249. pgoff_t index,
  2250. gfp_t gfp)
  2251. {
  2252. filler_t *filler = (filler_t *)mapping->a_ops->readpage;
  2253. return do_read_cache_page(mapping, index, filler, NULL, gfp);
  2254. }
  2255. EXPORT_SYMBOL(read_cache_page_gfp);
  2256. /*
  2257. * Performs necessary checks before doing a write
  2258. *
  2259. * Can adjust writing position or amount of bytes to write.
  2260. * Returns appropriate error code that caller should return or
  2261. * zero in case that write should be allowed.
  2262. */
  2263. inline ssize_t generic_write_checks(struct kiocb *iocb, struct iov_iter *from)
  2264. {
  2265. struct file *file = iocb->ki_filp;
  2266. struct inode *inode = file->f_mapping->host;
  2267. unsigned long limit = rlimit(RLIMIT_FSIZE);
  2268. loff_t pos;
  2269. if (!iov_iter_count(from))
  2270. return 0;
  2271. /* FIXME: this is for backwards compatibility with 2.4 */
  2272. if (iocb->ki_flags & IOCB_APPEND)
  2273. iocb->ki_pos = i_size_read(inode);
  2274. pos = iocb->ki_pos;
  2275. if (limit != RLIM_INFINITY) {
  2276. if (iocb->ki_pos >= limit) {
  2277. send_sig(SIGXFSZ, current, 0);
  2278. return -EFBIG;
  2279. }
  2280. iov_iter_truncate(from, limit - (unsigned long)pos);
  2281. }
  2282. /*
  2283. * LFS rule
  2284. */
  2285. if (unlikely(pos + iov_iter_count(from) > MAX_NON_LFS &&
  2286. !(file->f_flags & O_LARGEFILE))) {
  2287. if (pos >= MAX_NON_LFS)
  2288. return -EFBIG;
  2289. iov_iter_truncate(from, MAX_NON_LFS - (unsigned long)pos);
  2290. }
  2291. /*
  2292. * Are we about to exceed the fs block limit ?
  2293. *
  2294. * If we have written data it becomes a short write. If we have
  2295. * exceeded without writing data we send a signal and return EFBIG.
  2296. * Linus frestrict idea will clean these up nicely..
  2297. */
  2298. if (unlikely(pos >= inode->i_sb->s_maxbytes))
  2299. return -EFBIG;
  2300. iov_iter_truncate(from, inode->i_sb->s_maxbytes - pos);
  2301. return iov_iter_count(from);
  2302. }
  2303. EXPORT_SYMBOL(generic_write_checks);
  2304. int pagecache_write_begin(struct file *file, struct address_space *mapping,
  2305. loff_t pos, unsigned len, unsigned flags,
  2306. struct page **pagep, void **fsdata)
  2307. {
  2308. const struct address_space_operations *aops = mapping->a_ops;
  2309. return aops->write_begin(file, mapping, pos, len, flags,
  2310. pagep, fsdata);
  2311. }
  2312. EXPORT_SYMBOL(pagecache_write_begin);
  2313. int pagecache_write_end(struct file *file, struct address_space *mapping,
  2314. loff_t pos, unsigned len, unsigned copied,
  2315. struct page *page, void *fsdata)
  2316. {
  2317. const struct address_space_operations *aops = mapping->a_ops;
  2318. return aops->write_end(file, mapping, pos, len, copied, page, fsdata);
  2319. }
  2320. EXPORT_SYMBOL(pagecache_write_end);
  2321. ssize_t
  2322. generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from)
  2323. {
  2324. struct file *file = iocb->ki_filp;
  2325. struct address_space *mapping = file->f_mapping;
  2326. struct inode *inode = mapping->host;
  2327. loff_t pos = iocb->ki_pos;
  2328. ssize_t written;
  2329. size_t write_len;
  2330. pgoff_t end;
  2331. struct iov_iter data;
  2332. write_len = iov_iter_count(from);
  2333. end = (pos + write_len - 1) >> PAGE_SHIFT;
  2334. written = filemap_write_and_wait_range(mapping, pos, pos + write_len - 1);
  2335. if (written)
  2336. goto out;
  2337. /*
  2338. * After a write we want buffered reads to be sure to go to disk to get
  2339. * the new data. We invalidate clean cached page from the region we're
  2340. * about to write. We do this *before* the write so that we can return
  2341. * without clobbering -EIOCBQUEUED from ->direct_IO().
  2342. */
  2343. if (mapping->nrpages) {
  2344. written = invalidate_inode_pages2_range(mapping,
  2345. pos >> PAGE_SHIFT, end);
  2346. /*
  2347. * If a page can not be invalidated, return 0 to fall back
  2348. * to buffered write.
  2349. */
  2350. if (written) {
  2351. if (written == -EBUSY)
  2352. return 0;
  2353. goto out;
  2354. }
  2355. }
  2356. data = *from;
  2357. written = mapping->a_ops->direct_IO(iocb, &data);
  2358. /*
  2359. * Finally, try again to invalidate clean pages which might have been
  2360. * cached by non-direct readahead, or faulted in by get_user_pages()
  2361. * if the source of the write was an mmap'ed region of the file
  2362. * we're writing. Either one is a pretty crazy thing to do,
  2363. * so we don't support it 100%. If this invalidation
  2364. * fails, tough, the write still worked...
  2365. */
  2366. if (mapping->nrpages) {
  2367. invalidate_inode_pages2_range(mapping,
  2368. pos >> PAGE_SHIFT, end);
  2369. }
  2370. if (written > 0) {
  2371. pos += written;
  2372. iov_iter_advance(from, written);
  2373. if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) {
  2374. i_size_write(inode, pos);
  2375. mark_inode_dirty(inode);
  2376. }
  2377. iocb->ki_pos = pos;
  2378. }
  2379. out:
  2380. return written;
  2381. }
  2382. EXPORT_SYMBOL(generic_file_direct_write);
  2383. /*
  2384. * Find or create a page at the given pagecache position. Return the locked
  2385. * page. This function is specifically for buffered writes.
  2386. */
  2387. struct page *grab_cache_page_write_begin(struct address_space *mapping,
  2388. pgoff_t index, unsigned flags)
  2389. {
  2390. struct page *page;
  2391. int fgp_flags = FGP_LOCK|FGP_WRITE|FGP_CREAT;
  2392. if (flags & AOP_FLAG_NOFS)
  2393. fgp_flags |= FGP_NOFS;
  2394. page = pagecache_get_page(mapping, index, fgp_flags,
  2395. mapping_gfp_mask(mapping));
  2396. if (page)
  2397. wait_for_stable_page(page);
  2398. return page;
  2399. }
  2400. EXPORT_SYMBOL(grab_cache_page_write_begin);
  2401. ssize_t generic_perform_write(struct file *file,
  2402. struct iov_iter *i, loff_t pos)
  2403. {
  2404. struct address_space *mapping = file->f_mapping;
  2405. const struct address_space_operations *a_ops = mapping->a_ops;
  2406. long status = 0;
  2407. ssize_t written = 0;
  2408. unsigned int flags = 0;
  2409. /*
  2410. * Copies from kernel address space cannot fail (NFSD is a big user).
  2411. */
  2412. if (!iter_is_iovec(i))
  2413. flags |= AOP_FLAG_UNINTERRUPTIBLE;
  2414. do {
  2415. struct page *page;
  2416. unsigned long offset; /* Offset into pagecache page */
  2417. unsigned long bytes; /* Bytes to write to page */
  2418. size_t copied; /* Bytes copied from user */
  2419. void *fsdata;
  2420. offset = (pos & (PAGE_SIZE - 1));
  2421. bytes = min_t(unsigned long, PAGE_SIZE - offset,
  2422. iov_iter_count(i));
  2423. again:
  2424. /*
  2425. * Bring in the user page that we will copy from _first_.
  2426. * Otherwise there's a nasty deadlock on copying from the
  2427. * same page as we're writing to, without it being marked
  2428. * up-to-date.
  2429. *
  2430. * Not only is this an optimisation, but it is also required
  2431. * to check that the address is actually valid, when atomic
  2432. * usercopies are used, below.
  2433. */
  2434. if (unlikely(iov_iter_fault_in_readable(i, bytes))) {
  2435. status = -EFAULT;
  2436. break;
  2437. }
  2438. if (fatal_signal_pending(current)) {
  2439. status = -EINTR;
  2440. break;
  2441. }
  2442. status = a_ops->write_begin(file, mapping, pos, bytes, flags,
  2443. &page, &fsdata);
  2444. if (unlikely(status < 0))
  2445. break;
  2446. if (mapping_writably_mapped(mapping))
  2447. flush_dcache_page(page);
  2448. copied = iov_iter_copy_from_user_atomic(page, i, offset, bytes);
  2449. flush_dcache_page(page);
  2450. status = a_ops->write_end(file, mapping, pos, bytes, copied,
  2451. page, fsdata);
  2452. if (unlikely(status < 0))
  2453. break;
  2454. copied = status;
  2455. cond_resched();
  2456. iov_iter_advance(i, copied);
  2457. if (unlikely(copied == 0)) {
  2458. /*
  2459. * If we were unable to copy any data at all, we must
  2460. * fall back to a single segment length write.
  2461. *
  2462. * If we didn't fallback here, we could livelock
  2463. * because not all segments in the iov can be copied at
  2464. * once without a pagefault.
  2465. */
  2466. bytes = min_t(unsigned long, PAGE_SIZE - offset,
  2467. iov_iter_single_seg_count(i));
  2468. goto again;
  2469. }
  2470. pos += copied;
  2471. written += copied;
  2472. balance_dirty_pages_ratelimited(mapping);
  2473. } while (iov_iter_count(i));
  2474. return written ? written : status;
  2475. }
  2476. EXPORT_SYMBOL(generic_perform_write);
  2477. /**
  2478. * __generic_file_write_iter - write data to a file
  2479. * @iocb: IO state structure (file, offset, etc.)
  2480. * @from: iov_iter with data to write
  2481. *
  2482. * This function does all the work needed for actually writing data to a
  2483. * file. It does all basic checks, removes SUID from the file, updates
  2484. * modification times and calls proper subroutines depending on whether we
  2485. * do direct IO or a standard buffered write.
  2486. *
  2487. * It expects i_mutex to be grabbed unless we work on a block device or similar
  2488. * object which does not need locking at all.
  2489. *
  2490. * This function does *not* take care of syncing data in case of O_SYNC write.
  2491. * A caller has to handle it. This is mainly due to the fact that we want to
  2492. * avoid syncing under i_mutex.
  2493. */
  2494. ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
  2495. {
  2496. struct file *file = iocb->ki_filp;
  2497. struct address_space * mapping = file->f_mapping;
  2498. struct inode *inode = mapping->host;
  2499. ssize_t written = 0;
  2500. ssize_t err;
  2501. ssize_t status;
  2502. /* We can write back this queue in page reclaim */
  2503. current->backing_dev_info = inode_to_bdi(inode);
  2504. err = file_remove_privs(file);
  2505. if (err)
  2506. goto out;
  2507. err = file_update_time(file);
  2508. if (err)
  2509. goto out;
  2510. if (iocb->ki_flags & IOCB_DIRECT) {
  2511. loff_t pos, endbyte;
  2512. written = generic_file_direct_write(iocb, from);
  2513. /*
  2514. * If the write stopped short of completing, fall back to
  2515. * buffered writes. Some filesystems do this for writes to
  2516. * holes, for example. For DAX files, a buffered write will
  2517. * not succeed (even if it did, DAX does not handle dirty
  2518. * page-cache pages correctly).
  2519. */
  2520. if (written < 0 || !iov_iter_count(from) || IS_DAX(inode))
  2521. goto out;
  2522. status = generic_perform_write(file, from, pos = iocb->ki_pos);
  2523. /*
  2524. * If generic_perform_write() returned a synchronous error
  2525. * then we want to return the number of bytes which were
  2526. * direct-written, or the error code if that was zero. Note
  2527. * that this differs from normal direct-io semantics, which
  2528. * will return -EFOO even if some bytes were written.
  2529. */
  2530. if (unlikely(status < 0)) {
  2531. err = status;
  2532. goto out;
  2533. }
  2534. /*
  2535. * We need to ensure that the page cache pages are written to
  2536. * disk and invalidated to preserve the expected O_DIRECT
  2537. * semantics.
  2538. */
  2539. endbyte = pos + status - 1;
  2540. err = filemap_write_and_wait_range(mapping, pos, endbyte);
  2541. if (err == 0) {
  2542. iocb->ki_pos = endbyte + 1;
  2543. written += status;
  2544. invalidate_mapping_pages(mapping,
  2545. pos >> PAGE_SHIFT,
  2546. endbyte >> PAGE_SHIFT);
  2547. } else {
  2548. /*
  2549. * We don't know how much we wrote, so just return
  2550. * the number of bytes which were direct-written
  2551. */
  2552. }
  2553. } else {
  2554. written = generic_perform_write(file, from, iocb->ki_pos);
  2555. if (likely(written > 0))
  2556. iocb->ki_pos += written;
  2557. }
  2558. out:
  2559. current->backing_dev_info = NULL;
  2560. return written ? written : err;
  2561. }
  2562. EXPORT_SYMBOL(__generic_file_write_iter);
  2563. /**
  2564. * generic_file_write_iter - write data to a file
  2565. * @iocb: IO state structure
  2566. * @from: iov_iter with data to write
  2567. *
  2568. * This is a wrapper around __generic_file_write_iter() to be used by most
  2569. * filesystems. It takes care of syncing the file in case of O_SYNC file
  2570. * and acquires i_mutex as needed.
  2571. */
  2572. ssize_t generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
  2573. {
  2574. struct file *file = iocb->ki_filp;
  2575. struct inode *inode = file->f_mapping->host;
  2576. ssize_t ret;
  2577. inode_lock(inode);
  2578. ret = generic_write_checks(iocb, from);
  2579. if (ret > 0)
  2580. ret = __generic_file_write_iter(iocb, from);
  2581. inode_unlock(inode);
  2582. if (ret > 0)
  2583. ret = generic_write_sync(iocb, ret);
  2584. return ret;
  2585. }
  2586. EXPORT_SYMBOL(generic_file_write_iter);
  2587. /**
  2588. * try_to_release_page() - release old fs-specific metadata on a page
  2589. *
  2590. * @page: the page which the kernel is trying to free
  2591. * @gfp_mask: memory allocation flags (and I/O mode)
  2592. *
  2593. * The address_space is to try to release any data against the page
  2594. * (presumably at page->private). If the release was successful, return `1'.
  2595. * Otherwise return zero.
  2596. *
  2597. * This may also be called if PG_fscache is set on a page, indicating that the
  2598. * page is known to the local caching routines.
  2599. *
  2600. * The @gfp_mask argument specifies whether I/O may be performed to release
  2601. * this page (__GFP_IO), and whether the call may block (__GFP_RECLAIM & __GFP_FS).
  2602. *
  2603. */
  2604. int try_to_release_page(struct page *page, gfp_t gfp_mask)
  2605. {
  2606. struct address_space * const mapping = page->mapping;
  2607. BUG_ON(!PageLocked(page));
  2608. if (PageWriteback(page))
  2609. return 0;
  2610. if (mapping && mapping->a_ops->releasepage)
  2611. return mapping->a_ops->releasepage(page, gfp_mask);
  2612. return try_to_free_buffers(page);
  2613. }
  2614. EXPORT_SYMBOL(try_to_release_page);