123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822 |
- #include <linux/kernel.h>
- #include <linux/backing-dev.h>
- #include <linux/dax.h>
- #include <linux/gfp.h>
- #include <linux/mm.h>
- #include <linux/swap.h>
- #include <linux/export.h>
- #include <linux/pagemap.h>
- #include <linux/highmem.h>
- #include <linux/pagevec.h>
- #include <linux/task_io_accounting_ops.h>
- #include <linux/buffer_head.h> /* grr. try_to_release_page,
- do_invalidatepage */
- #include <linux/cleancache.h>
- #include <linux/rmap.h>
- #include "internal.h"
- static void clear_exceptional_entry(struct address_space *mapping,
- pgoff_t index, void *entry)
- {
- struct radix_tree_node *node;
- void **slot;
-
- if (shmem_mapping(mapping))
- return;
- if (dax_mapping(mapping)) {
- dax_delete_mapping_entry(mapping, index);
- return;
- }
- spin_lock_irq(&mapping->tree_lock);
-
- if (!__radix_tree_lookup(&mapping->page_tree, index, &node,
- &slot))
- goto unlock;
- if (*slot != entry)
- goto unlock;
- radix_tree_replace_slot(slot, NULL);
- mapping->nrexceptional--;
- if (!node)
- goto unlock;
- workingset_node_shadows_dec(node);
-
- if (!workingset_node_shadows(node) &&
- !list_empty(&node->private_list))
- list_lru_del(&workingset_shadow_nodes,
- &node->private_list);
- __radix_tree_delete_node(&mapping->page_tree, node);
- unlock:
- spin_unlock_irq(&mapping->tree_lock);
- }
- void do_invalidatepage(struct page *page, unsigned int offset,
- unsigned int length)
- {
- void (*invalidatepage)(struct page *, unsigned int, unsigned int);
- invalidatepage = page->mapping->a_ops->invalidatepage;
- #ifdef CONFIG_BLOCK
- if (!invalidatepage)
- invalidatepage = block_invalidatepage;
- #endif
- if (invalidatepage)
- (*invalidatepage)(page, offset, length);
- }
- static int
- truncate_complete_page(struct address_space *mapping, struct page *page)
- {
- if (page->mapping != mapping)
- return -EIO;
- if (page_has_private(page))
- do_invalidatepage(page, 0, PAGE_SIZE);
-
- cancel_dirty_page(page);
- ClearPageMappedToDisk(page);
- delete_from_page_cache(page);
- return 0;
- }
- static int
- invalidate_complete_page(struct address_space *mapping, struct page *page)
- {
- int ret;
- if (page->mapping != mapping)
- return 0;
- if (page_has_private(page) && !try_to_release_page(page, 0))
- return 0;
- ret = remove_mapping(mapping, page);
- return ret;
- }
- int truncate_inode_page(struct address_space *mapping, struct page *page)
- {
- loff_t holelen;
- VM_BUG_ON_PAGE(PageTail(page), page);
- holelen = PageTransHuge(page) ? HPAGE_PMD_SIZE : PAGE_SIZE;
- if (page_mapped(page)) {
- unmap_mapping_range(mapping,
- (loff_t)page->index << PAGE_SHIFT,
- holelen, 0);
- }
- return truncate_complete_page(mapping, page);
- }
- int generic_error_remove_page(struct address_space *mapping, struct page *page)
- {
- if (!mapping)
- return -EINVAL;
-
- if (!S_ISREG(mapping->host->i_mode))
- return -EIO;
- return truncate_inode_page(mapping, page);
- }
- EXPORT_SYMBOL(generic_error_remove_page);
- int invalidate_inode_page(struct page *page)
- {
- struct address_space *mapping = page_mapping(page);
- if (!mapping)
- return 0;
- if (PageDirty(page) || PageWriteback(page))
- return 0;
- if (page_mapped(page))
- return 0;
- return invalidate_complete_page(mapping, page);
- }
- void truncate_inode_pages_range(struct address_space *mapping,
- loff_t lstart, loff_t lend)
- {
- pgoff_t start;
- pgoff_t end;
- unsigned int partial_start;
- unsigned int partial_end;
- struct pagevec pvec;
- pgoff_t indices[PAGEVEC_SIZE];
- pgoff_t index;
- int i;
- cleancache_invalidate_inode(mapping);
- if (mapping->nrpages == 0 && mapping->nrexceptional == 0)
- return;
-
- partial_start = lstart & (PAGE_SIZE - 1);
- partial_end = (lend + 1) & (PAGE_SIZE - 1);
-
- start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
- if (lend == -1)
-
- end = -1;
- else
- end = (lend + 1) >> PAGE_SHIFT;
- pagevec_init(&pvec, 0);
- index = start;
- while (index < end && pagevec_lookup_entries(&pvec, mapping, index,
- min(end - index, (pgoff_t)PAGEVEC_SIZE),
- indices)) {
- for (i = 0; i < pagevec_count(&pvec); i++) {
- struct page *page = pvec.pages[i];
-
- index = indices[i];
- if (index >= end)
- break;
- if (radix_tree_exceptional_entry(page)) {
- clear_exceptional_entry(mapping, index, page);
- continue;
- }
- if (!trylock_page(page))
- continue;
- WARN_ON(page_to_index(page) != index);
- if (PageWriteback(page)) {
- unlock_page(page);
- continue;
- }
- truncate_inode_page(mapping, page);
- unlock_page(page);
- }
- pagevec_remove_exceptionals(&pvec);
- pagevec_release(&pvec);
- cond_resched();
- index++;
- }
- if (partial_start) {
- struct page *page = find_lock_page(mapping, start - 1);
- if (page) {
- unsigned int top = PAGE_SIZE;
- if (start > end) {
-
- top = partial_end;
- partial_end = 0;
- }
- wait_on_page_writeback(page);
- zero_user_segment(page, partial_start, top);
- cleancache_invalidate_page(mapping, page);
- if (page_has_private(page))
- do_invalidatepage(page, partial_start,
- top - partial_start);
- unlock_page(page);
- put_page(page);
- }
- }
- if (partial_end) {
- struct page *page = find_lock_page(mapping, end);
- if (page) {
- wait_on_page_writeback(page);
- zero_user_segment(page, 0, partial_end);
- cleancache_invalidate_page(mapping, page);
- if (page_has_private(page))
- do_invalidatepage(page, 0,
- partial_end);
- unlock_page(page);
- put_page(page);
- }
- }
-
- if (start >= end)
- return;
- index = start;
- for ( ; ; ) {
- cond_resched();
- if (!pagevec_lookup_entries(&pvec, mapping, index,
- min(end - index, (pgoff_t)PAGEVEC_SIZE), indices)) {
-
- if (index == start)
- break;
-
- index = start;
- continue;
- }
- if (index == start && indices[0] >= end) {
-
- pagevec_remove_exceptionals(&pvec);
- pagevec_release(&pvec);
- break;
- }
- for (i = 0; i < pagevec_count(&pvec); i++) {
- struct page *page = pvec.pages[i];
-
- index = indices[i];
- if (index >= end) {
-
- index = start - 1;
- break;
- }
- if (radix_tree_exceptional_entry(page)) {
- clear_exceptional_entry(mapping, index, page);
- continue;
- }
- lock_page(page);
- WARN_ON(page_to_index(page) != index);
- wait_on_page_writeback(page);
- truncate_inode_page(mapping, page);
- unlock_page(page);
- }
- pagevec_remove_exceptionals(&pvec);
- pagevec_release(&pvec);
- index++;
- }
- cleancache_invalidate_inode(mapping);
- }
- EXPORT_SYMBOL(truncate_inode_pages_range);
- void truncate_inode_pages(struct address_space *mapping, loff_t lstart)
- {
- truncate_inode_pages_range(mapping, lstart, (loff_t)-1);
- }
- EXPORT_SYMBOL(truncate_inode_pages);
- void truncate_inode_pages_final(struct address_space *mapping)
- {
- unsigned long nrexceptional;
- unsigned long nrpages;
-
- mapping_set_exiting(mapping);
-
- nrpages = mapping->nrpages;
- smp_rmb();
- nrexceptional = mapping->nrexceptional;
- if (nrpages || nrexceptional) {
-
- spin_lock_irq(&mapping->tree_lock);
- spin_unlock_irq(&mapping->tree_lock);
- truncate_inode_pages(mapping, 0);
- }
- }
- EXPORT_SYMBOL(truncate_inode_pages_final);
- unsigned long invalidate_mapping_pages(struct address_space *mapping,
- pgoff_t start, pgoff_t end)
- {
- pgoff_t indices[PAGEVEC_SIZE];
- struct pagevec pvec;
- pgoff_t index = start;
- unsigned long ret;
- unsigned long count = 0;
- int i;
- pagevec_init(&pvec, 0);
- while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,
- min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
- indices)) {
- for (i = 0; i < pagevec_count(&pvec); i++) {
- struct page *page = pvec.pages[i];
-
- index = indices[i];
- if (index > end)
- break;
- if (radix_tree_exceptional_entry(page)) {
- clear_exceptional_entry(mapping, index, page);
- continue;
- }
- if (!trylock_page(page))
- continue;
- WARN_ON(page_to_index(page) != index);
-
- if (PageTransTail(page)) {
- unlock_page(page);
- continue;
- } else if (PageTransHuge(page)) {
- index += HPAGE_PMD_NR - 1;
- i += HPAGE_PMD_NR - 1;
-
- if (index == round_down(end, HPAGE_PMD_NR))
- continue;
- }
- ret = invalidate_inode_page(page);
- unlock_page(page);
-
- if (!ret)
- deactivate_file_page(page);
- count += ret;
- }
- pagevec_remove_exceptionals(&pvec);
- pagevec_release(&pvec);
- cond_resched();
- index++;
- }
- return count;
- }
- EXPORT_SYMBOL(invalidate_mapping_pages);
- static int
- invalidate_complete_page2(struct address_space *mapping, struct page *page)
- {
- unsigned long flags;
- if (page->mapping != mapping)
- return 0;
- if (page_has_private(page) && !try_to_release_page(page, GFP_KERNEL))
- return 0;
- spin_lock_irqsave(&mapping->tree_lock, flags);
- if (PageDirty(page))
- goto failed;
- BUG_ON(page_has_private(page));
- __delete_from_page_cache(page, NULL);
- spin_unlock_irqrestore(&mapping->tree_lock, flags);
- if (mapping->a_ops->freepage)
- mapping->a_ops->freepage(page);
- put_page(page);
- return 1;
- failed:
- spin_unlock_irqrestore(&mapping->tree_lock, flags);
- return 0;
- }
- static int do_launder_page(struct address_space *mapping, struct page *page)
- {
- if (!PageDirty(page))
- return 0;
- if (page->mapping != mapping || mapping->a_ops->launder_page == NULL)
- return 0;
- return mapping->a_ops->launder_page(page);
- }
- int invalidate_inode_pages2_range(struct address_space *mapping,
- pgoff_t start, pgoff_t end)
- {
- pgoff_t indices[PAGEVEC_SIZE];
- struct pagevec pvec;
- pgoff_t index;
- int i;
- int ret = 0;
- int ret2 = 0;
- int did_range_unmap = 0;
- cleancache_invalidate_inode(mapping);
- pagevec_init(&pvec, 0);
- index = start;
- while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,
- min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
- indices)) {
- for (i = 0; i < pagevec_count(&pvec); i++) {
- struct page *page = pvec.pages[i];
-
- index = indices[i];
- if (index > end)
- break;
- if (radix_tree_exceptional_entry(page)) {
- clear_exceptional_entry(mapping, index, page);
- continue;
- }
- lock_page(page);
- WARN_ON(page_to_index(page) != index);
- if (page->mapping != mapping) {
- unlock_page(page);
- continue;
- }
- wait_on_page_writeback(page);
- if (page_mapped(page)) {
- if (!did_range_unmap) {
-
- unmap_mapping_range(mapping,
- (loff_t)index << PAGE_SHIFT,
- (loff_t)(1 + end - index)
- << PAGE_SHIFT,
- 0);
- did_range_unmap = 1;
- } else {
-
- unmap_mapping_range(mapping,
- (loff_t)index << PAGE_SHIFT,
- PAGE_SIZE, 0);
- }
- }
- BUG_ON(page_mapped(page));
- ret2 = do_launder_page(mapping, page);
- if (ret2 == 0) {
- if (!invalidate_complete_page2(mapping, page))
- ret2 = -EBUSY;
- }
- if (ret2 < 0)
- ret = ret2;
- unlock_page(page);
- }
- pagevec_remove_exceptionals(&pvec);
- pagevec_release(&pvec);
- cond_resched();
- index++;
- }
- cleancache_invalidate_inode(mapping);
- return ret;
- }
- EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);
- int invalidate_inode_pages2(struct address_space *mapping)
- {
- return invalidate_inode_pages2_range(mapping, 0, -1);
- }
- EXPORT_SYMBOL_GPL(invalidate_inode_pages2);
- void truncate_pagecache(struct inode *inode, loff_t newsize)
- {
- struct address_space *mapping = inode->i_mapping;
- loff_t holebegin = round_up(newsize, PAGE_SIZE);
-
- unmap_mapping_range(mapping, holebegin, 0, 1);
- truncate_inode_pages(mapping, newsize);
- unmap_mapping_range(mapping, holebegin, 0, 1);
- }
- EXPORT_SYMBOL(truncate_pagecache);
- void truncate_setsize(struct inode *inode, loff_t newsize)
- {
- loff_t oldsize = inode->i_size;
- i_size_write(inode, newsize);
- if (newsize > oldsize)
- pagecache_isize_extended(inode, oldsize, newsize);
- truncate_pagecache(inode, newsize);
- }
- EXPORT_SYMBOL(truncate_setsize);
- void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to)
- {
- int bsize = i_blocksize(inode);
- loff_t rounded_from;
- struct page *page;
- pgoff_t index;
- WARN_ON(to > inode->i_size);
- if (from >= to || bsize == PAGE_SIZE)
- return;
-
- rounded_from = round_up(from, bsize);
- if (to <= rounded_from || !(rounded_from & (PAGE_SIZE - 1)))
- return;
- index = from >> PAGE_SHIFT;
- page = find_lock_page(inode->i_mapping, index);
-
- if (!page)
- return;
-
- if (page_mkclean(page))
- set_page_dirty(page);
- unlock_page(page);
- put_page(page);
- }
- EXPORT_SYMBOL(pagecache_isize_extended);
- void truncate_pagecache_range(struct inode *inode, loff_t lstart, loff_t lend)
- {
- struct address_space *mapping = inode->i_mapping;
- loff_t unmap_start = round_up(lstart, PAGE_SIZE);
- loff_t unmap_end = round_down(1 + lend, PAGE_SIZE) - 1;
-
-
- if ((u64)unmap_end > (u64)unmap_start)
- unmap_mapping_range(mapping, unmap_start,
- 1 + unmap_end - unmap_start, 0);
- truncate_inode_pages_range(mapping, lstart, lend);
- }
- EXPORT_SYMBOL(truncate_pagecache_range);
|