12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952 |
- #include <linux/mm.h>
- #include <linux/pagemap.h>
- #include <linux/swap.h>
- #include <linux/swapops.h>
- #include <linux/slab.h>
- #include <linux/init.h>
- #include <linux/ksm.h>
- #include <linux/rmap.h>
- #include <linux/rcupdate.h>
- #include <linux/export.h>
- #include <linux/memcontrol.h>
- #include <linux/mmu_notifier.h>
- #include <linux/migrate.h>
- #include <linux/hugetlb.h>
- #include <linux/backing-dev.h>
- #include <linux/page_idle.h>
- #include <asm/tlbflush.h>
- #include <trace/events/tlb.h>
- #include "internal.h"
- static struct kmem_cache *anon_vma_cachep;
- static struct kmem_cache *anon_vma_chain_cachep;
- static inline struct anon_vma *anon_vma_alloc(void)
- {
- struct anon_vma *anon_vma;
- anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL);
- if (anon_vma) {
- atomic_set(&anon_vma->refcount, 1);
- anon_vma->degree = 1;
- anon_vma->parent = anon_vma;
-
- anon_vma->root = anon_vma;
- }
- return anon_vma;
- }
- static inline void anon_vma_free(struct anon_vma *anon_vma)
- {
- VM_BUG_ON(atomic_read(&anon_vma->refcount));
-
- might_sleep();
- if (rwsem_is_locked(&anon_vma->root->rwsem)) {
- anon_vma_lock_write(anon_vma);
- anon_vma_unlock_write(anon_vma);
- }
- kmem_cache_free(anon_vma_cachep, anon_vma);
- }
- static inline struct anon_vma_chain *anon_vma_chain_alloc(gfp_t gfp)
- {
- return kmem_cache_alloc(anon_vma_chain_cachep, gfp);
- }
- static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain)
- {
- kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain);
- }
- static void anon_vma_chain_link(struct vm_area_struct *vma,
- struct anon_vma_chain *avc,
- struct anon_vma *anon_vma)
- {
- avc->vma = vma;
- avc->anon_vma = anon_vma;
- list_add(&avc->same_vma, &vma->anon_vma_chain);
- anon_vma_interval_tree_insert(avc, &anon_vma->rb_root);
- }
- int anon_vma_prepare(struct vm_area_struct *vma)
- {
- struct anon_vma *anon_vma = vma->anon_vma;
- struct anon_vma_chain *avc;
- might_sleep();
- if (unlikely(!anon_vma)) {
- struct mm_struct *mm = vma->vm_mm;
- struct anon_vma *allocated;
- avc = anon_vma_chain_alloc(GFP_KERNEL);
- if (!avc)
- goto out_enomem;
- anon_vma = find_mergeable_anon_vma(vma);
- allocated = NULL;
- if (!anon_vma) {
- anon_vma = anon_vma_alloc();
- if (unlikely(!anon_vma))
- goto out_enomem_free_avc;
- allocated = anon_vma;
- }
- anon_vma_lock_write(anon_vma);
-
- spin_lock(&mm->page_table_lock);
- if (likely(!vma->anon_vma)) {
- vma->anon_vma = anon_vma;
- anon_vma_chain_link(vma, avc, anon_vma);
-
- anon_vma->degree++;
- allocated = NULL;
- avc = NULL;
- }
- spin_unlock(&mm->page_table_lock);
- anon_vma_unlock_write(anon_vma);
- if (unlikely(allocated))
- put_anon_vma(allocated);
- if (unlikely(avc))
- anon_vma_chain_free(avc);
- }
- return 0;
- out_enomem_free_avc:
- anon_vma_chain_free(avc);
- out_enomem:
- return -ENOMEM;
- }
- static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma)
- {
- struct anon_vma *new_root = anon_vma->root;
- if (new_root != root) {
- if (WARN_ON_ONCE(root))
- up_write(&root->rwsem);
- root = new_root;
- down_write(&root->rwsem);
- }
- return root;
- }
- static inline void unlock_anon_vma_root(struct anon_vma *root)
- {
- if (root)
- up_write(&root->rwsem);
- }
- int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src)
- {
- struct anon_vma_chain *avc, *pavc;
- struct anon_vma *root = NULL;
- list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) {
- struct anon_vma *anon_vma;
- avc = anon_vma_chain_alloc(GFP_NOWAIT | __GFP_NOWARN);
- if (unlikely(!avc)) {
- unlock_anon_vma_root(root);
- root = NULL;
- avc = anon_vma_chain_alloc(GFP_KERNEL);
- if (!avc)
- goto enomem_failure;
- }
- anon_vma = pavc->anon_vma;
- root = lock_anon_vma_root(root, anon_vma);
- anon_vma_chain_link(dst, avc, anon_vma);
-
- if (!dst->anon_vma && anon_vma != src->anon_vma &&
- anon_vma->degree < 2)
- dst->anon_vma = anon_vma;
- }
- if (dst->anon_vma)
- dst->anon_vma->degree++;
- unlock_anon_vma_root(root);
- return 0;
- enomem_failure:
-
- dst->anon_vma = NULL;
- unlink_anon_vmas(dst);
- return -ENOMEM;
- }
- int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
- {
- struct anon_vma_chain *avc;
- struct anon_vma *anon_vma;
- int error;
-
- if (!pvma->anon_vma)
- return 0;
-
- vma->anon_vma = NULL;
-
- error = anon_vma_clone(vma, pvma);
- if (error)
- return error;
-
- if (vma->anon_vma)
- return 0;
-
- anon_vma = anon_vma_alloc();
- if (!anon_vma)
- goto out_error;
- avc = anon_vma_chain_alloc(GFP_KERNEL);
- if (!avc)
- goto out_error_free_anon_vma;
-
- anon_vma->root = pvma->anon_vma->root;
- anon_vma->parent = pvma->anon_vma;
-
- get_anon_vma(anon_vma->root);
-
- vma->anon_vma = anon_vma;
- anon_vma_lock_write(anon_vma);
- anon_vma_chain_link(vma, avc, anon_vma);
- anon_vma->parent->degree++;
- anon_vma_unlock_write(anon_vma);
- return 0;
- out_error_free_anon_vma:
- put_anon_vma(anon_vma);
- out_error:
- unlink_anon_vmas(vma);
- return -ENOMEM;
- }
- void unlink_anon_vmas(struct vm_area_struct *vma)
- {
- struct anon_vma_chain *avc, *next;
- struct anon_vma *root = NULL;
-
- list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
- struct anon_vma *anon_vma = avc->anon_vma;
- root = lock_anon_vma_root(root, anon_vma);
- anon_vma_interval_tree_remove(avc, &anon_vma->rb_root);
-
- if (RB_EMPTY_ROOT(&anon_vma->rb_root)) {
- anon_vma->parent->degree--;
- continue;
- }
- list_del(&avc->same_vma);
- anon_vma_chain_free(avc);
- }
- if (vma->anon_vma)
- vma->anon_vma->degree--;
- unlock_anon_vma_root(root);
-
- list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
- struct anon_vma *anon_vma = avc->anon_vma;
- VM_WARN_ON(anon_vma->degree);
- put_anon_vma(anon_vma);
- list_del(&avc->same_vma);
- anon_vma_chain_free(avc);
- }
- }
- static void anon_vma_ctor(void *data)
- {
- struct anon_vma *anon_vma = data;
- init_rwsem(&anon_vma->rwsem);
- atomic_set(&anon_vma->refcount, 0);
- anon_vma->rb_root = RB_ROOT;
- }
- void __init anon_vma_init(void)
- {
- anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma),
- 0, SLAB_DESTROY_BY_RCU|SLAB_PANIC|SLAB_ACCOUNT,
- anon_vma_ctor);
- anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain,
- SLAB_PANIC|SLAB_ACCOUNT);
- }
- struct anon_vma *page_get_anon_vma(struct page *page)
- {
- struct anon_vma *anon_vma = NULL;
- unsigned long anon_mapping;
- rcu_read_lock();
- anon_mapping = (unsigned long)READ_ONCE(page->mapping);
- if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
- goto out;
- if (!page_mapped(page))
- goto out;
- anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
- if (!atomic_inc_not_zero(&anon_vma->refcount)) {
- anon_vma = NULL;
- goto out;
- }
-
- if (!page_mapped(page)) {
- rcu_read_unlock();
- put_anon_vma(anon_vma);
- return NULL;
- }
- out:
- rcu_read_unlock();
- return anon_vma;
- }
- struct anon_vma *page_lock_anon_vma_read(struct page *page)
- {
- struct anon_vma *anon_vma = NULL;
- struct anon_vma *root_anon_vma;
- unsigned long anon_mapping;
- rcu_read_lock();
- anon_mapping = (unsigned long)READ_ONCE(page->mapping);
- if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
- goto out;
- if (!page_mapped(page))
- goto out;
- anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
- root_anon_vma = READ_ONCE(anon_vma->root);
- if (down_read_trylock(&root_anon_vma->rwsem)) {
-
- if (!page_mapped(page)) {
- up_read(&root_anon_vma->rwsem);
- anon_vma = NULL;
- }
- goto out;
- }
-
- if (!atomic_inc_not_zero(&anon_vma->refcount)) {
- anon_vma = NULL;
- goto out;
- }
- if (!page_mapped(page)) {
- rcu_read_unlock();
- put_anon_vma(anon_vma);
- return NULL;
- }
-
- rcu_read_unlock();
- anon_vma_lock_read(anon_vma);
- if (atomic_dec_and_test(&anon_vma->refcount)) {
-
- anon_vma_unlock_read(anon_vma);
- __put_anon_vma(anon_vma);
- anon_vma = NULL;
- }
- return anon_vma;
- out:
- rcu_read_unlock();
- return anon_vma;
- }
- void page_unlock_anon_vma_read(struct anon_vma *anon_vma)
- {
- anon_vma_unlock_read(anon_vma);
- }
- #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
- void try_to_unmap_flush(void)
- {
- struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc;
- int cpu;
- if (!tlb_ubc->flush_required)
- return;
- cpu = get_cpu();
- if (cpumask_test_cpu(cpu, &tlb_ubc->cpumask)) {
- count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
- local_flush_tlb();
- trace_tlb_flush(TLB_LOCAL_SHOOTDOWN, TLB_FLUSH_ALL);
- }
- if (cpumask_any_but(&tlb_ubc->cpumask, cpu) < nr_cpu_ids)
- flush_tlb_others(&tlb_ubc->cpumask, NULL, 0, TLB_FLUSH_ALL);
- cpumask_clear(&tlb_ubc->cpumask);
- tlb_ubc->flush_required = false;
- tlb_ubc->writable = false;
- put_cpu();
- }
- void try_to_unmap_flush_dirty(void)
- {
- struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc;
- if (tlb_ubc->writable)
- try_to_unmap_flush();
- }
- static void set_tlb_ubc_flush_pending(struct mm_struct *mm,
- struct page *page, bool writable)
- {
- struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc;
- cpumask_or(&tlb_ubc->cpumask, &tlb_ubc->cpumask, mm_cpumask(mm));
- tlb_ubc->flush_required = true;
-
- barrier();
- mm->tlb_flush_batched = true;
-
- if (writable)
- tlb_ubc->writable = true;
- }
- static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags)
- {
- bool should_defer = false;
- if (!(flags & TTU_BATCH_FLUSH))
- return false;
-
- if (cpumask_any_but(mm_cpumask(mm), get_cpu()) < nr_cpu_ids)
- should_defer = true;
- put_cpu();
- return should_defer;
- }
- void flush_tlb_batched_pending(struct mm_struct *mm)
- {
- if (mm->tlb_flush_batched) {
- flush_tlb_mm(mm);
-
- barrier();
- mm->tlb_flush_batched = false;
- }
- }
- #else
- static void set_tlb_ubc_flush_pending(struct mm_struct *mm,
- struct page *page, bool writable)
- {
- }
- static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags)
- {
- return false;
- }
- #endif
- unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma)
- {
- unsigned long address;
- if (PageAnon(page)) {
- struct anon_vma *page__anon_vma = page_anon_vma(page);
-
- if (!vma->anon_vma || !page__anon_vma ||
- vma->anon_vma->root != page__anon_vma->root)
- return -EFAULT;
- } else if (page->mapping) {
- if (!vma->vm_file || vma->vm_file->f_mapping != page->mapping)
- return -EFAULT;
- } else
- return -EFAULT;
- address = __vma_address(page, vma);
- if (unlikely(address < vma->vm_start || address >= vma->vm_end))
- return -EFAULT;
- return address;
- }
- pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address)
- {
- pgd_t *pgd;
- pud_t *pud;
- pmd_t *pmd = NULL;
- pmd_t pmde;
- pgd = pgd_offset(mm, address);
- if (!pgd_present(*pgd))
- goto out;
- pud = pud_offset(pgd, address);
- if (!pud_present(*pud))
- goto out;
- pmd = pmd_offset(pud, address);
-
- pmde = *pmd;
- barrier();
- if (!pmd_present(pmde) || pmd_trans_huge(pmde))
- pmd = NULL;
- out:
- return pmd;
- }
- pte_t *__page_check_address(struct page *page, struct mm_struct *mm,
- unsigned long address, spinlock_t **ptlp, int sync)
- {
- pmd_t *pmd;
- pte_t *pte;
- spinlock_t *ptl;
- if (unlikely(PageHuge(page))) {
-
- pte = huge_pte_offset(mm, address);
- if (!pte)
- return NULL;
- ptl = huge_pte_lockptr(page_hstate(page), mm, pte);
- goto check;
- }
- pmd = mm_find_pmd(mm, address);
- if (!pmd)
- return NULL;
- pte = pte_offset_map(pmd, address);
-
- if (!sync && !pte_present(*pte)) {
- pte_unmap(pte);
- return NULL;
- }
- ptl = pte_lockptr(mm, pmd);
- check:
- spin_lock(ptl);
- if (pte_present(*pte) && page_to_pfn(page) == pte_pfn(*pte)) {
- *ptlp = ptl;
- return pte;
- }
- pte_unmap_unlock(pte, ptl);
- return NULL;
- }
- int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma)
- {
- unsigned long address;
- pte_t *pte;
- spinlock_t *ptl;
- address = __vma_address(page, vma);
- if (unlikely(address < vma->vm_start || address >= vma->vm_end))
- return 0;
- pte = page_check_address(page, vma->vm_mm, address, &ptl, 1);
- if (!pte)
- return 0;
- pte_unmap_unlock(pte, ptl);
- return 1;
- }
- #ifdef CONFIG_TRANSPARENT_HUGEPAGE
- bool page_check_address_transhuge(struct page *page, struct mm_struct *mm,
- unsigned long address, pmd_t **pmdp,
- pte_t **ptep, spinlock_t **ptlp)
- {
- pgd_t *pgd;
- pud_t *pud;
- pmd_t *pmd;
- pte_t *pte;
- spinlock_t *ptl;
- if (unlikely(PageHuge(page))) {
-
- pte = huge_pte_offset(mm, address);
- if (!pte)
- return false;
- ptl = huge_pte_lockptr(page_hstate(page), mm, pte);
- pmd = NULL;
- goto check_pte;
- }
- pgd = pgd_offset(mm, address);
- if (!pgd_present(*pgd))
- return false;
- pud = pud_offset(pgd, address);
- if (!pud_present(*pud))
- return false;
- pmd = pmd_offset(pud, address);
- if (pmd_trans_huge(*pmd)) {
- ptl = pmd_lock(mm, pmd);
- if (!pmd_present(*pmd))
- goto unlock_pmd;
- if (unlikely(!pmd_trans_huge(*pmd))) {
- spin_unlock(ptl);
- goto map_pte;
- }
- if (pmd_page(*pmd) != page)
- goto unlock_pmd;
- pte = NULL;
- goto found;
- unlock_pmd:
- spin_unlock(ptl);
- return false;
- } else {
- pmd_t pmde = *pmd;
- barrier();
- if (!pmd_present(pmde) || pmd_trans_huge(pmde))
- return false;
- }
- map_pte:
- pte = pte_offset_map(pmd, address);
- if (!pte_present(*pte)) {
- pte_unmap(pte);
- return false;
- }
- ptl = pte_lockptr(mm, pmd);
- check_pte:
- spin_lock(ptl);
- if (!pte_present(*pte)) {
- pte_unmap_unlock(pte, ptl);
- return false;
- }
-
- if (pte_pfn(*pte) - page_to_pfn(page) >= hpage_nr_pages(page)) {
- pte_unmap_unlock(pte, ptl);
- return false;
- }
- found:
- *ptep = pte;
- *pmdp = pmd;
- *ptlp = ptl;
- return true;
- }
- #endif
- struct page_referenced_arg {
- int mapcount;
- int referenced;
- unsigned long vm_flags;
- struct mem_cgroup *memcg;
- };
- static int page_referenced_one(struct page *page, struct vm_area_struct *vma,
- unsigned long address, void *arg)
- {
- struct mm_struct *mm = vma->vm_mm;
- struct page_referenced_arg *pra = arg;
- pmd_t *pmd;
- pte_t *pte;
- spinlock_t *ptl;
- int referenced = 0;
- if (!page_check_address_transhuge(page, mm, address, &pmd, &pte, &ptl))
- return SWAP_AGAIN;
- if (vma->vm_flags & VM_LOCKED) {
- if (pte)
- pte_unmap(pte);
- spin_unlock(ptl);
- pra->vm_flags |= VM_LOCKED;
- return SWAP_FAIL;
- }
- if (pte) {
- if (ptep_clear_flush_young_notify(vma, address, pte)) {
-
- if (likely(!(vma->vm_flags & VM_SEQ_READ)))
- referenced++;
- }
- pte_unmap(pte);
- } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
- if (pmdp_clear_flush_young_notify(vma, address, pmd))
- referenced++;
- } else {
-
- WARN_ON_ONCE(1);
- }
- spin_unlock(ptl);
- if (referenced)
- clear_page_idle(page);
- if (test_and_clear_page_young(page))
- referenced++;
- if (referenced) {
- pra->referenced++;
- pra->vm_flags |= vma->vm_flags;
- }
- pra->mapcount--;
- if (!pra->mapcount)
- return SWAP_SUCCESS;
- return SWAP_AGAIN;
- }
- static bool invalid_page_referenced_vma(struct vm_area_struct *vma, void *arg)
- {
- struct page_referenced_arg *pra = arg;
- struct mem_cgroup *memcg = pra->memcg;
- if (!mm_match_cgroup(vma->vm_mm, memcg))
- return true;
- return false;
- }
- int page_referenced(struct page *page,
- int is_locked,
- struct mem_cgroup *memcg,
- unsigned long *vm_flags)
- {
- int ret;
- int we_locked = 0;
- struct page_referenced_arg pra = {
- .mapcount = total_mapcount(page),
- .memcg = memcg,
- };
- struct rmap_walk_control rwc = {
- .rmap_one = page_referenced_one,
- .arg = (void *)&pra,
- .anon_lock = page_lock_anon_vma_read,
- };
- *vm_flags = 0;
- if (!page_mapped(page))
- return 0;
- if (!page_rmapping(page))
- return 0;
- if (!is_locked && (!PageAnon(page) || PageKsm(page))) {
- we_locked = trylock_page(page);
- if (!we_locked)
- return 1;
- }
-
- if (memcg) {
- rwc.invalid_vma = invalid_page_referenced_vma;
- }
- ret = rmap_walk(page, &rwc);
- *vm_flags = pra.vm_flags;
- if (we_locked)
- unlock_page(page);
- return pra.referenced;
- }
- static int page_mkclean_one(struct page *page, struct vm_area_struct *vma,
- unsigned long address, void *arg)
- {
- struct mm_struct *mm = vma->vm_mm;
- pte_t *pte;
- spinlock_t *ptl;
- int ret = 0;
- int *cleaned = arg;
- pte = page_check_address(page, mm, address, &ptl, 1);
- if (!pte)
- goto out;
- if (pte_dirty(*pte) || pte_write(*pte)) {
- pte_t entry;
- flush_cache_page(vma, address, pte_pfn(*pte));
- entry = ptep_clear_flush(vma, address, pte);
- entry = pte_wrprotect(entry);
- entry = pte_mkclean(entry);
- set_pte_at(mm, address, pte, entry);
- ret = 1;
- }
- pte_unmap_unlock(pte, ptl);
- if (ret) {
- mmu_notifier_invalidate_page(mm, address);
- (*cleaned)++;
- }
- out:
- return SWAP_AGAIN;
- }
- static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg)
- {
- if (vma->vm_flags & VM_SHARED)
- return false;
- return true;
- }
- int page_mkclean(struct page *page)
- {
- int cleaned = 0;
- struct address_space *mapping;
- struct rmap_walk_control rwc = {
- .arg = (void *)&cleaned,
- .rmap_one = page_mkclean_one,
- .invalid_vma = invalid_mkclean_vma,
- };
- BUG_ON(!PageLocked(page));
- if (!page_mapped(page))
- return 0;
- mapping = page_mapping(page);
- if (!mapping)
- return 0;
- rmap_walk(page, &rwc);
- return cleaned;
- }
- EXPORT_SYMBOL_GPL(page_mkclean);
- void page_move_anon_rmap(struct page *page, struct vm_area_struct *vma)
- {
- struct anon_vma *anon_vma = vma->anon_vma;
- page = compound_head(page);
- VM_BUG_ON_PAGE(!PageLocked(page), page);
- VM_BUG_ON_VMA(!anon_vma, vma);
- anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
-
- WRITE_ONCE(page->mapping, (struct address_space *) anon_vma);
- }
- static void __page_set_anon_rmap(struct page *page,
- struct vm_area_struct *vma, unsigned long address, int exclusive)
- {
- struct anon_vma *anon_vma = vma->anon_vma;
- BUG_ON(!anon_vma);
- if (PageAnon(page))
- return;
-
- if (!exclusive)
- anon_vma = anon_vma->root;
- anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
- page->mapping = (struct address_space *) anon_vma;
- page->index = linear_page_index(vma, address);
- }
- static void __page_check_anon_rmap(struct page *page,
- struct vm_area_struct *vma, unsigned long address)
- {
- #ifdef CONFIG_DEBUG_VM
-
- BUG_ON(page_anon_vma(page)->root != vma->anon_vma->root);
- BUG_ON(page_to_pgoff(page) != linear_page_index(vma, address));
- #endif
- }
- void page_add_anon_rmap(struct page *page,
- struct vm_area_struct *vma, unsigned long address, bool compound)
- {
- do_page_add_anon_rmap(page, vma, address, compound ? RMAP_COMPOUND : 0);
- }
- void do_page_add_anon_rmap(struct page *page,
- struct vm_area_struct *vma, unsigned long address, int flags)
- {
- bool compound = flags & RMAP_COMPOUND;
- bool first;
- if (compound) {
- atomic_t *mapcount;
- VM_BUG_ON_PAGE(!PageLocked(page), page);
- VM_BUG_ON_PAGE(!PageTransHuge(page), page);
- mapcount = compound_mapcount_ptr(page);
- first = atomic_inc_and_test(mapcount);
- } else {
- first = atomic_inc_and_test(&page->_mapcount);
- }
- if (first) {
- int nr = compound ? hpage_nr_pages(page) : 1;
-
- if (compound)
- __inc_node_page_state(page, NR_ANON_THPS);
- __mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, nr);
- }
- if (unlikely(PageKsm(page)))
- return;
- VM_BUG_ON_PAGE(!PageLocked(page), page);
-
- if (first)
- __page_set_anon_rmap(page, vma, address,
- flags & RMAP_EXCLUSIVE);
- else
- __page_check_anon_rmap(page, vma, address);
- }
- void page_add_new_anon_rmap(struct page *page,
- struct vm_area_struct *vma, unsigned long address, bool compound)
- {
- int nr = compound ? hpage_nr_pages(page) : 1;
- VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
- __SetPageSwapBacked(page);
- if (compound) {
- VM_BUG_ON_PAGE(!PageTransHuge(page), page);
-
- atomic_set(compound_mapcount_ptr(page), 0);
- __inc_node_page_state(page, NR_ANON_THPS);
- } else {
-
- VM_BUG_ON_PAGE(PageTransCompound(page), page);
-
- atomic_set(&page->_mapcount, 0);
- }
- __mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, nr);
- __page_set_anon_rmap(page, vma, address, 1);
- }
- void page_add_file_rmap(struct page *page, bool compound)
- {
- int i, nr = 1;
- VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page);
- lock_page_memcg(page);
- if (compound && PageTransHuge(page)) {
- for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) {
- if (atomic_inc_and_test(&page[i]._mapcount))
- nr++;
- }
- if (!atomic_inc_and_test(compound_mapcount_ptr(page)))
- goto out;
- VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
- __inc_node_page_state(page, NR_SHMEM_PMDMAPPED);
- } else {
- if (PageTransCompound(page) && page_mapping(page)) {
- VM_WARN_ON_ONCE(!PageLocked(page));
- SetPageDoubleMap(compound_head(page));
- if (PageMlocked(page))
- clear_page_mlock(compound_head(page));
- }
- if (!atomic_inc_and_test(&page->_mapcount))
- goto out;
- }
- __mod_node_page_state(page_pgdat(page), NR_FILE_MAPPED, nr);
- mem_cgroup_update_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED, nr);
- out:
- unlock_page_memcg(page);
- }
- static void page_remove_file_rmap(struct page *page, bool compound)
- {
- int i, nr = 1;
- VM_BUG_ON_PAGE(compound && !PageHead(page), page);
- lock_page_memcg(page);
-
- if (unlikely(PageHuge(page))) {
-
- atomic_dec(compound_mapcount_ptr(page));
- goto out;
- }
-
- if (compound && PageTransHuge(page)) {
- for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) {
- if (atomic_add_negative(-1, &page[i]._mapcount))
- nr++;
- }
- if (!atomic_add_negative(-1, compound_mapcount_ptr(page)))
- goto out;
- VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
- __dec_node_page_state(page, NR_SHMEM_PMDMAPPED);
- } else {
- if (!atomic_add_negative(-1, &page->_mapcount))
- goto out;
- }
-
- __mod_node_page_state(page_pgdat(page), NR_FILE_MAPPED, -nr);
- mem_cgroup_update_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED, -nr);
- if (unlikely(PageMlocked(page)))
- clear_page_mlock(page);
- out:
- unlock_page_memcg(page);
- }
- static void page_remove_anon_compound_rmap(struct page *page)
- {
- int i, nr;
- if (!atomic_add_negative(-1, compound_mapcount_ptr(page)))
- return;
-
- if (unlikely(PageHuge(page)))
- return;
- if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
- return;
- __dec_node_page_state(page, NR_ANON_THPS);
- if (TestClearPageDoubleMap(page)) {
-
- for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) {
- if (atomic_add_negative(-1, &page[i]._mapcount))
- nr++;
- }
- } else {
- nr = HPAGE_PMD_NR;
- }
- if (unlikely(PageMlocked(page)))
- clear_page_mlock(page);
- if (nr) {
- __mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, -nr);
- deferred_split_huge_page(page);
- }
- }
- void page_remove_rmap(struct page *page, bool compound)
- {
- if (!PageAnon(page))
- return page_remove_file_rmap(page, compound);
- if (compound)
- return page_remove_anon_compound_rmap(page);
-
- if (!atomic_add_negative(-1, &page->_mapcount))
- return;
-
- __dec_node_page_state(page, NR_ANON_MAPPED);
- if (unlikely(PageMlocked(page)))
- clear_page_mlock(page);
- if (PageTransCompound(page))
- deferred_split_huge_page(compound_head(page));
-
- }
- struct rmap_private {
- enum ttu_flags flags;
- int lazyfreed;
- };
- static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
- unsigned long address, void *arg)
- {
- struct mm_struct *mm = vma->vm_mm;
- pte_t *pte;
- pte_t pteval;
- spinlock_t *ptl;
- int ret = SWAP_AGAIN;
- struct rmap_private *rp = arg;
- enum ttu_flags flags = rp->flags;
-
- if ((flags & TTU_MUNLOCK) && !(vma->vm_flags & VM_LOCKED))
- goto out;
- if (flags & TTU_SPLIT_HUGE_PMD) {
- split_huge_pmd_address(vma, address,
- flags & TTU_MIGRATION, page);
-
- if (page_mapcount(page) == 0)
- goto out;
- }
- pte = page_check_address(page, mm, address, &ptl,
- PageTransCompound(page));
- if (!pte)
- goto out;
-
- if (!(flags & TTU_IGNORE_MLOCK)) {
- if (vma->vm_flags & VM_LOCKED) {
-
- if (!PageTransCompound(page)) {
-
- mlock_vma_page(page);
- }
- ret = SWAP_MLOCK;
- goto out_unmap;
- }
- if (flags & TTU_MUNLOCK)
- goto out_unmap;
- }
- if (!(flags & TTU_IGNORE_ACCESS)) {
- if (ptep_clear_flush_young_notify(vma, address, pte)) {
- ret = SWAP_FAIL;
- goto out_unmap;
- }
- }
-
- flush_cache_page(vma, address, page_to_pfn(page));
- if (should_defer_flush(mm, flags)) {
-
- pteval = ptep_get_and_clear(mm, address, pte);
- set_tlb_ubc_flush_pending(mm, page, pte_dirty(pteval));
- } else {
- pteval = ptep_clear_flush(vma, address, pte);
- }
-
- if (pte_dirty(pteval))
- set_page_dirty(page);
-
- update_hiwater_rss(mm);
- if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) {
- if (PageHuge(page)) {
- hugetlb_count_sub(1 << compound_order(page), mm);
- } else {
- dec_mm_counter(mm, mm_counter(page));
- }
- set_pte_at(mm, address, pte,
- swp_entry_to_pte(make_hwpoison_entry(page)));
- } else if (pte_unused(pteval)) {
-
- dec_mm_counter(mm, mm_counter(page));
- } else if (IS_ENABLED(CONFIG_MIGRATION) && (flags & TTU_MIGRATION)) {
- swp_entry_t entry;
- pte_t swp_pte;
-
- entry = make_migration_entry(page, pte_write(pteval));
- swp_pte = swp_entry_to_pte(entry);
- if (pte_soft_dirty(pteval))
- swp_pte = pte_swp_mksoft_dirty(swp_pte);
- set_pte_at(mm, address, pte, swp_pte);
- } else if (PageAnon(page)) {
- swp_entry_t entry = { .val = page_private(page) };
- pte_t swp_pte;
-
- VM_BUG_ON_PAGE(!PageSwapCache(page), page);
- if (!PageDirty(page) && (flags & TTU_LZFREE)) {
-
- dec_mm_counter(mm, MM_ANONPAGES);
- rp->lazyfreed++;
- goto discard;
- }
- if (swap_duplicate(entry) < 0) {
- set_pte_at(mm, address, pte, pteval);
- ret = SWAP_FAIL;
- goto out_unmap;
- }
- if (list_empty(&mm->mmlist)) {
- spin_lock(&mmlist_lock);
- if (list_empty(&mm->mmlist))
- list_add(&mm->mmlist, &init_mm.mmlist);
- spin_unlock(&mmlist_lock);
- }
- dec_mm_counter(mm, MM_ANONPAGES);
- inc_mm_counter(mm, MM_SWAPENTS);
- swp_pte = swp_entry_to_pte(entry);
- if (pte_soft_dirty(pteval))
- swp_pte = pte_swp_mksoft_dirty(swp_pte);
- set_pte_at(mm, address, pte, swp_pte);
- } else
- dec_mm_counter(mm, mm_counter_file(page));
- discard:
- page_remove_rmap(page, PageHuge(page));
- put_page(page);
- out_unmap:
- pte_unmap_unlock(pte, ptl);
- if (ret != SWAP_FAIL && ret != SWAP_MLOCK && !(flags & TTU_MUNLOCK))
- mmu_notifier_invalidate_page(mm, address);
- out:
- return ret;
- }
- bool is_vma_temporary_stack(struct vm_area_struct *vma)
- {
- int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP);
- if (!maybe_stack)
- return false;
- if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) ==
- VM_STACK_INCOMPLETE_SETUP)
- return true;
- return false;
- }
- static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg)
- {
- return is_vma_temporary_stack(vma);
- }
- static int page_mapcount_is_zero(struct page *page)
- {
- return !page_mapcount(page);
- }
- int try_to_unmap(struct page *page, enum ttu_flags flags)
- {
- int ret;
- struct rmap_private rp = {
- .flags = flags,
- .lazyfreed = 0,
- };
- struct rmap_walk_control rwc = {
- .rmap_one = try_to_unmap_one,
- .arg = &rp,
- .done = page_mapcount_is_zero,
- .anon_lock = page_lock_anon_vma_read,
- };
-
- if ((flags & TTU_MIGRATION) && !PageKsm(page) && PageAnon(page))
- rwc.invalid_vma = invalid_migration_vma;
- if (flags & TTU_RMAP_LOCKED)
- ret = rmap_walk_locked(page, &rwc);
- else
- ret = rmap_walk(page, &rwc);
- if (ret != SWAP_MLOCK && !page_mapcount(page)) {
- ret = SWAP_SUCCESS;
- if (rp.lazyfreed && !PageDirty(page))
- ret = SWAP_LZFREE;
- }
- return ret;
- }
- static int page_not_mapped(struct page *page)
- {
- return !page_mapped(page);
- };
- int try_to_munlock(struct page *page)
- {
- int ret;
- struct rmap_private rp = {
- .flags = TTU_MUNLOCK,
- .lazyfreed = 0,
- };
- struct rmap_walk_control rwc = {
- .rmap_one = try_to_unmap_one,
- .arg = &rp,
- .done = page_not_mapped,
- .anon_lock = page_lock_anon_vma_read,
- };
- VM_BUG_ON_PAGE(!PageLocked(page) || PageLRU(page), page);
- ret = rmap_walk(page, &rwc);
- return ret;
- }
- void __put_anon_vma(struct anon_vma *anon_vma)
- {
- struct anon_vma *root = anon_vma->root;
- anon_vma_free(anon_vma);
- if (root != anon_vma && atomic_dec_and_test(&root->refcount))
- anon_vma_free(root);
- }
- static struct anon_vma *rmap_walk_anon_lock(struct page *page,
- struct rmap_walk_control *rwc)
- {
- struct anon_vma *anon_vma;
- if (rwc->anon_lock)
- return rwc->anon_lock(page);
-
- anon_vma = page_anon_vma(page);
- if (!anon_vma)
- return NULL;
- anon_vma_lock_read(anon_vma);
- return anon_vma;
- }
- static int rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc,
- bool locked)
- {
- struct anon_vma *anon_vma;
- pgoff_t pgoff;
- struct anon_vma_chain *avc;
- int ret = SWAP_AGAIN;
- if (locked) {
- anon_vma = page_anon_vma(page);
-
- VM_BUG_ON_PAGE(!anon_vma, page);
- } else {
- anon_vma = rmap_walk_anon_lock(page, rwc);
- }
- if (!anon_vma)
- return ret;
- pgoff = page_to_pgoff(page);
- anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, pgoff) {
- struct vm_area_struct *vma = avc->vma;
- unsigned long address = vma_address(page, vma);
- cond_resched();
- if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
- continue;
- ret = rwc->rmap_one(page, vma, address, rwc->arg);
- if (ret != SWAP_AGAIN)
- break;
- if (rwc->done && rwc->done(page))
- break;
- }
- if (!locked)
- anon_vma_unlock_read(anon_vma);
- return ret;
- }
- static int rmap_walk_file(struct page *page, struct rmap_walk_control *rwc,
- bool locked)
- {
- struct address_space *mapping = page_mapping(page);
- pgoff_t pgoff;
- struct vm_area_struct *vma;
- int ret = SWAP_AGAIN;
-
- VM_BUG_ON_PAGE(!PageLocked(page), page);
- if (!mapping)
- return ret;
- pgoff = page_to_pgoff(page);
- if (!locked)
- i_mmap_lock_read(mapping);
- vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
- unsigned long address = vma_address(page, vma);
- cond_resched();
- if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
- continue;
- ret = rwc->rmap_one(page, vma, address, rwc->arg);
- if (ret != SWAP_AGAIN)
- goto done;
- if (rwc->done && rwc->done(page))
- goto done;
- }
- done:
- if (!locked)
- i_mmap_unlock_read(mapping);
- return ret;
- }
- int rmap_walk(struct page *page, struct rmap_walk_control *rwc)
- {
- if (unlikely(PageKsm(page)))
- return rmap_walk_ksm(page, rwc);
- else if (PageAnon(page))
- return rmap_walk_anon(page, rwc, false);
- else
- return rmap_walk_file(page, rwc, false);
- }
- int rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc)
- {
-
- VM_BUG_ON_PAGE(PageKsm(page), page);
- if (PageAnon(page))
- return rmap_walk_anon(page, rwc, true);
- else
- return rmap_walk_file(page, rwc, true);
- }
- #ifdef CONFIG_HUGETLB_PAGE
- static void __hugepage_set_anon_rmap(struct page *page,
- struct vm_area_struct *vma, unsigned long address, int exclusive)
- {
- struct anon_vma *anon_vma = vma->anon_vma;
- BUG_ON(!anon_vma);
- if (PageAnon(page))
- return;
- if (!exclusive)
- anon_vma = anon_vma->root;
- anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
- page->mapping = (struct address_space *) anon_vma;
- page->index = linear_page_index(vma, address);
- }
- void hugepage_add_anon_rmap(struct page *page,
- struct vm_area_struct *vma, unsigned long address)
- {
- struct anon_vma *anon_vma = vma->anon_vma;
- int first;
- BUG_ON(!PageLocked(page));
- BUG_ON(!anon_vma);
-
- first = atomic_inc_and_test(compound_mapcount_ptr(page));
- if (first)
- __hugepage_set_anon_rmap(page, vma, address, 0);
- }
- void hugepage_add_new_anon_rmap(struct page *page,
- struct vm_area_struct *vma, unsigned long address)
- {
- BUG_ON(address < vma->vm_start || address >= vma->vm_end);
- atomic_set(compound_mapcount_ptr(page), 0);
- __hugepage_set_anon_rmap(page, vma, address, 1);
- }
- #endif
|