|
1 #ifndef _LINUX_MMU_NOTIFIER_H |
|
2 #define _LINUX_MMU_NOTIFIER_H |
|
3 |
|
4 #include <linux/list.h> |
|
5 #include <linux/spinlock.h> |
|
6 #include <linux/mm_types.h> |
|
7 |
|
8 struct mmu_notifier; |
|
9 struct mmu_notifier_ops; |
|
10 |
|
11 #ifdef CONFIG_MMU_NOTIFIER |
|
12 |
|
13 /* |
|
14 * The mmu notifier_mm structure is allocated and installed in |
|
15 * mm->mmu_notifier_mm inside the mm_take_all_locks() protected |
|
16 * critical section and it's released only when mm_count reaches zero |
|
17 * in mmdrop(). |
|
18 */ |
|
19 struct mmu_notifier_mm { |
|
20 /* all mmu notifiers registerd in this mm are queued in this list */ |
|
21 struct hlist_head list; |
|
22 /* to serialize the list modifications and hlist_unhashed */ |
|
23 spinlock_t lock; |
|
24 }; |
|
25 |
|
26 struct mmu_notifier_ops { |
|
27 /* |
|
28 * Called either by mmu_notifier_unregister or when the mm is |
|
29 * being destroyed by exit_mmap, always before all pages are |
|
30 * freed. This can run concurrently with other mmu notifier |
|
31 * methods (the ones invoked outside the mm context) and it |
|
32 * should tear down all secondary mmu mappings and freeze the |
|
33 * secondary mmu. If this method isn't implemented you've to |
|
34 * be sure that nothing could possibly write to the pages |
|
35 * through the secondary mmu by the time the last thread with |
|
36 * tsk->mm == mm exits. |
|
37 * |
|
38 * As side note: the pages freed after ->release returns could |
|
39 * be immediately reallocated by the gart at an alias physical |
|
40 * address with a different cache model, so if ->release isn't |
|
41 * implemented because all _software_ driven memory accesses |
|
42 * through the secondary mmu are terminated by the time the |
|
43 * last thread of this mm quits, you've also to be sure that |
|
44 * speculative _hardware_ operations can't allocate dirty |
|
45 * cachelines in the cpu that could not be snooped and made |
|
46 * coherent with the other read and write operations happening |
|
47 * through the gart alias address, so leading to memory |
|
48 * corruption. |
|
49 */ |
|
50 void (*release)(struct mmu_notifier *mn, |
|
51 struct mm_struct *mm); |
|
52 |
|
53 /* |
|
54 * clear_flush_young is called after the VM is |
|
55 * test-and-clearing the young/accessed bitflag in the |
|
56 * pte. This way the VM will provide proper aging to the |
|
57 * accesses to the page through the secondary MMUs and not |
|
58 * only to the ones through the Linux pte. |
|
59 */ |
|
60 int (*clear_flush_young)(struct mmu_notifier *mn, |
|
61 struct mm_struct *mm, |
|
62 unsigned long address); |
|
63 |
|
64 /* |
|
65 * Before this is invoked any secondary MMU is still ok to |
|
66 * read/write to the page previously pointed to by the Linux |
|
67 * pte because the page hasn't been freed yet and it won't be |
|
68 * freed until this returns. If required set_page_dirty has to |
|
69 * be called internally to this method. |
|
70 */ |
|
71 void (*invalidate_page)(struct mmu_notifier *mn, |
|
72 struct mm_struct *mm, |
|
73 unsigned long address); |
|
74 |
|
75 /* |
|
76 * invalidate_range_start() and invalidate_range_end() must be |
|
77 * paired and are called only when the mmap_sem and/or the |
|
78 * locks protecting the reverse maps are held. The subsystem |
|
79 * must guarantee that no additional references are taken to |
|
80 * the pages in the range established between the call to |
|
81 * invalidate_range_start() and the matching call to |
|
82 * invalidate_range_end(). |
|
83 * |
|
84 * Invalidation of multiple concurrent ranges may be |
|
85 * optionally permitted by the driver. Either way the |
|
86 * establishment of sptes is forbidden in the range passed to |
|
87 * invalidate_range_begin/end for the whole duration of the |
|
88 * invalidate_range_begin/end critical section. |
|
89 * |
|
90 * invalidate_range_start() is called when all pages in the |
|
91 * range are still mapped and have at least a refcount of one. |
|
92 * |
|
93 * invalidate_range_end() is called when all pages in the |
|
94 * range have been unmapped and the pages have been freed by |
|
95 * the VM. |
|
96 * |
|
97 * The VM will remove the page table entries and potentially |
|
98 * the page between invalidate_range_start() and |
|
99 * invalidate_range_end(). If the page must not be freed |
|
100 * because of pending I/O or other circumstances then the |
|
101 * invalidate_range_start() callback (or the initial mapping |
|
102 * by the driver) must make sure that the refcount is kept |
|
103 * elevated. |
|
104 * |
|
105 * If the driver increases the refcount when the pages are |
|
106 * initially mapped into an address space then either |
|
107 * invalidate_range_start() or invalidate_range_end() may |
|
108 * decrease the refcount. If the refcount is decreased on |
|
109 * invalidate_range_start() then the VM can free pages as page |
|
110 * table entries are removed. If the refcount is only |
|
111 * droppped on invalidate_range_end() then the driver itself |
|
112 * will drop the last refcount but it must take care to flush |
|
113 * any secondary tlb before doing the final free on the |
|
114 * page. Pages will no longer be referenced by the linux |
|
115 * address space but may still be referenced by sptes until |
|
116 * the last refcount is dropped. |
|
117 */ |
|
118 void (*invalidate_range_start)(struct mmu_notifier *mn, |
|
119 struct mm_struct *mm, |
|
120 unsigned long start, unsigned long end); |
|
121 void (*invalidate_range_end)(struct mmu_notifier *mn, |
|
122 struct mm_struct *mm, |
|
123 unsigned long start, unsigned long end); |
|
124 }; |
|
125 |
|
126 /* |
|
127 * The notifier chains are protected by mmap_sem and/or the reverse map |
|
128 * semaphores. Notifier chains are only changed when all reverse maps and |
|
129 * the mmap_sem locks are taken. |
|
130 * |
|
131 * Therefore notifier chains can only be traversed when either |
|
132 * |
|
133 * 1. mmap_sem is held. |
|
134 * 2. One of the reverse map locks is held (i_mmap_lock or anon_vma->lock). |
|
135 * 3. No other concurrent thread can access the list (release) |
|
136 */ |
|
137 struct mmu_notifier { |
|
138 struct hlist_node hlist; |
|
139 const struct mmu_notifier_ops *ops; |
|
140 }; |
|
141 |
|
142 static inline int mm_has_notifiers(struct mm_struct *mm) |
|
143 { |
|
144 return unlikely(mm->mmu_notifier_mm); |
|
145 } |
|
146 |
|
147 extern int mmu_notifier_register(struct mmu_notifier *mn, |
|
148 struct mm_struct *mm); |
|
149 extern int __mmu_notifier_register(struct mmu_notifier *mn, |
|
150 struct mm_struct *mm); |
|
151 extern void mmu_notifier_unregister(struct mmu_notifier *mn, |
|
152 struct mm_struct *mm); |
|
153 extern void __mmu_notifier_mm_destroy(struct mm_struct *mm); |
|
154 extern void __mmu_notifier_release(struct mm_struct *mm); |
|
155 extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm, |
|
156 unsigned long address); |
|
157 extern void __mmu_notifier_invalidate_page(struct mm_struct *mm, |
|
158 unsigned long address); |
|
159 extern void __mmu_notifier_invalidate_range_start(struct mm_struct *mm, |
|
160 unsigned long start, unsigned long end); |
|
161 extern void __mmu_notifier_invalidate_range_end(struct mm_struct *mm, |
|
162 unsigned long start, unsigned long end); |
|
163 |
|
164 static inline void mmu_notifier_release(struct mm_struct *mm) |
|
165 { |
|
166 if (mm_has_notifiers(mm)) |
|
167 __mmu_notifier_release(mm); |
|
168 } |
|
169 |
|
170 static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm, |
|
171 unsigned long address) |
|
172 { |
|
173 if (mm_has_notifiers(mm)) |
|
174 return __mmu_notifier_clear_flush_young(mm, address); |
|
175 return 0; |
|
176 } |
|
177 |
|
178 static inline void mmu_notifier_invalidate_page(struct mm_struct *mm, |
|
179 unsigned long address) |
|
180 { |
|
181 if (mm_has_notifiers(mm)) |
|
182 __mmu_notifier_invalidate_page(mm, address); |
|
183 } |
|
184 |
|
185 static inline void mmu_notifier_invalidate_range_start(struct mm_struct *mm, |
|
186 unsigned long start, unsigned long end) |
|
187 { |
|
188 if (mm_has_notifiers(mm)) |
|
189 __mmu_notifier_invalidate_range_start(mm, start, end); |
|
190 } |
|
191 |
|
192 static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm, |
|
193 unsigned long start, unsigned long end) |
|
194 { |
|
195 if (mm_has_notifiers(mm)) |
|
196 __mmu_notifier_invalidate_range_end(mm, start, end); |
|
197 } |
|
198 |
|
199 static inline void mmu_notifier_mm_init(struct mm_struct *mm) |
|
200 { |
|
201 mm->mmu_notifier_mm = NULL; |
|
202 } |
|
203 |
|
204 static inline void mmu_notifier_mm_destroy(struct mm_struct *mm) |
|
205 { |
|
206 if (mm_has_notifiers(mm)) |
|
207 __mmu_notifier_mm_destroy(mm); |
|
208 } |
|
209 |
|
210 /* |
|
211 * These two macros will sometime replace ptep_clear_flush. |
|
212 * ptep_clear_flush is impleemnted as macro itself, so this also is |
|
213 * implemented as a macro until ptep_clear_flush will converted to an |
|
214 * inline function, to diminish the risk of compilation failure. The |
|
215 * invalidate_page method over time can be moved outside the PT lock |
|
216 * and these two macros can be later removed. |
|
217 */ |
|
218 #define ptep_clear_flush_notify(__vma, __address, __ptep) \ |
|
219 ({ \ |
|
220 pte_t __pte; \ |
|
221 struct vm_area_struct *___vma = __vma; \ |
|
222 unsigned long ___address = __address; \ |
|
223 __pte = ptep_clear_flush(___vma, ___address, __ptep); \ |
|
224 mmu_notifier_invalidate_page(___vma->vm_mm, ___address); \ |
|
225 __pte; \ |
|
226 }) |
|
227 |
|
228 #define ptep_clear_flush_young_notify(__vma, __address, __ptep) \ |
|
229 ({ \ |
|
230 int __young; \ |
|
231 struct vm_area_struct *___vma = __vma; \ |
|
232 unsigned long ___address = __address; \ |
|
233 __young = ptep_clear_flush_young(___vma, ___address, __ptep); \ |
|
234 __young |= mmu_notifier_clear_flush_young(___vma->vm_mm, \ |
|
235 ___address); \ |
|
236 __young; \ |
|
237 }) |
|
238 |
|
239 #else /* CONFIG_MMU_NOTIFIER */ |
|
240 |
|
241 static inline void mmu_notifier_release(struct mm_struct *mm) |
|
242 { |
|
243 } |
|
244 |
|
245 static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm, |
|
246 unsigned long address) |
|
247 { |
|
248 return 0; |
|
249 } |
|
250 |
|
251 static inline void mmu_notifier_invalidate_page(struct mm_struct *mm, |
|
252 unsigned long address) |
|
253 { |
|
254 } |
|
255 |
|
256 static inline void mmu_notifier_invalidate_range_start(struct mm_struct *mm, |
|
257 unsigned long start, unsigned long end) |
|
258 { |
|
259 } |
|
260 |
|
261 static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm, |
|
262 unsigned long start, unsigned long end) |
|
263 { |
|
264 } |
|
265 |
|
266 static inline void mmu_notifier_mm_init(struct mm_struct *mm) |
|
267 { |
|
268 } |
|
269 |
|
270 static inline void mmu_notifier_mm_destroy(struct mm_struct *mm) |
|
271 { |
|
272 } |
|
273 |
|
274 #define ptep_clear_flush_young_notify ptep_clear_flush_young |
|
275 #define ptep_clear_flush_notify ptep_clear_flush |
|
276 |
|
277 #endif /* CONFIG_MMU_NOTIFIER */ |
|
278 |
|
279 #endif /* _LINUX_MMU_NOTIFIER_H */ |