24 |
#endif |
#endif |
25 |
|
|
26 |
#include <assert.h> |
#include <assert.h> |
27 |
|
#include <string.h> |
28 |
#include <errno.h> |
#include <errno.h> |
29 |
#include <hurd/btree.h> |
#include <hurd/btree.h> |
30 |
#include <hurd/slab.h> |
#include <hurd/slab.h> |
31 |
|
|
32 |
#include <compiler.h> |
#include <compiler.h> |
33 |
|
|
34 |
|
#include "priv.h" |
35 |
#include "physmem.h" |
#include "physmem.h" |
36 |
|
#include "zalloc.h" |
37 |
|
|
38 |
|
static error_t |
39 |
|
frame_entry_constructor (void *hook, void *buffer) |
40 |
|
{ |
41 |
|
struct frame_entry *frame_entry = buffer; |
42 |
|
|
43 |
|
frame_entry->shared_next = frame_entry; |
44 |
|
frame_entry->shared_prevp = &frame_entry->shared_next; |
45 |
|
|
46 |
|
return 0; |
47 |
|
} |
48 |
|
|
49 |
static struct hurd_slab_space frame_entry_space |
static struct hurd_slab_space frame_entry_space |
50 |
= HURD_SLAB_SPACE_INITIALIZER (struct frame_entry, |
= HURD_SLAB_SPACE_INITIALIZER (struct frame_entry, |
51 |
NULL, NULL, |
NULL, NULL, |
52 |
NULL, NULL, |
frame_entry_constructor, NULL, |
53 |
NULL); |
NULL); |
54 |
|
|
55 |
static inline void |
void |
56 |
frame_entry_dump (struct container *cont) |
frame_entry_dump (struct frame_entry *fe) |
57 |
{ |
{ |
58 |
struct frame_entry *fe; |
printf ("frame_entry: %x:%x+%x@%x, frame: %x:%x+%x, ->cow: %d, ->refs: %d, shared list: ", |
59 |
|
fe, fe->region.start, fe->region.size, fe->frame_offset, |
60 |
|
fe->frame, l4_address (fe->frame->memory), |
61 |
|
l4_size (fe->frame->memory), |
62 |
|
fe->frame->cow, fe->frame->refs); |
63 |
|
|
64 |
|
int shares = 0; |
65 |
|
struct frame_entry *f = fe; |
66 |
|
do |
67 |
|
{ |
68 |
|
printf ("%x:%x+%x->", f, f->frame_offset, f->region.size); |
69 |
|
shares ++; |
70 |
|
assert (f->frame == fe->frame); |
71 |
|
f = f->shared_next; |
72 |
|
} |
73 |
|
while (f != fe); |
74 |
|
printf ("(=%d)\n", shares); |
75 |
|
|
76 |
for (fe = hurd_btree_frame_entry_first (&cont->frame_entries); fe; |
int entries = 0; |
77 |
fe = hurd_btree_frame_entry_next (fe)) |
for (f = fe->frame->frame_entries; f; f = f->next) |
78 |
printf ("%x+%x@%x ", fe->region.start, fe->region.size, |
entries ++; |
79 |
l4_address (fe->frame->memory)); |
assert (entries == fe->frame->refs); |
|
printf ("\n"); |
|
80 |
} |
} |
81 |
|
|
82 |
struct frame_entry * |
struct frame_entry * |
89 |
if (err) |
if (err) |
90 |
return 0; |
return 0; |
91 |
|
|
92 |
|
assert (frame_entry->shared_next == frame_entry); |
93 |
|
assert (frame_entry->shared_prevp == &frame_entry->shared_next); |
94 |
|
|
95 |
return frame_entry; |
return frame_entry; |
96 |
} |
} |
97 |
|
|
98 |
void |
void |
99 |
frame_entry_dealloc (struct frame_entry *frame_entry) |
frame_entry_free (struct frame_entry *frame_entry) |
100 |
{ |
{ |
101 |
|
assert (frame_entry->shared_next == frame_entry); |
102 |
|
#ifndef NDEBUG |
103 |
|
memset (frame_entry, 0xfe, sizeof (struct frame_entry)); |
104 |
|
frame_entry_constructor (0, frame_entry); |
105 |
|
#endif |
106 |
hurd_slab_dealloc (&frame_entry_space, frame_entry); |
hurd_slab_dealloc (&frame_entry_space, frame_entry); |
107 |
} |
} |
108 |
|
|
109 |
|
/* If SHARE is non-NULL, add FRAME_ENTRY (which is not attach to any |
110 |
|
share list) to SHARE's share list. Otherwise, remove FRAME_ENTRY |
111 |
|
from the share list to which it is currently attached. */ |
112 |
|
static void |
113 |
|
frame_entry_share_with (struct frame_entry *frame_entry, |
114 |
|
struct frame_entry *share) |
115 |
|
{ |
116 |
|
if (share) |
117 |
|
/* Add FRAME_ENTRY to SHARE's share list. */ |
118 |
|
{ |
119 |
|
/* FRAME_ENTRY shouldn't be on a shared list. */ |
120 |
|
assert (frame_entry->shared_next == frame_entry); |
121 |
|
|
122 |
|
frame_entry->shared_next = share; |
123 |
|
frame_entry->shared_prevp = share->shared_prevp; |
124 |
|
*frame_entry->shared_prevp = frame_entry; |
125 |
|
share->shared_prevp = &frame_entry->shared_next; |
126 |
|
} |
127 |
|
else |
128 |
|
/* Remove FRAME_ENTRY from any share list. */ |
129 |
|
{ |
130 |
|
*frame_entry->shared_prevp = frame_entry->shared_next; |
131 |
|
frame_entry->shared_next->shared_prevp = frame_entry->shared_prevp; |
132 |
|
|
133 |
|
frame_entry->shared_next = frame_entry; |
134 |
|
frame_entry->shared_prevp = &frame_entry->shared_next; |
135 |
|
} |
136 |
|
} |
137 |
|
|
138 |
error_t |
error_t |
139 |
frame_entry_new (struct container *cont, |
frame_entry_create (struct container *cont, |
140 |
struct frame_entry *frame_entry, |
struct frame_entry *frame_entry, |
141 |
uintptr_t start, size_t size) |
uintptr_t cont_addr, size_t size) |
142 |
{ |
{ |
143 |
error_t err; |
error_t err; |
144 |
|
|
145 |
|
assert (pthread_mutex_trylock (&cont->lock) == EBUSY); |
146 |
/* Size must be a power of 2. */ |
/* Size must be a power of 2. */ |
147 |
assert (size > 0 && (size & (size - 1)) == 0); |
assert (size > 0 && (size & (size - 1)) == 0); |
148 |
|
|
149 |
/* Initialize the frame_entry region. */ |
frame_entry->container = cont; |
150 |
frame_entry->region.start = start; |
frame_entry->region.start = cont_addr; |
151 |
frame_entry->region.size = size; |
frame_entry->region.size = size; |
152 |
frame_entry->frame_offset = 0; |
frame_entry->frame_offset = 0; |
153 |
|
|
155 |
if (! frame_entry->frame) |
if (! frame_entry->frame) |
156 |
return errno; |
return errno; |
157 |
|
|
158 |
err = frame_entry_attach (cont, frame_entry); |
err = container_attach (cont, frame_entry); |
159 |
if (EXPECT_FALSE (err)) |
if (EXPECT_FALSE (err)) |
160 |
{ |
{ |
161 |
debug ("Overlap: %x+%x\n", start, size); |
debug ("Overlap: %x+%x\n", cont_addr, size); |
162 |
frame_deref (frame_entry->frame); |
frame_deref (frame_entry->frame); |
163 |
return EEXIST; |
return EEXIST; |
164 |
} |
} |
165 |
|
|
166 |
frame_add_user (frame_entry->frame, frame_entry); |
frame_add_user (frame_entry->frame, frame_entry); |
167 |
|
|
168 |
|
frame_entry_share_with (frame_entry, NULL); |
169 |
|
|
170 |
return 0; |
return 0; |
171 |
} |
} |
172 |
|
|
173 |
error_t |
error_t |
174 |
frame_entry_use_frame (struct container *cont, |
frame_entry_copy (struct container *cont, |
175 |
struct frame_entry *frame_entry, |
struct frame_entry *frame_entry, |
176 |
uintptr_t start, size_t size, |
uintptr_t cont_addr, size_t size, |
177 |
struct frame *frame, size_t offset) |
struct frame_entry *source, |
178 |
|
size_t frame_offset, |
179 |
|
bool shared_memory) |
180 |
{ |
{ |
181 |
error_t err; |
error_t err; |
182 |
|
|
183 |
|
assert (pthread_mutex_trylock (&cont->lock) == EBUSY); |
184 |
|
assert (pthread_mutex_trylock (&source->frame->lock) == EBUSY); |
185 |
/* Size must be a power of 2. */ |
/* Size must be a power of 2. */ |
186 |
assert (size > 0 && (size & (size - 1)) == 0); |
assert (size > 0 && (size & (size - 1)) == 0); |
187 |
assert (frame); |
assert (source->frame); |
188 |
/* Make sure that the provided offset is valid. */ |
/* Make sure that the provided offset is valid. */ |
189 |
assert (offset >= 0 && offset <= l4_size (frame->memory) - size); |
assert (frame_offset >= 0 |
190 |
/* The frame entry must refer to memory starting at a self-aligned |
&& frame_offset <= l4_size (source->frame->memory) - size); |
191 |
|
/* The frame entry must refer to memory starting at a size aligned |
192 |
boundary. */ |
boundary. */ |
193 |
assert ((offset & (size - 1)) == 0); |
assert ((frame_offset & (size - 1)) == 0); |
194 |
|
|
195 |
/* Initialize the frame_entry region. */ |
frame_entry->container = cont; |
196 |
frame_entry->region.start = start; |
frame_entry->region.start = cont_addr; |
197 |
frame_entry->region.size = size; |
frame_entry->region.size = size; |
198 |
frame_entry->frame = frame; |
frame_entry->frame = source->frame; |
199 |
frame_entry->frame_offset = offset; |
frame_entry->frame_offset = frame_offset; |
200 |
|
|
201 |
err = frame_entry_attach (cont, frame_entry); |
err = container_attach (cont, frame_entry); |
202 |
if (EXPECT_FALSE (err)) |
if (EXPECT_FALSE (err)) |
203 |
{ |
{ |
204 |
debug ("Overlap: %x+%x\n", start, size); |
debug ("Overlap: %x+%x\n", cont_addr, size); |
|
frame_deref (frame_entry->frame); |
|
205 |
return EEXIST; |
return EEXIST; |
206 |
} |
} |
207 |
|
|
208 |
frame_ref (frame); |
frame_ref (source->frame); |
209 |
frame_add_user (frame, frame_entry); |
frame_add_user (source->frame, frame_entry); |
210 |
|
|
211 |
|
if (shared_memory) |
212 |
|
/* This is a copy of the entry but the physical memory is |
213 |
|
shared. */ |
214 |
|
frame_entry_share_with (frame_entry, source); |
215 |
|
else |
216 |
|
/* Copy on write. */ |
217 |
|
{ |
218 |
|
source->frame->cow ++; |
219 |
|
frame_entry_share_with (frame_entry, NULL); |
220 |
|
} |
221 |
|
|
222 |
return 0; |
return 0; |
223 |
} |
} |
224 |
|
|
225 |
struct frame_entry * |
error_t |
226 |
frame_entry_find (struct container *cont, uintptr_t start, size_t size) |
frame_entry_use (struct container *cont, |
227 |
|
struct frame_entry *frame_entry, |
228 |
|
uintptr_t cont_addr, size_t size, |
229 |
|
struct frame *source, |
230 |
|
size_t frame_offset) |
231 |
{ |
{ |
232 |
struct region region = { start, size }; |
error_t err; |
233 |
|
|
234 |
return hurd_btree_frame_entry_find (&cont->frame_entries, ®ion); |
assert (pthread_mutex_trylock (&cont->lock) == EBUSY); |
235 |
|
assert (pthread_mutex_trylock (&source->lock) == EBUSY); |
236 |
|
/* SIZE must be a power of 2. */ |
237 |
|
assert (size > 0 && (size & (size - 1)) == 0); |
238 |
|
/* FRAME_OFFSET must be a multiple of SIZE. */ |
239 |
|
assert ((frame_offset & (size - 1)) == 0); |
240 |
|
/* The frame entry must actually cover the FRAME. */ |
241 |
|
assert (frame_offset + size <= l4_size (source->memory)); |
242 |
|
|
243 |
|
frame_entry->container = cont; |
244 |
|
frame_entry->region.start = cont_addr; |
245 |
|
frame_entry->region.size = size; |
246 |
|
frame_entry->frame_offset = frame_offset; |
247 |
|
frame_entry->frame = source; |
248 |
|
|
249 |
|
err = container_attach (cont, frame_entry); |
250 |
|
if (EXPECT_FALSE (err)) |
251 |
|
{ |
252 |
|
debug ("Overlap: %x+%x\n", cont_addr, size); |
253 |
|
return EEXIST; |
254 |
|
} |
255 |
|
|
256 |
|
frame_ref (frame_entry->frame); |
257 |
|
frame_add_user (frame_entry->frame, frame_entry); |
258 |
|
|
259 |
|
frame_entry_share_with (frame_entry, 0); |
260 |
|
|
261 |
|
return 0; |
262 |
} |
} |
263 |
|
|
264 |
void |
void |
265 |
frame_entry_drop (struct container *cont, struct frame_entry *frame_entry) |
frame_entry_destroy (struct container *cont, struct frame_entry *frame_entry, |
266 |
|
bool do_unlock) |
267 |
{ |
{ |
268 |
|
if (cont) |
269 |
|
{ |
270 |
|
assert (pthread_mutex_trylock (&cont->lock) == EBUSY); |
271 |
|
container_detach (cont, frame_entry); |
272 |
|
} |
273 |
|
|
274 |
assert (frame_entry->frame); |
assert (frame_entry->frame); |
275 |
|
assert (pthread_mutex_trylock (&frame_entry->frame->lock) == EBUSY); |
276 |
|
|
277 |
frame_drop_user (frame_entry->frame, frame_entry); |
frame_drop_user (frame_entry->frame, frame_entry); |
278 |
frame_entry_detach (cont, frame_entry); |
|
279 |
|
if (frame_entry->shared_next != frame_entry) |
280 |
|
/* FRAME_ENTRY is on a share list, remove it. */ |
281 |
|
frame_entry_share_with (frame_entry, NULL); |
282 |
|
else |
283 |
|
/* FRAME_ENTRY is not on a share list and therefore holds a COW |
284 |
|
copy if there are other users of the underlying frame. */ |
285 |
|
{ |
286 |
|
if (frame_entry->frame->frame_entries) |
287 |
|
{ |
288 |
|
assert (frame_entry->frame->cow > 0); |
289 |
|
frame_entry->frame->cow --; |
290 |
|
} |
291 |
|
else |
292 |
|
assert (frame_entry->frame->cow == 0); |
293 |
|
} |
294 |
|
|
295 |
|
if (do_unlock) |
296 |
|
frame_deref (frame_entry->frame); |
297 |
|
else |
298 |
|
frame_release (frame_entry->frame); |
299 |
|
} |
300 |
|
|
301 |
|
struct frame_entry * |
302 |
|
frame_entry_find (struct container *cont, uintptr_t cont_addr, size_t size) |
303 |
|
{ |
304 |
|
assert (pthread_mutex_trylock (&cont->lock) == EBUSY); |
305 |
|
|
306 |
|
struct region region = { cont_addr, size }; |
307 |
|
return hurd_btree_frame_entry_find (&cont->frame_entries, ®ion); |
308 |
} |
} |
309 |
|
|
310 |
error_t |
error_t |
311 |
frame_entry_attach (struct container *cont, struct frame_entry *frame_entry) |
frame_entry_map (struct frame_entry *fe, |
312 |
|
size_t start, size_t len, int access, |
313 |
|
uintptr_t vaddr, l4_msg_t msg, |
314 |
|
size_t *amount) |
315 |
{ |
{ |
316 |
return hurd_btree_frame_entry_insert (&cont->frame_entries, frame_entry); |
error_t err; |
317 |
|
|
318 |
|
assert (pthread_mutex_trylock (&fe->frame->lock) == EBUSY); |
319 |
|
assert (start < fe->region.size); |
320 |
|
assert (len <= fe->region.size); |
321 |
|
assert (start + len <= fe->region.size); |
322 |
|
assert ((access & ~HURD_PM_CONT_RWX) == 0); |
323 |
|
|
324 |
|
if (EXPECT_FALSE ((access & HURD_PM_CONT_WRITE) && fe->frame->cow)) |
325 |
|
/* The caller requests a mapping with write access and the |
326 |
|
frame is marked COW; now's the time to do the copy. */ |
327 |
|
{ |
328 |
|
/* If the frame has COW copies, there must be no extant |
329 |
|
writable mappings. */ |
330 |
|
assert (! (fe->frame->may_be_mapped & HURD_PM_CONT_WRITE)); |
331 |
|
/* If the frame has COW copies there has to be at least two |
332 |
|
users. */ |
333 |
|
assert (fe->frame->refs > 1); |
334 |
|
|
335 |
|
/* If this is a shared memory copy, we need to allocate a |
336 |
|
frame to cover the largest frame entry. */ |
337 |
|
struct frame_entry *base = fe; |
338 |
|
|
339 |
|
if (! (base->frame_offset == 0 |
340 |
|
&& base->region.size == l4_size (fe->frame->memory))) |
341 |
|
for (struct frame_entry *s = fe->shared_next; |
342 |
|
s != fe; s = s->shared_next) |
343 |
|
{ |
344 |
|
assert (s->frame == fe->frame); |
345 |
|
|
346 |
|
/* Does S contain BASE? */ |
347 |
|
if (s->frame_offset <= base->frame_offset |
348 |
|
&& (base->frame_offset + base->region.size |
349 |
|
<= s->frame_offset + s->region.size)) |
350 |
|
{ |
351 |
|
base = s; |
352 |
|
|
353 |
|
if (base->frame_offset == 0 |
354 |
|
&& base->region.size == l4_size (fe->frame->memory)) |
355 |
|
break; |
356 |
|
} |
357 |
|
} |
358 |
|
|
359 |
|
struct frame *frame = frame_alloc (base->region.size); |
360 |
|
|
361 |
|
/* The point of this function is to get a mapping of the |
362 |
|
memory. Even if ORIGINAL_FRAME doesn't have memory |
363 |
|
allocated yet (and hence no copy needs to be done), FRAME |
364 |
|
will need the memory shortly. */ |
365 |
|
frame_memory_bind (frame); |
366 |
|
|
367 |
|
/* We only have to do a memcpy if the source has memory |
368 |
|
allocated. */ |
369 |
|
if (l4_address (fe->frame->memory)) |
370 |
|
memcpy ((void *) l4_address (frame->memory), |
371 |
|
(void *) (l4_address (fe->frame->memory) |
372 |
|
+ base->frame_offset), |
373 |
|
base->region.size); |
374 |
|
|
375 |
|
/* Change everyone using this copy of FE->FRAME to use FRAME |
376 |
|
(i.e. all frame entries on FE's shared frame list). */ |
377 |
|
|
378 |
|
struct frame *original_frame = fe->frame; |
379 |
|
|
380 |
|
/* Iterate over the shared list moving all but BASE. */ |
381 |
|
struct frame_entry *falsely_shared = 0; |
382 |
|
struct frame_entry *next = base->shared_next; |
383 |
|
while (next != base) |
384 |
|
{ |
385 |
|
/* S will be removed from the shared list. Get the next |
386 |
|
element now. */ |
387 |
|
struct frame_entry *s = next; |
388 |
|
next = s->shared_next; |
389 |
|
|
390 |
|
if (s->frame_offset < base->frame_offset |
391 |
|
|| s->frame_offset >= base->frame_offset + base->region.size) |
392 |
|
/* S is falsely sharing with FE. This can happen |
393 |
|
when, for instance, a client A shares the middle |
394 |
|
8kb of a 16kb frame with a second client, B. |
395 |
|
(Hence B references 2 4kb frame entries.) If A |
396 |
|
deallocates the 16kb region, B's two frame entries |
397 |
|
are still marked as shared, however, they do not |
398 |
|
actually share any physical memory. If there is an |
399 |
|
extant COW on the physical memory then we only want |
400 |
|
to copy the memory that is actually shared (there |
401 |
|
is no need to allocate more physical memory than |
402 |
|
necessary). */ |
403 |
|
{ |
404 |
|
assert ((s->frame_offset + base->region.size |
405 |
|
<= base->frame_offset) |
406 |
|
|| (s->frame_offset |
407 |
|
>= base->frame_offset + base->region.size)); |
408 |
|
|
409 |
|
/* Remove S from the shared list. */ |
410 |
|
frame_entry_share_with (s, NULL); |
411 |
|
|
412 |
|
if (! falsely_shared) |
413 |
|
/* First one. Reset. */ |
414 |
|
falsely_shared = s; |
415 |
|
else |
416 |
|
/* Add to the falsely shared list with its |
417 |
|
possible real sharers. */ |
418 |
|
frame_entry_share_with (s, falsely_shared); |
419 |
|
} |
420 |
|
else |
421 |
|
/* Migrate S from ORIGINAL_FRAME to the copy, FRAME. |
422 |
|
(If S is BASE we migrate later.) */ |
423 |
|
{ |
424 |
|
frame_drop_user (original_frame, s); |
425 |
|
frame_release (original_frame); |
426 |
|
|
427 |
|
frame_ref (frame); |
428 |
|
frame_add_user (frame, s); |
429 |
|
s->frame = frame; |
430 |
|
s->frame_offset -= base->frame_offset; |
431 |
|
|
432 |
|
assert (s->frame_offset >= 0); |
433 |
|
assert (s->frame_offset + s->region.size |
434 |
|
<= l4_size (s->frame->memory)); |
435 |
|
} |
436 |
|
} |
437 |
|
|
438 |
|
/* Of those on the shared list, only BASE still references |
439 |
|
the original frame. Removing BASE may case some of |
440 |
|
ORIGINAL_FRAME to now be unreferenced. Hence, we cannot |
441 |
|
simply move it as we did with the others. */ |
442 |
|
uintptr_t bstart = base->region.start; |
443 |
|
size_t bsize = base->region.size; |
444 |
|
bool fe_is_base = (base == fe); |
445 |
|
|
446 |
|
frame_entry_share_with (base, NULL); |
447 |
|
|
448 |
|
/* Reallocate the frame entry. */ |
449 |
|
err = frame_entry_deallocate (base->container, base, bstart, bsize); |
450 |
|
assert_perror (err); |
451 |
|
base = frame_entry_alloc (); |
452 |
|
err = frame_entry_use (base->container, base, bstart, bsize, frame, 0); |
453 |
|
assert_perror (err); |
454 |
|
|
455 |
|
/* All the frame entries using FRAME are on the shared |
456 |
|
list. */ |
457 |
|
frame_entry_share_with (base, frame->frame_entries); |
458 |
|
|
459 |
|
if (fe_is_base) |
460 |
|
fe = base; |
461 |
|
|
462 |
|
/* We managed to pick up an extra reference to FRAME in the |
463 |
|
loop (we already had one before we entered the loop and |
464 |
|
we added a reference for each entry which shares the |
465 |
|
frame including FE). Drop it now. */ |
466 |
|
frame_release (frame); |
467 |
|
} |
468 |
|
else |
469 |
|
/* Allocate the memory (if needed). */ |
470 |
|
frame_memory_bind (fe->frame); |
471 |
|
|
472 |
|
fe->frame->may_be_mapped |= access; |
473 |
|
|
474 |
|
/* Get the start of the mem. */ |
475 |
|
l4_word_t mem = l4_address (fe->frame->memory) + fe->frame_offset + start; |
476 |
|
|
477 |
|
l4_fpage_t fpages[(L4_NUM_MRS - (l4_untyped_words (l4_msg_msg_tag (msg)) |
478 |
|
+ l4_untyped_words (l4_msg_msg_tag (msg)))) |
479 |
|
/ 2]; |
480 |
|
int nr_fpages = l4_fpage_xspan (mem, mem + len - 1, vaddr, |
481 |
|
fpages, sizeof (fpages) / sizeof (*fpages)); |
482 |
|
|
483 |
|
for (int i = 0; i < nr_fpages; i ++) |
484 |
|
{ |
485 |
|
/* Set the desired permissions. */ |
486 |
|
l4_set_rights (&fpages[i], access); |
487 |
|
|
488 |
|
/* Add the map item to the message. */ |
489 |
|
l4_msg_append_map_item (msg, l4_map_item (fpages[i], vaddr)); |
490 |
|
|
491 |
|
vaddr += l4_size (fpages[i]); |
492 |
|
} |
493 |
|
|
494 |
|
if (amount) |
495 |
|
*amount = l4_address (fpages[nr_fpages - 1]) |
496 |
|
+ l4_size (fpages[nr_fpages - 1]) - mem; |
497 |
|
|
498 |
|
return |
499 |
|
l4_address (fpages[nr_fpages]) + l4_size (fpages[nr_fpages]) - mem < len |
500 |
|
? ENOSPC : 0; |
501 |
} |
} |
502 |
|
|
503 |
void |
error_t |
504 |
frame_entry_detach (struct container *cont, struct frame_entry *frame_entry) |
frame_entry_deallocate (struct container *cont, |
505 |
|
struct frame_entry *frame_entry, |
506 |
|
const uintptr_t cont_addr, |
507 |
|
const size_t len) |
508 |
{ |
{ |
509 |
assert (hurd_btree_frame_entry_find (&cont->frame_entries, |
const uintptr_t cont_start |
510 |
&frame_entry->region)); |
= cont_addr - frame_entry->region.start + frame_entry->frame_offset; |
511 |
hurd_btree_frame_entry_detach (&cont->frame_entries, frame_entry); |
struct frame *frame; |
512 |
|
|
513 |
|
/* FE currently uses FE->FRAME. Make the TODO bytes of memory FE |
514 |
|
references starting at byte SKIP (relative to the base of FE) use |
515 |
|
the array of FRAMES. |
516 |
|
|
517 |
|
Returns the last frame used (i.e. the one which contains byte |
518 |
|
SKIP+TODO). */ |
519 |
|
struct frame **move (struct frame_entry *fe, size_t skip, size_t todo, |
520 |
|
l4_fpage_t *fpages, struct frame **frames) |
521 |
|
{ |
522 |
|
error_t err; |
523 |
|
|
524 |
|
assert (todo > 0); |
525 |
|
/* The first byte of FE may not by the first byte of |
526 |
|
*FRAMES. For instance, FRAMES may be 8kb long |
527 |
|
but FE references only the second 4kb. */ |
528 |
|
assert (skip < fe->region.size); |
529 |
|
assert (fe->frame_offset + skip >= l4_address (*fpages)); |
530 |
|
uintptr_t frame_offset = fe->frame_offset + skip - l4_address (*fpages); |
531 |
|
|
532 |
|
/* The container address of the first byte. */ |
533 |
|
uintptr_t addr = fe->region.start + skip; |
534 |
|
|
535 |
|
for (; todo > 0; frames ++, fpages ++) |
536 |
|
{ |
537 |
|
size_t count = l4_size (*fpages) - frame_offset; |
538 |
|
if (count > todo) |
539 |
|
count = todo; |
540 |
|
|
541 |
|
l4_fpage_t subfpages[L4_FPAGE_SPAN_MAX]; |
542 |
|
int n = l4_fpage_span (frame_offset, |
543 |
|
frame_offset + count - 1, |
544 |
|
subfpages); |
545 |
|
|
546 |
|
for (int i = 0; i < n; i ++) |
547 |
|
{ |
548 |
|
struct frame_entry *n = frame_entry_alloc (); |
549 |
|
|
550 |
|
err = frame_entry_use (cont, n, addr, |
551 |
|
l4_size (subfpages[i]), |
552 |
|
*frames, l4_address (subfpages[i])); |
553 |
|
assert_perror (err); |
554 |
|
|
555 |
|
/* Although they only falsely share FE->FRAME (which is |
556 |
|
perfectly correct), the new frame entries are on a |
557 |
|
share list to reduce the number of gratuitous COWs: |
558 |
|
there is one COW for each shared copy; if there is |
559 |
|
only a single shared copy then no COWs need to be |
560 |
|
performed. */ |
561 |
|
frame_entry_share_with (n, fe); |
562 |
|
|
563 |
|
frame_offset = 0; |
564 |
|
addr += l4_size (subfpages[i]); |
565 |
|
todo -= l4_size (subfpages[i]); |
566 |
|
} |
567 |
|
} |
568 |
|
|
569 |
|
frames --; |
570 |
|
|
571 |
|
return frames; |
572 |
|
} |
573 |
|
|
574 |
|
/* Migrate the frame entries using FRAME between byte START and END |
575 |
|
to new frame structures which use the same physical memory. */ |
576 |
|
void migrate (uintptr_t start, uintptr_t end) |
577 |
|
{ |
578 |
|
assert (start < end); |
579 |
|
/* START must come before the end of the deallocation zone. */ |
580 |
|
assert (start <= cont_start + len); |
581 |
|
/* END must come after the start of the deallocation zone. */ |
582 |
|
assert (end + 1 >= cont_start); |
583 |
|
|
584 |
|
/* FRAME_ENTRY must cover all of the underlying frame. */ |
585 |
|
assert (frame_entry->frame_offset == 0); |
586 |
|
assert (frame_entry->region.size == l4_size (frame->memory)); |
587 |
|
|
588 |
|
/* Allocate new frames and point them to their respective pieces |
589 |
|
of FRAME. */ |
590 |
|
l4_fpage_t fpages[L4_FPAGE_SPAN_MAX]; |
591 |
|
int nr_fpages = l4_fpage_span (start, end, fpages); |
592 |
|
struct frame **frames = alloca (sizeof (struct frame *) * nr_fpages); |
593 |
|
|
594 |
|
for (int i = 0; i < nr_fpages; i ++) |
595 |
|
{ |
596 |
|
frames[i] = frame_alloc (l4_size (fpages[i])); |
597 |
|
frames[i]->memory = l4_fpage (l4_address (frame->memory) |
598 |
|
+ l4_address (fpages[i]), |
599 |
|
l4_size (fpages[i])); |
600 |
|
frames[i]->may_be_mapped = frame->may_be_mapped; |
601 |
|
} |
602 |
|
|
603 |
|
/* Move the parts of FRAME_ENTRY which are not going to be |
604 |
|
deallocated to the new frames. */ |
605 |
|
|
606 |
|
int i = 0; |
607 |
|
|
608 |
|
/* If START is before CONT_START then we need to relocate some |
609 |
|
of FRAME_ENTRY. |
610 |
|
|
611 |
|
START END |
612 |
|
v v |
613 |
|
[ | | | | | ] <- FRAME_ENTRY |
614 |
|
^ ^ |
615 |
|
| CONT_START+LEN |
616 |
|
CONT_START |
617 |
|
\/ |
618 |
|
keep |
619 |
|
|
620 |
|
(END can be before CONT_START.) |
621 |
|
*/ |
622 |
|
if (start < cont_start) |
623 |
|
{ |
624 |
|
size_t todo; |
625 |
|
if (end < cont_start) |
626 |
|
todo = end - start + 1; |
627 |
|
else |
628 |
|
todo = cont_start - start; |
629 |
|
|
630 |
|
/* Find the frame which contains byte START. */ |
631 |
|
for (; i < nr_fpages; i ++) |
632 |
|
if (start < l4_address (fpages[i]) + l4_size (fpages[i])) |
633 |
|
break; |
634 |
|
|
635 |
|
struct frame **l |
636 |
|
= move (frame_entry, start, todo, &fpages[i], &frames[i]); |
637 |
|
|
638 |
|
int last = i + ((void *) l - (void *) &frames[i]) / sizeof (*l); |
639 |
|
assert (last < nr_fpages); |
640 |
|
for (int j = i; j <= last; j ++) |
641 |
|
frames[j]->cow ++; |
642 |
|
i = last; |
643 |
|
} |
644 |
|
|
645 |
|
/* If CONT_START+LEN is before END then we need to relocate some |
646 |
|
of FRAME_ENTRY. |
647 |
|
|
648 |
|
START END |
649 |
|
v v |
650 |
|
[ | | | | | ] <- FRAME_ENTRY |
651 |
|
^ ^ |
652 |
|
| CONT_START+LEN |
653 |
|
CONT_START |
654 |
|
\ / |
655 |
|
keep |
656 |
|
|
657 |
|
(START can be after CONT_START+LEN.) |
658 |
|
*/ |
659 |
|
if (cont_start + len < end) |
660 |
|
{ |
661 |
|
size_t skip; |
662 |
|
|
663 |
|
if (start < cont_start + len) |
664 |
|
skip = cont_start + len; |
665 |
|
else |
666 |
|
skip = start; |
667 |
|
|
668 |
|
/* Find the frame which contains the first byte referenced |
669 |
|
by FRAME_ENTRY after the region to deallocate. */ |
670 |
|
for (; i < nr_fpages; i ++) |
671 |
|
if (skip >= l4_address (fpages[i])) |
672 |
|
break; |
673 |
|
|
674 |
|
struct frame **l |
675 |
|
= move (frame_entry, skip, end - skip + 1, &fpages[i], &frames[i]); |
676 |
|
|
677 |
|
int last = i + ((void *) l - (void *) &frames[i]) / sizeof (*l); |
678 |
|
assert (last < nr_fpages); |
679 |
|
for (int j = i; j <= last; j ++) |
680 |
|
frames[j]->cow ++; |
681 |
|
} |
682 |
|
|
683 |
|
/* Change the rest of the frame entries referencing FRAME |
684 |
|
between START and END to reference the respective frames in |
685 |
|
FRAMES. */ |
686 |
|
|
687 |
|
struct frame_entry *n = frame->frame_entries; |
688 |
|
while (n) |
689 |
|
{ |
690 |
|
struct frame_entry *fe = n; |
691 |
|
n = fe->next; |
692 |
|
|
693 |
|
/* Any frame entries connected to FRAME_ENTRY should |
694 |
|
reference the same frame. */ |
695 |
|
assert (frame == fe->frame); |
696 |
|
|
697 |
|
/* Any frames entries referencing memory before START should |
698 |
|
have been relocated from FRAME_ENTRY in a prior pass |
699 |
|
(except for FRAME_ENTRY, of course). */ |
700 |
|
assert (fe == frame_entry || fe->frame_offset >= start); |
701 |
|
|
702 |
|
if (fe == frame_entry) |
703 |
|
continue; |
704 |
|
else if (fe->frame_offset < end) |
705 |
|
{ |
706 |
|
/* END is either the end of the frame or the memory |
707 |
|
following END is completely unreferenced (and to be |
708 |
|
deallocated). Hence any frame entry which starts |
709 |
|
before END ends before it as well. */ |
710 |
|
assert (fe->frame_offset + fe->region.size - 1 <= end); |
711 |
|
|
712 |
|
void adjust (struct frame_entry *fe, int i) |
713 |
|
{ |
714 |
|
assert (fe->frame == frame); |
715 |
|
/* FE fits entirely in FRAMES[I]. */ |
716 |
|
assert (l4_address (fpages[i]) <= fe->frame_offset |
717 |
|
&& (fe->frame_offset + fe->region.size |
718 |
|
<= (l4_address (fpages[i]) |
719 |
|
+ l4_size (fpages[i])))); |
720 |
|
|
721 |
|
/* Adjust the frame offset. */ |
722 |
|
fe->frame_offset -= l4_address (fpages[i]); |
723 |
|
|
724 |
|
/* Make sure N always points to an unprocessed frame |
725 |
|
entry on FRAME->FRAME_ENTRIES. */ |
726 |
|
if (fe == n) |
727 |
|
n = fe->next; |
728 |
|
|
729 |
|
/* Move from the old frame to the new one. */ |
730 |
|
frame_drop_user (frame, fe); |
731 |
|
frame_release (frame); |
732 |
|
|
733 |
|
fe->frame = frames[i]; |
734 |
|
|
735 |
|
frame_ref (fe->frame); |
736 |
|
frame_add_user (fe->frame, fe); |
737 |
|
} |
738 |
|
|
739 |
|
/* Find the frame which holds the start of the |
740 |
|
memory E references. */ |
741 |
|
int i; |
742 |
|
for (i = 0; i < nr_fpages; i ++) |
743 |
|
if (fe->frame_offset >= l4_address (fpages[i]) |
744 |
|
&& (fe->frame_offset |
745 |
|
< l4_address (fpages[i]) + l4_size (fpages[i]))) |
746 |
|
break; |
747 |
|
|
748 |
|
adjust (fe, i); |
749 |
|
|
750 |
|
if (fe->shared_next == fe) |
751 |
|
/* FE was not on a shared list. Remove its COW from |
752 |
|
FRAME. Add a COW to the new frame. */ |
753 |
|
{ |
754 |
|
assert (frame->cow > 0); |
755 |
|
frame->cow --; |
756 |
|
fe->frame->cow ++; |
757 |
|
} |
758 |
|
else |
759 |
|
/* FE was on a shared list. Fix it up. */ |
760 |
|
{ |
761 |
|
bool shares_old_cow = false; |
762 |
|
bool shares_new_cow_with_frame_entry = false; |
763 |
|
|
764 |
|
/* We need to use FE as an anchor hence we attach to |
765 |
|
here and then at the end detach FE and attach it |
766 |
|
to the resulting list. */ |
767 |
|
struct frame_entry *shared_list = 0; |
768 |
|
|
769 |
|
struct frame_entry *sn = fe->shared_next; |
770 |
|
while (sn != fe) |
771 |
|
{ |
772 |
|
struct frame_entry *s = sn; |
773 |
|
sn = s->shared_next; |
774 |
|
|
775 |
|
if (s == frame_entry) |
776 |
|
shares_old_cow = true; |
777 |
|
else if (s->frame != frame) |
778 |
|
/* S was already relocated which means that it |
779 |
|
was split off from FRAME_ENTRY which means |
780 |
|
that the cow was already counted. */ |
781 |
|
{ |
782 |
|
shares_old_cow = true; |
783 |
|
|
784 |
|
if (s->frame == fe->frame) |
785 |
|
{ |
786 |
|
shares_new_cow_with_frame_entry = true; |
787 |
|
|
788 |
|
frame_entry_share_with (s, NULL); |
789 |
|
if (! shared_list) |
790 |
|
shared_list = s; |
791 |
|
else |
792 |
|
frame_entry_share_with (s, shared_list); |
793 |
|
} |
794 |
|
} |
795 |
|
else if (l4_address (fpages[i]) <= s->frame_offset |
796 |
|
&& (s->frame_offset < l4_address (fpages[i]) |
797 |
|
+ l4_size (fpages[i]))) |
798 |
|
/* S and FE continue to share a copy of the |
799 |
|
underlying frame (i.e. no false |
800 |
|
sharing). */ |
801 |
|
{ |
802 |
|
adjust (s, i); |
803 |
|
|
804 |
|
frame_entry_share_with (s, NULL); |
805 |
|
if (! shared_list) |
806 |
|
shared_list = s; |
807 |
|
else |
808 |
|
frame_entry_share_with (s, shared_list); |
809 |
|
} |
810 |
|
else |
811 |
|
shares_old_cow = true; |
812 |
|
} |
813 |
|
|
814 |
|
frame_entry_share_with (fe, 0); |
815 |
|
if (shared_list) |
816 |
|
frame_entry_share_with (fe, shared_list); |
817 |
|
|
818 |
|
if (! shares_old_cow) |
819 |
|
/* There was no false sharing, i.e. there are no |
820 |
|
frame entries still using this copy of the old |
821 |
|
frame. */ |
822 |
|
{ |
823 |
|
assert (frame->cow > 0); |
824 |
|
frame->cow --; |
825 |
|
} |
826 |
|
|
827 |
|
if (! shares_new_cow_with_frame_entry) |
828 |
|
/* Unless we share our copy of the underlying |
829 |
|
frame with FRAME_ENTRY, we need to add a |
830 |
|
COW. */ |
831 |
|
fe->frame->cow ++; |
832 |
|
} |
833 |
|
} |
834 |
|
else |
835 |
|
assert (fe->frame_offset > end); |
836 |
|
} |
837 |
|
|
838 |
|
/* Any new frame entries created from FRAME_ENTRY are put on its |
839 |
|
shared list. If they were not picked up above (because |
840 |
|
FRAME_ENTRY is on a share list) then some of them may not |
841 |
|
have been properly fixed up. */ |
842 |
|
if (frame_entry->shared_next != frame_entry) |
843 |
|
{ |
844 |
|
struct frame_entry *n = frame_entry->shared_next; |
845 |
|
while (n != frame_entry) |
846 |
|
{ |
847 |
|
struct frame_entry *fe = n; |
848 |
|
n = fe->shared_next; |
849 |
|
|
850 |
|
if (fe->frame != frame) |
851 |
|
/* This is a new frame entry. */ |
852 |
|
{ |
853 |
|
assert (l4_address (frame->memory) |
854 |
|
<= l4_address (fe->frame->memory)); |
855 |
|
assert (l4_address (fe->frame->memory) |
856 |
|
<= (l4_address (frame->memory) |
857 |
|
+ l4_size (frame->memory))); |
858 |
|
|
859 |
|
struct frame_entry *shared_list = 0; |
860 |
|
struct frame_entry *m = fe->shared_next; |
861 |
|
while (m != fe) |
862 |
|
{ |
863 |
|
struct frame_entry *b = m; |
864 |
|
m = m->shared_next; |
865 |
|
|
866 |
|
if (fe->frame == b->frame) |
867 |
|
{ |
868 |
|
if (b == n) |
869 |
|
n = n->shared_next; |
870 |
|
|
871 |
|
frame_entry_share_with (b, NULL); |
872 |
|
if (! shared_list) |
873 |
|
shared_list = b; |
874 |
|
else |
875 |
|
frame_entry_share_with (b, shared_list); |
876 |
|
} |
877 |
|
} |
878 |
|
|
879 |
|
frame_entry_share_with (fe, 0); |
880 |
|
if (shared_list) |
881 |
|
frame_entry_share_with (fe, shared_list); |
882 |
|
} |
883 |
|
} |
884 |
|
} |
885 |
|
|
886 |
|
/* Tidy up the new frames. */ |
887 |
|
for (i = 0; i < nr_fpages; i ++) |
888 |
|
{ |
889 |
|
/* Each frame should have picked up at least one frame |
890 |
|
entry. */ |
891 |
|
assert (frames[i]->frame_entries); |
892 |
|
|
893 |
|
/* Each user of FRAMES[i] added a cow. That is one too |
894 |
|
many. Remove it now. */ |
895 |
|
assert (frames[i]->cow > 0); |
896 |
|
frames[i]->cow --; |
897 |
|
if (frames[i]->cow > 0 |
898 |
|
&& (frames[i]->may_be_mapped & L4_FPAGE_WRITABLE)) |
899 |
|
{ |
900 |
|
l4_unmap_fpage (l4_fpage_add_rights (frames[i]->memory, |
901 |
|
L4_FPAGE_WRITABLE)); |
902 |
|
frames[i]->may_be_mapped |
903 |
|
&= L4_FPAGE_EXECUTABLE|L4_FPAGE_READABLE; |
904 |
|
} |
905 |
|
|
906 |
|
/* A new frame starts life with a single reference (even |
907 |
|
though no frame entries use it). We drop that extra one |
908 |
|
now. */ |
909 |
|
assert (frames[i]->refs > 1); |
910 |
|
frame_deref (frames[i]); |
911 |
|
} |
912 |
|
} |
913 |
|
|
914 |
|
assert (pthread_mutex_trylock (&cont->lock) == EBUSY); |
915 |
|
/* Assert that the region to deallocate falls completely within |
916 |
|
FRAME_ENTRY. */ |
917 |
|
assert (cont_addr >= frame_entry->region.start |
918 |
|
&& (cont_addr + len |
919 |
|
<= frame_entry->region.start + frame_entry->region.size)); |
920 |
|
/* Assert that CONT_ADDR refers to memory which starts on a |
921 |
|
multiple of the base page size. */ |
922 |
|
assert ((cont_start & (L4_MIN_PAGE_SIZE - 1)) == 0); |
923 |
|
/* And that LEN is a multiple of the base page size. */ |
924 |
|
assert ((len & (L4_MIN_PAGE_SIZE - 1)) == 0); |
925 |
|
|
926 |
|
frame = frame_entry->frame; |
927 |
|
assert (pthread_mutex_trylock (&frame->lock) == EBUSY); |
928 |
|
assert (frame->frame_entries); |
929 |
|
|
930 |
|
#if 0 |
931 |
|
printf ("%s (cont:%x, fe: %x, dzone:%x+%x); ", |
932 |
|
__FUNCTION__, cont, frame_entry, cont_start, len); |
933 |
|
frame_entry_dump (frame_entry); |
934 |
|
#endif |
935 |
|
|
936 |
|
/* Before we do anything else, we need to make sure that any |
937 |
|
mappings via FRAME_ENTRY are removed: most importantly, if we |
938 |
|
zfree any memory and then reallocate (either internally or by |
939 |
|
another process) before it is unmapped, any extant mappers may |
940 |
|
have the opportunity to see (or modify) it; but also, any |
941 |
|
mappings made via FRAME_ENTRY of the region to deallocate must |
942 |
|
(eventually) be invalidated. Unfortunately, this means |
943 |
|
invalidating all mappings of FRAME. */ |
944 |
|
if (frame->may_be_mapped) |
945 |
|
{ |
946 |
|
l4_fpage_t fpage = l4_fpage (l4_address (frame->memory) |
947 |
|
+ frame_entry->frame_offset, |
948 |
|
frame_entry->region.size); |
949 |
|
l4_unmap_fpage (l4_fpage_add_rights (fpage, frame->may_be_mapped)); |
950 |
|
|
951 |
|
/* If we unmapped the whole frame then we can clear |
952 |
|
FRAME->MAY_BE_MAPPED. */ |
953 |
|
if (frame_entry->frame_offset == 0 |
954 |
|
&& frame_entry->region.size == l4_size (frame->memory)) |
955 |
|
frame->may_be_mapped = 0; |
956 |
|
} |
957 |
|
|
958 |
|
/* Detach FRAME_ENTRY from its container: frame entries in the same |
959 |
|
container cannot overlap and we are going to replace the parts of |
960 |
|
FRAME_ENTRY with a set of smaller frame entries covering the |
961 |
|
physical memory which will not be deallocated. */ |
962 |
|
container_detach (cont, frame_entry); |
963 |
|
|
964 |
|
|
965 |
|
if (! frame->frame_entries->next) |
966 |
|
/* FRAME_ENTRY is the only frame entry using FRAME. */ |
967 |
|
{ |
968 |
|
/* Make sure it is using the entire frame. */ |
969 |
|
assert (frame_entry->frame_offset == 0); |
970 |
|
assert (frame_entry->region.size == l4_size (frame->memory)); |
971 |
|
|
972 |
|
if (cont_start > 0) |
973 |
|
migrate (0, cont_start - 1); |
974 |
|
if (cont_start + len < l4_size (frame->memory)) |
975 |
|
migrate (cont_start + len, l4_size (frame->memory) - 1); |
976 |
|
|
977 |
|
assert (frame->refs == 1); |
978 |
|
|
979 |
|
/* If some of the frame entry was migrated, we manually free any |
980 |
|
physical memory. */ |
981 |
|
if (cont_start > 0 || cont_start + len < l4_size (frame->memory)) |
982 |
|
{ |
983 |
|
l4_fpage_t fpages[L4_FPAGE_SPAN_MAX]; |
984 |
|
int nr_fpages = l4_fpage_span (cont_start, |
985 |
|
cont_start + len - 1, fpages); |
986 |
|
|
987 |
|
for (int i = 0; i < nr_fpages; i ++) |
988 |
|
{ |
989 |
|
#ifndef NDEBUG |
990 |
|
memset ((void *) l4_address (frame->memory) |
991 |
|
+ l4_address (fpages[i]), |
992 |
|
0xde, l4_size (fpages[i])); |
993 |
|
#endif |
994 |
|
zfree (l4_address (frame->memory) + l4_address (fpages[i]), |
995 |
|
l4_size (fpages[i])); |
996 |
|
} |
997 |
|
|
998 |
|
frame->may_be_mapped = 0; |
999 |
|
frame->memory = l4_fpage (0, l4_size (frame->memory)); |
1000 |
|
} |
1001 |
|
|
1002 |
|
frame_entry_destroy (NULL, frame_entry, true); |
1003 |
|
frame_entry_free (frame_entry); |
1004 |
|
return 0; |
1005 |
|
} |
1006 |
|
|
1007 |
|
if (frame_entry->frame_offset > 0 |
1008 |
|
|| frame_entry->region.size < l4_size (frame->memory)) |
1009 |
|
/* FRAME_ENTRY does not cover all of the underlying frame. By |
1010 |
|
definition, some other frame entry must. As such, all we have |
1011 |
|
to do is fix up the parts of FRAME_ENTRY which will not be |
1012 |
|
deallocated and then drop it. */ |
1013 |
|
{ |
1014 |
|
#ifndef NDEBUG |
1015 |
|
/* Assert that a frame entry covers all of FE. */ |
1016 |
|
struct frame_entry *fe; |
1017 |
|
for (fe = frame->frame_entries; fe; fe = fe->next) |
1018 |
|
if (fe->frame_offset == 0 |
1019 |
|
&& fe->region.size == l4_size (frame->memory)) |
1020 |
|
break; |
1021 |
|
assert (fe); |
1022 |
|
#endif |
1023 |
|
|
1024 |
|
l4_fpage_t fpage = l4_fpage (0, l4_size (fe->frame->memory)); |
1025 |
|
struct frame **f; |
1026 |
|
|
1027 |
|
if (frame_entry->frame_offset < cont_start) |
1028 |
|
{ |
1029 |
|
f = move (frame_entry, 0, cont_start - frame_entry->frame_offset, |
1030 |
|
&fpage, &frame_entry->frame); |
1031 |
|
assert (f == &frame_entry->frame); |
1032 |
|
} |
1033 |
|
|
1034 |
|
if (cont_start + len |
1035 |
|
< frame_entry->frame_offset + frame_entry->region.size) |
1036 |
|
{ |
1037 |
|
f = move (frame_entry, |
1038 |
|
cont_start + len - frame_entry->frame_offset, |
1039 |
|
frame_entry->frame_offset + frame_entry->region.size |
1040 |
|
- (cont_start + len), |
1041 |
|
&fpage, &frame_entry->frame); |
1042 |
|
assert (f == &frame_entry->frame); |
1043 |
|
} |
1044 |
|
|
1045 |
|
frame_entry_destroy (NULL, frame_entry, true); |
1046 |
|
frame_entry_free (frame_entry); |
1047 |
|
return 0; |
1048 |
|
} |
1049 |
|
|
1050 |
|
|
1051 |
|
/* Multiple frame entries reference FRAME_ENTRY->FRAME. Since frame |
1052 |
|
entries may reference only part of the underlying frame, by |
1053 |
|
releasing FRAME_ENTRY, 1) no single frame entry may now reference |
1054 |
|
all of the underlying frame and 2) some of FRAME_ENTRY->FRAME may |
1055 |
|
no longer be referenced and thus can be freed. For example, a |
1056 |
|
client, A, may allocate a 16kb frame. A may give a second |
1057 |
|
client, B, a copy of the middle 8kb. (Since the start address of |
1058 |
|
the 8kb area is not a multiple of the areas size, we create two |
1059 |
|
frame entries: one for each 4kb region.) If A then deallocates |
1060 |
|
the 16kb region, we would like to release the unreferenced |
1061 |
|
physical memory. By inspection, we see that the first 4kb and |
1062 |
|
the last 4kb are no longer used and could be freed: |
1063 |
|
|
1064 |
|
A |
1065 |
|
/ \ |
1066 |
|
0kb| | | | |16kb |
1067 |
|
\ / \ / |
1068 |
|
B.1 B.2 |
1069 |
|
|
1070 |
|
This problem becomes slightly more complicated when only part of |
1071 |
|
the frame entry is freed, e.g. if the client only deallocate the |
1072 |
|
first 8kb of A. Further, we must maintain the predicate that all |
1073 |
|
frames have at least one frame entry which references them in |
1074 |
|
their entirety. |
1075 |
|
|
1076 |
|
We take the following approach: |
1077 |
|
|
1078 |
|
Set A=(DEALLOC_START, LEN) to the region to deallocate (relative |
1079 |
|
to the start of the underlying frame). To identify unreferenced |
1080 |
|
regions, iterate over the frame entries referencing the frame |
1081 |
|
(excluding the one to deallocate). If the intersection of the |
1082 |
|
frame entry and A is non-empty (i.e. A contains any part of the |
1083 |
|
frame entry), save the result in T. If R is not set, set it to |
1084 |
|
T. Otherwise, if T occurs before R, set R to T. |
1085 |
|
|
1086 |
|
If R completely covers the region to deallocate, A, we are done. |
1087 |
|
Otherwise, any memory between the start of A and the start of R |
1088 |
|
is unreferenced and we can free it (doing any required frame |
1089 |
|
splitting). Move the start of A to end of R. If the area is |
1090 |
|
non-NULL, repeat from the beginning. Otherwise, we are done. */ |
1091 |
|
|
1092 |
|
|
1093 |
|
/* Start of the region to deallocate relative to the start of the |
1094 |
|
frame which we still need to confirmed as referenced or not. */ |
1095 |
|
uintptr_t dealloc_start = cont_start; |
1096 |
|
size_t dealloc_len = len; |
1097 |
|
|
1098 |
|
/* Area within DEALLOC_START+DEALLOC_LEN which we've confirmed |
1099 |
|
another frame entry references. |
1100 |
|
|
1101 |
|
Initially, we (consistent with the above predicate) set the start |
1102 |
|
address to the end of the region to deallocate with a length of |
1103 |
|
0. Because we prefer earlier and larger regions, if this isn't |
1104 |
|
changed after iterating over all of the frame entries, we know it |
1105 |
|
is safe to free the region. */ |
1106 |
|
uintptr_t refed_start = dealloc_start + dealloc_len; |
1107 |
|
size_t refed_len = 0; |
1108 |
|
|
1109 |
|
/* Once we've identified a region which we can free (i.e. a region |
1110 |
|
which no frame entry references), we will need to split memory |
1111 |
|
which is still referenced into smaller frames. This is |
1112 |
|
complicated by the fact that there may be multiple holes. |
1113 |
|
Consider: |
1114 |
|
|
1115 |
|
FE |
1116 |
|
/ \ |
1117 |
|
[ | | | ] |
1118 |
|
0 4 8 C F |
1119 |
|
\ / \ / |
1120 |
|
A B |
1121 |
|
|
1122 |
|
If all of frame entry FE is freed, frame entry A and B still |
1123 |
|
reference two 4k segments of the frame. We can free from the 4k |
1124 |
|
regions starting at 0k and 8k. |
1125 |
|
|
1126 |
|
After each iteration, during which we've identified a region |
1127 |
|
which is referenced, we free the memory between PROCESSED and |
1128 |
|
REFED_START and relocate the frame entries between |
1129 |
|
REFED_START+REFED_LEN. We then set PROCESSED to |
1130 |
|
REFED_START+REFED_LEN. */ |
1131 |
|
uintptr_t processed = 0; |
1132 |
|
|
1133 |
|
for (;;) |
1134 |
|
{ |
1135 |
|
/* Iterate over the frame entries checking to see if they |
1136 |
|
reference DEALLOC_START+DEALLOC_LEN and modifying |
1137 |
|
REFED_START+REFED_LEN appropriately. */ |
1138 |
|
for (struct frame_entry *fe = frame->frame_entries; fe; fe = fe->next) |
1139 |
|
{ |
1140 |
|
assert (fe->frame == frame); |
1141 |
|
|
1142 |
|
/* Don't consider the frame entry we are deallocating. */ |
1143 |
|
if (fe == frame_entry) |
1144 |
|
continue; |
1145 |
|
|
1146 |
|
if (fe->frame_offset + fe->region.size <= dealloc_start) |
1147 |
|
/* FE ends before the region to deallocate begins. */ |
1148 |
|
continue; |
1149 |
|
|
1150 |
|
if (fe->frame_offset >= dealloc_start + dealloc_len) |
1151 |
|
/* FE starts after the region to deallocate ends. */ |
1152 |
|
continue; |
1153 |
|
|
1154 |
|
if (fe->frame_offset < refed_start) |
1155 |
|
/* FE covers at least part of the region to deallocate and |
1156 |
|
starts before what we've found so far. */ |
1157 |
|
{ |
1158 |
|
refed_start = fe->frame_offset; |
1159 |
|
refed_len = fe->region.size; |
1160 |
|
} |
1161 |
|
else if (fe->frame_offset == refed_start |
1162 |
|
&& fe->region.size > refed_len) |
1163 |
|
/* FE starts at REFED_START and is larger than |
1164 |
|
REFED_LEN. */ |
1165 |
|
refed_len = fe->region.size; |
1166 |
|
} |
1167 |
|
|
1168 |
|
if (processed < refed_start && processed < dealloc_start) |
1169 |
|
/* PROCESSED comes before both REFED_START and DEALLOC_START. |
1170 |
|
If there is memory to be freed, that memory is between |
1171 |
|
DEALLOC_START and REFED_START. On the other hand, |
1172 |
|
REFED_START may come before DEALLOC_START if a frame |
1173 |
|
straddles DEALLOC_START. There is no need to gratuitously |
1174 |
|
split it apart. */ |
1175 |
|
migrate (processed, |
1176 |
|
refed_start < dealloc_start |
1177 |
|
? refed_start - 1 : dealloc_start - 1); |
1178 |
|
|
1179 |
|
/* The area between DEALLOC_START and REFED_START is not |
1180 |
|
referenced. Free it and adjust the frame entries. */ |
1181 |
|
|
1182 |
|
if (dealloc_start < refed_start && l4_address (frame->memory)) |
1183 |
|
{ |
1184 |
|
l4_fpage_t fpages[L4_FPAGE_SPAN_MAX]; |
1185 |
|
int nr_fpages = l4_fpage_span (dealloc_start, |
1186 |
|
refed_start - 1, fpages); |
1187 |
|
|
1188 |
|
for (int i = 0; i < nr_fpages; i ++) |
1189 |
|
{ |
1190 |
|
#ifndef NDEBUG |
1191 |
|
memset ((void *) l4_address (frame->memory) |
1192 |
|
+ l4_address (fpages[i]), |
1193 |
|
0xde, l4_size (fpages[i])); |
1194 |
|
#endif |
1195 |
|
zfree (l4_address (frame->memory) + l4_address (fpages[i]), |
1196 |
|
l4_size (fpages[i])); |
1197 |
|
} |
1198 |
|
} |
1199 |
|
|
1200 |
|
if (refed_len > 0) |
1201 |
|
migrate (refed_start, refed_start + refed_len - 1); |
1202 |
|
processed = refed_start + refed_len; |
1203 |
|
|
1204 |
|
if (refed_start + refed_len >= dealloc_start + dealloc_len) |
1205 |
|
break; |
1206 |
|
|
1207 |
|
dealloc_len -= refed_start + refed_len - dealloc_start; |
1208 |
|
dealloc_start = refed_start + refed_len; |
1209 |
|
|
1210 |
|
refed_start = dealloc_start + dealloc_len; |
1211 |
|
refed_len = 0; |
1212 |
|
} |
1213 |
|
|
1214 |
|
/* Move any remaining frame entries over. */ |
1215 |
|
if (processed < l4_size (frame->memory)) |
1216 |
|
migrate (processed, l4_size (frame->memory) - 1); |
1217 |
|
|
1218 |
|
/* And destroy the now redundant FRAME_ENTRY. But don't let it |
1219 |
|
deallocate the physical memory! */ |
1220 |
|
frame->memory = l4_fpage (0, l4_size (frame->memory)); |
1221 |
|
|
1222 |
|
assert (frame->refs == 1); |
1223 |
|
assert (frame->frame_entries == frame_entry); |
1224 |
|
assert (! frame->frame_entries->next); |
1225 |
|
assert (frame_entry->shared_next == frame_entry); |
1226 |
|
frame_entry_destroy (NULL, frame_entry, true); |
1227 |
|
frame_entry_free (frame_entry); |
1228 |
|
return 0; |
1229 |
} |
} |