/[hurd]/hurd-l4/physmem/frame-entry.c
ViewVC logotype

Diff of /hurd-l4/physmem/frame-entry.c

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.2 by neal, Tue Mar 8 17:44:53 2005 UTC revision 1.3 by neal, Wed Apr 6 14:07:29 2005 UTC
# Line 24  Line 24 
24  #endif  #endif
25    
26  #include <assert.h>  #include <assert.h>
27    #include <string.h>
28  #include <errno.h>  #include <errno.h>
29  #include <hurd/btree.h>  #include <hurd/btree.h>
30  #include <hurd/slab.h>  #include <hurd/slab.h>
31    
32  #include <compiler.h>  #include <compiler.h>
33    
34    #include "priv.h"
35  #include "physmem.h"  #include "physmem.h"
36    #include "zalloc.h"
37    
38    static error_t
39    frame_entry_constructor (void *hook, void *buffer)
40    {
41      struct frame_entry *frame_entry = buffer;
42    
43      frame_entry->shared_next = frame_entry;
44      frame_entry->shared_prevp = &frame_entry->shared_next;
45    
46      return 0;
47    }
48    
49  static struct hurd_slab_space frame_entry_space  static struct hurd_slab_space frame_entry_space
50    = HURD_SLAB_SPACE_INITIALIZER (struct frame_entry,    = HURD_SLAB_SPACE_INITIALIZER (struct frame_entry,
51                                   NULL, NULL,                                   NULL, NULL,
52                                   NULL, NULL,                                   frame_entry_constructor, NULL,
53                                   NULL);                                   NULL);
54    
55  static inline void  void
56  frame_entry_dump (struct container *cont)  frame_entry_dump (struct frame_entry *fe)
57  {  {
58    struct frame_entry *fe;    printf ("frame_entry: %x:%x+%x@%x, frame: %x:%x+%x, ->cow: %d, ->refs: %d, shared list: ",
59              fe, fe->region.start, fe->region.size, fe->frame_offset,
60              fe->frame, l4_address (fe->frame->memory),
61              l4_size (fe->frame->memory),
62              fe->frame->cow, fe->frame->refs);
63    
64      int shares = 0;
65      struct frame_entry *f = fe;
66      do
67        {
68          printf ("%x:%x+%x->", f, f->frame_offset, f->region.size);
69          shares ++;
70          assert (f->frame == fe->frame);
71          f = f->shared_next;
72        }
73      while (f != fe);
74      printf ("(=%d)\n", shares);
75    
76    for (fe = hurd_btree_frame_entry_first (&cont->frame_entries); fe;    int entries = 0;
77         fe = hurd_btree_frame_entry_next (fe))    for (f = fe->frame->frame_entries; f; f = f->next)
78      printf ("%x+%x@%x ", fe->region.start, fe->region.size,      entries ++;
79              l4_address (fe->frame->memory));    assert (entries == fe->frame->refs);
   printf ("\n");  
80  }  }
81    
82  struct frame_entry *  struct frame_entry *
# Line 60  frame_entry_alloc (void) Line 89  frame_entry_alloc (void)
89    if (err)    if (err)
90      return 0;      return 0;
91    
92      assert (frame_entry->shared_next == frame_entry);
93      assert (frame_entry->shared_prevp == &frame_entry->shared_next);
94    
95    return frame_entry;    return frame_entry;
96  }  }
97    
98  void  void
99  frame_entry_dealloc (struct frame_entry *frame_entry)  frame_entry_free (struct frame_entry *frame_entry)
100  {  {
101      assert (frame_entry->shared_next == frame_entry);
102    #ifndef NDEBUG
103      memset (frame_entry, 0xfe, sizeof (struct frame_entry));
104      frame_entry_constructor (0, frame_entry);
105    #endif
106    hurd_slab_dealloc (&frame_entry_space, frame_entry);    hurd_slab_dealloc (&frame_entry_space, frame_entry);
107  }  }
108    
109    /* If SHARE is non-NULL, add FRAME_ENTRY (which is not attach to any
110       share list) to SHARE's share list.  Otherwise, remove FRAME_ENTRY
111       from the share list to which it is currently attached.  */
112    static void
113    frame_entry_share_with (struct frame_entry *frame_entry,
114                            struct frame_entry *share)
115    {
116      if (share)
117        /* Add FRAME_ENTRY to SHARE's share list.  */
118        {
119          /* FRAME_ENTRY shouldn't be on a shared list.  */
120          assert (frame_entry->shared_next == frame_entry);
121    
122          frame_entry->shared_next = share;
123          frame_entry->shared_prevp = share->shared_prevp;
124          *frame_entry->shared_prevp = frame_entry;
125          share->shared_prevp = &frame_entry->shared_next;
126        }
127      else
128        /* Remove FRAME_ENTRY from any share list.  */
129        {
130          *frame_entry->shared_prevp = frame_entry->shared_next;
131          frame_entry->shared_next->shared_prevp = frame_entry->shared_prevp;
132    
133          frame_entry->shared_next = frame_entry;
134          frame_entry->shared_prevp = &frame_entry->shared_next;
135        }
136    }
137    
138  error_t  error_t
139  frame_entry_new (struct container *cont,  frame_entry_create (struct container *cont,
140                   struct frame_entry *frame_entry,                      struct frame_entry *frame_entry,
141                   uintptr_t start, size_t size)                      uintptr_t cont_addr, size_t size)
142  {  {
143    error_t err;    error_t err;
144    
145      assert (pthread_mutex_trylock (&cont->lock) == EBUSY);
146    /* Size must be a power of 2.  */    /* Size must be a power of 2.  */
147    assert (size > 0 && (size & (size - 1)) == 0);    assert (size > 0 && (size & (size - 1)) == 0);
148    
149    /* Initialize the frame_entry region.  */    frame_entry->container = cont;
150    frame_entry->region.start = start;    frame_entry->region.start = cont_addr;
151    frame_entry->region.size = size;    frame_entry->region.size = size;
152    frame_entry->frame_offset = 0;    frame_entry->frame_offset = 0;
153    
# Line 88  frame_entry_new (struct container *cont, Line 155  frame_entry_new (struct container *cont,
155    if (! frame_entry->frame)    if (! frame_entry->frame)
156      return errno;      return errno;
157    
158    err = frame_entry_attach (cont, frame_entry);    err = container_attach (cont, frame_entry);
159    if (EXPECT_FALSE (err))    if (EXPECT_FALSE (err))
160      {      {
161        debug ("Overlap: %x+%x\n", start, size);        debug ("Overlap: %x+%x\n", cont_addr, size);
162        frame_deref (frame_entry->frame);        frame_deref (frame_entry->frame);
163        return EEXIST;        return EEXIST;
164      }      }
165    
166    frame_add_user (frame_entry->frame, frame_entry);    frame_add_user (frame_entry->frame, frame_entry);
167    
168      frame_entry_share_with (frame_entry, NULL);
169    
170    return 0;    return 0;
171  }  }
172    
173  error_t  error_t
174  frame_entry_use_frame (struct container *cont,  frame_entry_copy (struct container *cont,
175                         struct frame_entry *frame_entry,                    struct frame_entry *frame_entry,
176                         uintptr_t start, size_t size,                    uintptr_t cont_addr, size_t size,
177                         struct frame *frame, size_t offset)                    struct frame_entry *source,
178                      size_t frame_offset,
179                      bool shared_memory)
180  {  {
181    error_t err;    error_t err;
182    
183      assert (pthread_mutex_trylock (&cont->lock) == EBUSY);
184      assert (pthread_mutex_trylock (&source->frame->lock) == EBUSY);
185    /* Size must be a power of 2.  */    /* Size must be a power of 2.  */
186    assert (size > 0 && (size & (size - 1)) == 0);    assert (size > 0 && (size & (size - 1)) == 0);
187    assert (frame);    assert (source->frame);
188    /* Make sure that the provided offset is valid.  */    /* Make sure that the provided offset is valid.  */
189    assert (offset >= 0 && offset <= l4_size (frame->memory) - size);    assert (frame_offset >= 0
190    /* The frame entry must refer to memory starting at a self-aligned            && frame_offset <= l4_size (source->frame->memory) - size);
191      /* The frame entry must refer to memory starting at a size aligned
192       boundary.  */       boundary.  */
193    assert ((offset & (size - 1)) == 0);    assert ((frame_offset & (size - 1)) == 0);
194    
195    /* Initialize the frame_entry region.  */    frame_entry->container = cont;
196    frame_entry->region.start = start;    frame_entry->region.start = cont_addr;
197    frame_entry->region.size = size;    frame_entry->region.size = size;
198    frame_entry->frame = frame;    frame_entry->frame = source->frame;
199    frame_entry->frame_offset = offset;    frame_entry->frame_offset = frame_offset;
200    
201    err = frame_entry_attach (cont, frame_entry);    err = container_attach (cont, frame_entry);
202    if (EXPECT_FALSE (err))    if (EXPECT_FALSE (err))
203      {      {
204        debug ("Overlap: %x+%x\n", start, size);        debug ("Overlap: %x+%x\n", cont_addr, size);
       frame_deref (frame_entry->frame);  
205        return EEXIST;        return EEXIST;
206      }      }
207    
208    frame_ref (frame);    frame_ref (source->frame);
209    frame_add_user (frame, frame_entry);    frame_add_user (source->frame, frame_entry);
210    
211      if (shared_memory)
212        /* This is a copy of the entry but the physical memory is
213           shared.  */
214        frame_entry_share_with (frame_entry, source);
215      else
216        /* Copy on write.  */
217        {
218          source->frame->cow ++;
219          frame_entry_share_with (frame_entry, NULL);
220        }
221    
222    return 0;    return 0;
223  }  }
224    
225  struct frame_entry *  error_t
226  frame_entry_find (struct container *cont, uintptr_t start, size_t size)  frame_entry_use (struct container *cont,
227                     struct frame_entry *frame_entry,
228                     uintptr_t cont_addr, size_t size,
229                     struct frame *source,
230                     size_t frame_offset)
231  {  {
232    struct region region = { start, size };    error_t err;
233    
234    return hurd_btree_frame_entry_find (&cont->frame_entries, &region);    assert (pthread_mutex_trylock (&cont->lock) == EBUSY);
235      assert (pthread_mutex_trylock (&source->lock) == EBUSY);
236      /* SIZE must be a power of 2.  */
237      assert (size > 0 && (size & (size - 1)) == 0);
238      /* FRAME_OFFSET must be a multiple of SIZE.  */
239      assert ((frame_offset & (size - 1)) == 0);
240      /* The frame entry must actually cover the FRAME.  */
241      assert (frame_offset + size <= l4_size (source->memory));
242    
243      frame_entry->container = cont;
244      frame_entry->region.start = cont_addr;
245      frame_entry->region.size = size;
246      frame_entry->frame_offset = frame_offset;
247      frame_entry->frame = source;
248    
249      err = container_attach (cont, frame_entry);
250      if (EXPECT_FALSE (err))
251        {
252          debug ("Overlap: %x+%x\n", cont_addr, size);
253          return EEXIST;
254        }
255    
256      frame_ref (frame_entry->frame);
257      frame_add_user (frame_entry->frame, frame_entry);
258    
259      frame_entry_share_with (frame_entry, 0);
260    
261      return 0;
262  }  }
263    
264  void  void
265  frame_entry_drop (struct container *cont, struct frame_entry *frame_entry)  frame_entry_destroy (struct container *cont, struct frame_entry *frame_entry,
266                         bool do_unlock)
267  {  {
268      if (cont)
269        {
270          assert (pthread_mutex_trylock (&cont->lock) == EBUSY);
271          container_detach (cont, frame_entry);
272        }
273    
274    assert (frame_entry->frame);    assert (frame_entry->frame);
275      assert (pthread_mutex_trylock (&frame_entry->frame->lock) == EBUSY);
276    
277    frame_drop_user (frame_entry->frame, frame_entry);    frame_drop_user (frame_entry->frame, frame_entry);
278    frame_entry_detach (cont, frame_entry);  
279      if (frame_entry->shared_next != frame_entry)
280        /* FRAME_ENTRY is on a share list, remove it.  */
281        frame_entry_share_with (frame_entry, NULL);
282      else
283        /* FRAME_ENTRY is not on a share list and therefore holds a COW
284           copy if there are other users of the underlying frame.  */
285        {
286          if (frame_entry->frame->frame_entries)
287            {
288              assert (frame_entry->frame->cow > 0);
289              frame_entry->frame->cow --;
290            }
291          else
292            assert (frame_entry->frame->cow == 0);
293        }
294    
295      if (do_unlock)
296        frame_deref (frame_entry->frame);
297      else
298        frame_release (frame_entry->frame);
299    }
300    
301    struct frame_entry *
302    frame_entry_find (struct container *cont, uintptr_t cont_addr, size_t size)
303    {
304      assert (pthread_mutex_trylock (&cont->lock) == EBUSY);
305    
306      struct region region = { cont_addr, size };
307      return hurd_btree_frame_entry_find (&cont->frame_entries, &region);
308  }  }
309    
310  error_t  error_t
311  frame_entry_attach (struct container *cont, struct frame_entry *frame_entry)  frame_entry_map (struct frame_entry *fe,
312                     size_t start, size_t len, int access,
313                     uintptr_t vaddr, l4_msg_t msg,
314                     size_t *amount)
315  {  {
316    return hurd_btree_frame_entry_insert (&cont->frame_entries, frame_entry);    error_t err;
317    
318      assert (pthread_mutex_trylock (&fe->frame->lock) == EBUSY);
319      assert (start < fe->region.size);
320      assert (len <= fe->region.size);
321      assert (start + len <= fe->region.size);
322      assert ((access & ~HURD_PM_CONT_RWX) == 0);
323    
324      if (EXPECT_FALSE ((access & HURD_PM_CONT_WRITE) && fe->frame->cow))
325        /* The caller requests a mapping with write access and the
326           frame is marked COW; now's the time to do the copy.  */
327        {
328          /* If the frame has COW copies, there must be no extant
329             writable mappings.  */
330          assert (! (fe->frame->may_be_mapped & HURD_PM_CONT_WRITE));
331          /* If the frame has COW copies there has to be at least two
332             users.  */
333          assert (fe->frame->refs > 1);
334    
335          /* If this is a shared memory copy, we need to allocate a
336             frame to cover the largest frame entry.  */
337          struct frame_entry *base = fe;
338    
339          if (! (base->frame_offset == 0
340                 && base->region.size == l4_size (fe->frame->memory)))
341            for (struct frame_entry *s = fe->shared_next;
342                 s != fe; s = s->shared_next)
343              {
344                assert (s->frame == fe->frame);
345    
346                /* Does S contain BASE?  */
347                if (s->frame_offset <= base->frame_offset
348                    && (base->frame_offset + base->region.size
349                        <= s->frame_offset + s->region.size))
350                  {
351                    base = s;
352    
353                    if (base->frame_offset == 0
354                        && base->region.size == l4_size (fe->frame->memory))
355                      break;
356                  }
357              }
358    
359          struct frame *frame = frame_alloc (base->region.size);
360    
361          /* The point of this function is to get a mapping of the
362             memory.  Even if ORIGINAL_FRAME doesn't have memory
363             allocated yet (and hence no copy needs to be done), FRAME
364             will need the memory shortly.  */
365          frame_memory_bind (frame);
366    
367          /* We only have to do a memcpy if the source has memory
368             allocated.  */
369          if (l4_address (fe->frame->memory))
370            memcpy ((void *) l4_address (frame->memory),
371                    (void *) (l4_address (fe->frame->memory)
372                              + base->frame_offset),
373                    base->region.size);
374    
375          /* Change everyone using this copy of FE->FRAME to use FRAME
376             (i.e. all frame entries on FE's shared frame list).  */
377    
378          struct frame *original_frame = fe->frame;
379    
380          /* Iterate over the shared list moving all but BASE.  */
381          struct frame_entry *falsely_shared = 0;
382          struct frame_entry *next = base->shared_next;
383          while (next != base)
384            {
385              /* S will be removed from the shared list.  Get the next
386                 element now.  */
387              struct frame_entry *s = next;
388              next = s->shared_next;
389    
390              if (s->frame_offset < base->frame_offset
391                  || s->frame_offset >= base->frame_offset + base->region.size)
392                /* S is falsely sharing with FE.  This can happen
393                   when, for instance, a client A shares the middle
394                   8kb of a 16kb frame with a second client, B.
395                   (Hence B references 2 4kb frame entries.)  If A
396                   deallocates the 16kb region, B's two frame entries
397                   are still marked as shared, however, they do not
398                   actually share any physical memory.  If there is an
399                   extant COW on the physical memory then we only want
400                   to copy the memory that is actually shared (there
401                   is no need to allocate more physical memory than
402                   necessary).  */
403                {
404                  assert ((s->frame_offset + base->region.size
405                           <= base->frame_offset)
406                          || (s->frame_offset
407                              >= base->frame_offset + base->region.size));
408    
409                  /* Remove S from the shared list.  */
410                  frame_entry_share_with (s, NULL);
411    
412                  if (! falsely_shared)
413                    /* First one.  Reset.  */
414                    falsely_shared = s;
415                  else
416                    /* Add to the falsely shared list with its
417                       possible real sharers.  */
418                    frame_entry_share_with (s, falsely_shared);
419                }
420              else
421                /* Migrate S from ORIGINAL_FRAME to the copy, FRAME.
422                   (If S is BASE we migrate later.)  */
423                {
424                  frame_drop_user (original_frame, s);
425                  frame_release (original_frame);
426    
427                  frame_ref (frame);
428                  frame_add_user (frame, s);
429                  s->frame = frame;
430                  s->frame_offset -= base->frame_offset;
431    
432                  assert (s->frame_offset >= 0);
433                  assert (s->frame_offset + s->region.size
434                          <= l4_size (s->frame->memory));
435                }
436            }
437    
438          /* Of those on the shared list, only BASE still references
439             the original frame.  Removing BASE may case some of
440             ORIGINAL_FRAME to now be unreferenced.  Hence, we cannot
441             simply move it as we did with the others.  */
442          uintptr_t bstart = base->region.start;
443          size_t bsize = base->region.size;
444          bool fe_is_base = (base == fe);
445    
446          frame_entry_share_with (base, NULL);
447    
448          /* Reallocate the frame entry.  */
449          err = frame_entry_deallocate (base->container, base, bstart, bsize);
450          assert_perror (err);
451          base = frame_entry_alloc ();
452          err = frame_entry_use (base->container, base, bstart, bsize, frame, 0);
453          assert_perror (err);
454    
455          /* All the frame entries using FRAME are on the shared
456             list.  */
457          frame_entry_share_with (base, frame->frame_entries);
458    
459          if (fe_is_base)
460            fe = base;
461    
462          /* We managed to pick up an extra reference to FRAME in the
463             loop (we already had one before we entered the loop and
464             we added a reference for each entry which shares the
465             frame including FE).  Drop it now.  */
466          frame_release (frame);
467        }
468      else
469        /* Allocate the memory (if needed).  */
470        frame_memory_bind (fe->frame);
471    
472      fe->frame->may_be_mapped |= access;
473    
474      /* Get the start of the mem.  */
475      l4_word_t mem = l4_address (fe->frame->memory) + fe->frame_offset + start;
476    
477      l4_fpage_t fpages[(L4_NUM_MRS - (l4_untyped_words (l4_msg_msg_tag (msg))
478                                       + l4_untyped_words (l4_msg_msg_tag (msg))))
479                        / 2];
480      int nr_fpages = l4_fpage_xspan (mem, mem + len - 1, vaddr,
481                                      fpages, sizeof (fpages) / sizeof (*fpages));
482    
483      for (int i = 0; i < nr_fpages; i ++)
484        {
485          /* Set the desired permissions.  */
486          l4_set_rights (&fpages[i], access);
487    
488          /* Add the map item to the message.  */
489          l4_msg_append_map_item (msg, l4_map_item (fpages[i], vaddr));
490    
491          vaddr += l4_size (fpages[i]);
492        }
493    
494      if (amount)
495        *amount = l4_address (fpages[nr_fpages - 1])
496          + l4_size (fpages[nr_fpages - 1]) - mem;
497    
498      return
499        l4_address (fpages[nr_fpages]) + l4_size (fpages[nr_fpages]) - mem < len
500        ? ENOSPC : 0;
501  }  }
502    
503  void  error_t
504  frame_entry_detach (struct container *cont, struct frame_entry *frame_entry)  frame_entry_deallocate (struct container *cont,
505                            struct frame_entry *frame_entry,
506                            const uintptr_t cont_addr,
507                            const size_t len)
508  {  {
509    assert (hurd_btree_frame_entry_find (&cont->frame_entries,    const uintptr_t cont_start
510                                         &frame_entry->region));      = cont_addr - frame_entry->region.start + frame_entry->frame_offset;
511    hurd_btree_frame_entry_detach (&cont->frame_entries, frame_entry);    struct frame *frame;
512    
513      /* FE currently uses FE->FRAME.  Make the TODO bytes of memory FE
514         references starting at byte SKIP (relative to the base of FE) use
515         the array of FRAMES.
516    
517         Returns the last frame used (i.e. the one which contains byte
518         SKIP+TODO).  */
519      struct frame **move (struct frame_entry *fe, size_t skip, size_t todo,
520                           l4_fpage_t *fpages, struct frame **frames)
521        {
522          error_t err;
523    
524          assert (todo > 0);
525          /* The first byte of FE may not by the first byte of
526             *FRAMES.  For instance, FRAMES may be 8kb long
527             but FE references only the second 4kb.  */
528          assert (skip < fe->region.size);
529          assert (fe->frame_offset + skip >= l4_address (*fpages));
530          uintptr_t frame_offset = fe->frame_offset + skip - l4_address (*fpages);
531    
532          /* The container address of the first byte.  */
533          uintptr_t addr = fe->region.start + skip;
534    
535          for (; todo > 0; frames ++, fpages ++)
536            {
537              size_t count = l4_size (*fpages) - frame_offset;
538              if (count > todo)
539                count = todo;
540    
541              l4_fpage_t subfpages[L4_FPAGE_SPAN_MAX];
542              int n = l4_fpage_span (frame_offset,
543                                     frame_offset + count - 1,
544                                     subfpages);
545    
546              for (int i = 0; i < n; i ++)
547                {
548                  struct frame_entry *n = frame_entry_alloc ();
549    
550                  err = frame_entry_use (cont, n, addr,
551                                         l4_size (subfpages[i]),
552                                         *frames, l4_address (subfpages[i]));
553                  assert_perror (err);
554    
555                  /* Although they only falsely share FE->FRAME (which is
556                     perfectly correct), the new frame entries are on a
557                     share list to reduce the number of gratuitous COWs:
558                     there is one COW for each shared copy; if there is
559                     only a single shared copy then no COWs need to be
560                     performed.  */
561                  frame_entry_share_with (n, fe);
562    
563                  frame_offset = 0;
564                  addr += l4_size (subfpages[i]);
565                  todo -= l4_size (subfpages[i]);
566                }
567            }
568    
569          frames --;
570    
571          return frames;
572        }
573    
574      /* Migrate the frame entries using FRAME between byte START and END
575         to new frame structures which use the same physical memory.  */
576      void migrate (uintptr_t start, uintptr_t end)
577        {
578          assert (start < end);
579          /* START must come before the end of the deallocation zone.  */
580          assert (start <= cont_start + len);
581          /* END must come after the start of the deallocation zone.  */
582          assert (end + 1 >= cont_start);
583    
584          /* FRAME_ENTRY must cover all of the underlying frame.  */
585          assert (frame_entry->frame_offset == 0);
586          assert (frame_entry->region.size == l4_size (frame->memory));
587    
588          /* Allocate new frames and point them to their respective pieces
589             of FRAME.  */
590          l4_fpage_t fpages[L4_FPAGE_SPAN_MAX];
591          int nr_fpages = l4_fpage_span (start, end, fpages);
592          struct frame **frames = alloca (sizeof (struct frame *) * nr_fpages);
593    
594          for (int i = 0; i < nr_fpages; i ++)
595            {
596              frames[i] = frame_alloc (l4_size (fpages[i]));
597              frames[i]->memory = l4_fpage (l4_address (frame->memory)
598                                            + l4_address (fpages[i]),
599                                            l4_size (fpages[i]));
600              frames[i]->may_be_mapped = frame->may_be_mapped;
601            }
602    
603          /* Move the parts of FRAME_ENTRY which are not going to be
604             deallocated to the new frames.  */
605    
606          int i = 0;
607    
608          /* If START is before CONT_START then we need to relocate some
609             of FRAME_ENTRY.
610    
611                    START END
612                    v     v
613                 [  |  |  |  |  |  ]  <- FRAME_ENTRY
614                       ^     ^
615                       |     CONT_START+LEN
616                       CONT_START
617                     \/
618                    keep
619    
620              (END can be before CONT_START.)
621           */
622          if (start < cont_start)
623            {
624              size_t todo;
625              if (end < cont_start)
626                todo = end - start + 1;
627              else
628                todo = cont_start - start;
629                
630              /* Find the frame which contains byte START.  */
631              for (; i < nr_fpages; i ++)
632                if (start < l4_address (fpages[i]) + l4_size (fpages[i]))
633                  break;
634    
635              struct frame **l
636                = move (frame_entry, start, todo, &fpages[i], &frames[i]);
637    
638              int last = i + ((void *) l - (void *) &frames[i]) / sizeof (*l);
639              assert (last < nr_fpages);
640              for (int j = i; j <= last; j ++)
641                frames[j]->cow ++;
642              i = last;
643            }
644    
645          /* If CONT_START+LEN is before END then we need to relocate some
646             of FRAME_ENTRY.
647    
648                          START    END
649                          v        v
650                 [  |  |  |  |  |  ]  <- FRAME_ENTRY
651                       ^     ^
652                       |     CONT_START+LEN
653                       CONT_START
654                              \    /
655                               keep  
656    
657              (START can be after CONT_START+LEN.)
658           */
659          if (cont_start + len < end)
660            {
661              size_t skip;
662    
663              if (start < cont_start + len)
664                skip = cont_start + len;
665              else
666                skip = start;
667    
668              /* Find the frame which contains the first byte referenced
669                 by FRAME_ENTRY after the region to deallocate.  */
670              for (; i < nr_fpages; i ++)
671                if (skip >= l4_address (fpages[i]))
672                  break;
673    
674              struct frame **l
675                = move (frame_entry, skip, end - skip + 1, &fpages[i], &frames[i]);
676    
677              int last = i + ((void *) l - (void *) &frames[i]) / sizeof (*l);
678              assert (last < nr_fpages);
679              for (int j = i; j <= last; j ++)
680                frames[j]->cow ++;
681            }
682    
683          /* Change the rest of the frame entries referencing FRAME
684             between START and END to reference the respective frames in
685             FRAMES.  */
686    
687          struct frame_entry *n = frame->frame_entries;
688          while (n)
689            {
690              struct frame_entry *fe = n;
691              n = fe->next;
692    
693              /* Any frame entries connected to FRAME_ENTRY should
694                 reference the same frame.  */
695              assert (frame == fe->frame);
696    
697              /* Any frames entries referencing memory before START should
698                 have been relocated from FRAME_ENTRY in a prior pass
699                 (except for FRAME_ENTRY, of course).  */
700              assert (fe == frame_entry || fe->frame_offset >= start);
701    
702              if (fe == frame_entry)
703                continue;
704              else if (fe->frame_offset < end)
705                {
706                  /* END is either the end of the frame or the memory
707                     following END is completely unreferenced (and to be
708                     deallocated).  Hence any frame entry which starts
709                     before END ends before it as well.  */
710                  assert (fe->frame_offset + fe->region.size - 1 <= end);
711    
712                  void adjust (struct frame_entry *fe, int i)
713                    {
714                      assert (fe->frame == frame);
715                      /* FE fits entirely in FRAMES[I].  */
716                      assert (l4_address (fpages[i]) <= fe->frame_offset
717                              && (fe->frame_offset + fe->region.size
718                                  <= (l4_address (fpages[i])
719                                      + l4_size (fpages[i]))));
720    
721                      /* Adjust the frame offset.  */
722                      fe->frame_offset -= l4_address (fpages[i]);
723    
724                      /* Make sure N always points to an unprocessed frame
725                         entry on FRAME->FRAME_ENTRIES.  */
726                      if (fe == n)
727                        n = fe->next;
728    
729                      /* Move from the old frame to the new one.  */
730                      frame_drop_user (frame, fe);
731                      frame_release (frame);
732    
733                      fe->frame = frames[i];
734    
735                      frame_ref (fe->frame);
736                      frame_add_user (fe->frame, fe);
737                    }
738    
739                  /* Find the frame which holds the start of the
740                     memory E references.  */
741                  int i;
742                  for (i = 0; i < nr_fpages; i ++)
743                    if (fe->frame_offset >= l4_address (fpages[i])
744                        && (fe->frame_offset
745                            < l4_address (fpages[i]) + l4_size (fpages[i])))
746                      break;
747    
748                  adjust (fe, i);
749    
750                  if (fe->shared_next == fe)
751                    /* FE was not on a shared list.  Remove its COW from
752                       FRAME.  Add a COW to the new frame.  */
753                    {
754                      assert (frame->cow > 0);
755                      frame->cow --;
756                      fe->frame->cow ++;
757                    }
758                  else
759                    /* FE was on a shared list.  Fix it up.  */
760                    {
761                      bool shares_old_cow = false;
762                      bool shares_new_cow_with_frame_entry = false;
763    
764                      /* We need to use FE as an anchor hence we attach to
765                         here and then at the end detach FE and attach it
766                         to the resulting list.  */
767                      struct frame_entry *shared_list = 0;
768    
769                      struct frame_entry *sn = fe->shared_next;
770                      while (sn != fe)
771                        {
772                          struct frame_entry *s = sn;
773                          sn = s->shared_next;
774    
775                          if (s == frame_entry)
776                            shares_old_cow = true;
777                          else if (s->frame != frame)
778                            /* S was already relocated which means that it
779                               was split off from FRAME_ENTRY which means
780                               that the cow was already counted.  */
781                            {
782                              shares_old_cow = true;
783    
784                              if (s->frame == fe->frame)
785                                {
786                                  shares_new_cow_with_frame_entry = true;
787    
788                                  frame_entry_share_with (s, NULL);
789                                  if (! shared_list)
790                                    shared_list = s;
791                                  else
792                                    frame_entry_share_with (s, shared_list);
793                                }
794                            }
795                          else if (l4_address (fpages[i]) <= s->frame_offset
796                                   && (s->frame_offset < l4_address (fpages[i])
797                                       + l4_size (fpages[i])))
798                            /* S and FE continue to share a copy of the
799                               underlying frame (i.e. no false
800                               sharing).  */
801                            {
802                              adjust (s, i);
803    
804                              frame_entry_share_with (s, NULL);
805                              if (! shared_list)
806                                shared_list = s;
807                              else
808                                frame_entry_share_with (s, shared_list);
809                            }
810                          else
811                            shares_old_cow = true;
812                        }
813    
814                      frame_entry_share_with (fe, 0);
815                      if (shared_list)
816                        frame_entry_share_with (fe, shared_list);
817    
818                      if (! shares_old_cow)
819                        /* There was no false sharing, i.e. there are no
820                           frame entries still using this copy of the old
821                           frame.  */
822                        {
823                          assert (frame->cow > 0);
824                          frame->cow --;
825                        }
826    
827                      if (! shares_new_cow_with_frame_entry)
828                        /* Unless we share our copy of the underlying
829                           frame with FRAME_ENTRY, we need to add a
830                           COW.  */
831                        fe->frame->cow ++;
832                    }
833                }
834              else
835                assert (fe->frame_offset > end);
836            }
837    
838          /* Any new frame entries created from FRAME_ENTRY are put on its
839             shared list.  If they were not picked up above (because
840             FRAME_ENTRY is on a share list) then some of them may not
841             have been properly fixed up.  */
842          if (frame_entry->shared_next != frame_entry)
843            {
844              struct frame_entry *n = frame_entry->shared_next;
845              while (n != frame_entry)
846                {
847                  struct frame_entry *fe = n;
848                  n = fe->shared_next;
849    
850                  if (fe->frame != frame)
851                    /* This is a new frame entry.  */
852                    {
853                      assert (l4_address (frame->memory)
854                              <= l4_address (fe->frame->memory));
855                      assert (l4_address (fe->frame->memory)
856                              <= (l4_address (frame->memory)
857                                  + l4_size (frame->memory)));
858    
859                      struct frame_entry *shared_list = 0;
860                      struct frame_entry *m = fe->shared_next;
861                      while (m != fe)
862                        {
863                          struct frame_entry *b = m;
864                          m = m->shared_next;
865    
866                          if (fe->frame == b->frame)
867                            {
868                              if (b == n)
869                                n = n->shared_next;
870    
871                              frame_entry_share_with (b, NULL);
872                              if (! shared_list)
873                                shared_list = b;
874                              else
875                                frame_entry_share_with (b, shared_list);
876                            }
877                        }
878    
879                      frame_entry_share_with (fe, 0);
880                      if (shared_list)
881                        frame_entry_share_with (fe, shared_list);
882                    }
883                }
884            }
885    
886          /* Tidy up the new frames.  */
887          for (i = 0; i < nr_fpages; i ++)
888            {
889              /* Each frame should have picked up at least one frame
890                 entry.  */
891              assert (frames[i]->frame_entries);
892    
893              /* Each user of FRAMES[i] added a cow.  That is one too
894                 many.  Remove it now.  */
895              assert (frames[i]->cow > 0);
896              frames[i]->cow --;
897              if (frames[i]->cow > 0
898                  && (frames[i]->may_be_mapped & L4_FPAGE_WRITABLE))
899                {
900                  l4_unmap_fpage (l4_fpage_add_rights (frames[i]->memory,
901                                                       L4_FPAGE_WRITABLE));
902                  frames[i]->may_be_mapped
903                    &= L4_FPAGE_EXECUTABLE|L4_FPAGE_READABLE;
904                }
905    
906              /* A new frame starts life with a single reference (even
907                 though no frame entries use it).  We drop that extra one
908                 now.  */
909              assert (frames[i]->refs > 1);
910              frame_deref (frames[i]);
911            }
912        }
913    
914      assert (pthread_mutex_trylock (&cont->lock) == EBUSY);
915      /* Assert that the region to deallocate falls completely within
916         FRAME_ENTRY.  */
917      assert (cont_addr >= frame_entry->region.start
918              && (cont_addr + len
919                  <= frame_entry->region.start + frame_entry->region.size));
920      /* Assert that CONT_ADDR refers to memory which starts on a
921         multiple of the base page size.  */
922      assert ((cont_start & (L4_MIN_PAGE_SIZE - 1)) == 0);
923      /* And that LEN is a multiple of the base page size.  */
924      assert ((len & (L4_MIN_PAGE_SIZE - 1)) == 0);
925    
926      frame = frame_entry->frame;
927      assert (pthread_mutex_trylock (&frame->lock) == EBUSY);
928      assert (frame->frame_entries);
929    
930    #if 0
931      printf ("%s (cont:%x, fe: %x, dzone:%x+%x); ",
932              __FUNCTION__, cont, frame_entry, cont_start, len);
933      frame_entry_dump (frame_entry);
934    #endif
935    
936      /* Before we do anything else, we need to make sure that any
937         mappings via FRAME_ENTRY are removed: most importantly, if we
938         zfree any memory and then reallocate (either internally or by
939         another process) before it is unmapped, any extant mappers may
940         have the opportunity to see (or modify) it; but also, any
941         mappings made via FRAME_ENTRY of the region to deallocate must
942         (eventually) be invalidated.  Unfortunately, this means
943         invalidating all mappings of FRAME.  */
944      if (frame->may_be_mapped)
945        {
946          l4_fpage_t fpage = l4_fpage (l4_address (frame->memory)
947                                       + frame_entry->frame_offset,
948                                       frame_entry->region.size);
949          l4_unmap_fpage (l4_fpage_add_rights (fpage, frame->may_be_mapped));
950    
951          /* If we unmapped the whole frame then we can clear
952             FRAME->MAY_BE_MAPPED.  */
953          if (frame_entry->frame_offset == 0
954              && frame_entry->region.size == l4_size (frame->memory))
955            frame->may_be_mapped = 0;
956        }
957    
958      /* Detach FRAME_ENTRY from its container: frame entries in the same
959         container cannot overlap and we are going to replace the parts of
960         FRAME_ENTRY with a set of smaller frame entries covering the
961         physical memory which will not be deallocated.  */
962      container_detach (cont, frame_entry);
963    
964    
965      if (! frame->frame_entries->next)
966        /* FRAME_ENTRY is the only frame entry using FRAME.  */
967        {
968          /* Make sure it is using the entire frame.  */
969          assert (frame_entry->frame_offset == 0);
970          assert (frame_entry->region.size == l4_size (frame->memory));
971    
972          if (cont_start > 0)
973            migrate (0, cont_start - 1);
974          if (cont_start + len < l4_size (frame->memory))
975            migrate (cont_start + len, l4_size (frame->memory) - 1);
976    
977          assert (frame->refs == 1);
978    
979          /* If some of the frame entry was migrated, we manually free any
980             physical memory.  */
981          if (cont_start > 0 || cont_start + len < l4_size (frame->memory))
982            {
983              l4_fpage_t fpages[L4_FPAGE_SPAN_MAX];
984              int nr_fpages = l4_fpage_span (cont_start,
985                                             cont_start + len - 1, fpages);
986    
987              for (int i = 0; i < nr_fpages; i ++)
988                {
989    #ifndef NDEBUG
990                  memset ((void *) l4_address (frame->memory)
991                          + l4_address (fpages[i]),
992                          0xde, l4_size (fpages[i]));
993    #endif
994                  zfree (l4_address (frame->memory) + l4_address (fpages[i]),
995                         l4_size (fpages[i]));
996                }
997    
998              frame->may_be_mapped = 0;
999              frame->memory = l4_fpage (0, l4_size (frame->memory));
1000            }
1001    
1002          frame_entry_destroy (NULL, frame_entry, true);
1003          frame_entry_free (frame_entry);
1004          return 0;
1005        }
1006    
1007      if (frame_entry->frame_offset > 0
1008          || frame_entry->region.size < l4_size (frame->memory))
1009        /* FRAME_ENTRY does not cover all of the underlying frame.  By
1010           definition, some other frame entry must.  As such, all we have
1011           to do is fix up the parts of FRAME_ENTRY which will not be
1012           deallocated and then drop it.  */
1013        {
1014    #ifndef NDEBUG
1015          /* Assert that a frame entry covers all of FE.  */
1016          struct frame_entry *fe;
1017          for (fe = frame->frame_entries; fe; fe = fe->next)
1018            if (fe->frame_offset == 0
1019                && fe->region.size == l4_size (frame->memory))
1020              break;
1021          assert (fe);
1022    #endif
1023    
1024          l4_fpage_t fpage = l4_fpage (0, l4_size (fe->frame->memory));
1025          struct frame **f;
1026    
1027          if (frame_entry->frame_offset < cont_start)
1028            {
1029              f = move (frame_entry, 0, cont_start - frame_entry->frame_offset,
1030                        &fpage, &frame_entry->frame);
1031              assert (f == &frame_entry->frame);
1032            }
1033                  
1034          if (cont_start + len
1035              < frame_entry->frame_offset + frame_entry->region.size)
1036            {
1037              f = move (frame_entry,
1038                        cont_start + len - frame_entry->frame_offset,
1039                        frame_entry->frame_offset + frame_entry->region.size
1040                        - (cont_start + len),
1041                        &fpage, &frame_entry->frame);
1042              assert (f == &frame_entry->frame);
1043            }
1044    
1045          frame_entry_destroy (NULL, frame_entry, true);
1046          frame_entry_free (frame_entry);
1047          return 0;
1048        }
1049    
1050    
1051      /* Multiple frame entries reference FRAME_ENTRY->FRAME.  Since frame
1052         entries may reference only part of the underlying frame, by
1053         releasing FRAME_ENTRY, 1) no single frame entry may now reference
1054         all of the underlying frame and 2) some of FRAME_ENTRY->FRAME may
1055         no longer be referenced and thus can be freed.  For example, a
1056         client, A, may allocate a 16kb frame.  A may give a second
1057         client, B, a copy of the middle 8kb.  (Since the start address of
1058         the 8kb area is not a multiple of the areas size, we create two
1059         frame entries: one for each 4kb region.)  If A then deallocates
1060         the 16kb region, we would like to release the unreferenced
1061         physical memory.  By inspection, we see that the first 4kb and
1062         the last 4kb are no longer used and could be freed:
1063      
1064                             A
1065                      /             \
1066                  0kb|   |   |   |   |16kb
1067                          \ / \ /
1068                          B.1 B.2
1069      
1070         This problem becomes slightly more complicated when only part of
1071         the frame entry is freed, e.g. if the client only deallocate the
1072         first 8kb of A.  Further, we must maintain the predicate that all
1073         frames have at least one frame entry which references them in
1074         their entirety.
1075    
1076         We take the following approach:
1077      
1078         Set A=(DEALLOC_START, LEN) to the region to deallocate (relative
1079         to the start of the underlying frame).  To identify unreferenced
1080         regions, iterate over the frame entries referencing the frame
1081         (excluding the one to deallocate).  If the intersection of the
1082         frame entry and A is non-empty (i.e. A contains any part of the
1083         frame entry), save the result in T.  If R is not set, set it to
1084         T.  Otherwise, if T occurs before R, set R to T.
1085      
1086         If R completely covers the region to deallocate, A, we are done.
1087         Otherwise, any memory between the start of A and the start of R
1088         is unreferenced and we can free it (doing any required frame
1089         splitting).  Move the start of A to end of R.  If the area is
1090         non-NULL, repeat from the beginning.  Otherwise, we are done.  */
1091    
1092    
1093      /* Start of the region to deallocate relative to the start of the
1094         frame which we still need to confirmed as referenced or not.  */
1095      uintptr_t dealloc_start = cont_start;
1096      size_t dealloc_len = len;
1097      
1098      /* Area within DEALLOC_START+DEALLOC_LEN which we've confirmed
1099         another frame entry references.
1100    
1101         Initially, we (consistent with the above predicate) set the start
1102         address to the end of the region to deallocate with a length of
1103         0.  Because we prefer earlier and larger regions, if this isn't
1104         changed after iterating over all of the frame entries, we know it
1105         is safe to free the region.  */
1106      uintptr_t refed_start = dealloc_start + dealloc_len;
1107      size_t refed_len = 0;
1108    
1109      /* Once we've identified a region which we can free (i.e. a region
1110         which no frame entry references), we will need to split memory
1111         which is still referenced into smaller frames.  This is
1112         complicated by the fact that there may be multiple holes.
1113         Consider:
1114    
1115                           FE
1116                     /             \
1117                    [   |   |   |   ]
1118                    0   4   8   C   F
1119                         \ /     \ /
1120                          A       B
1121    
1122         If all of frame entry FE is freed, frame entry A and B still
1123         reference two 4k segments of the frame.  We can free from the 4k
1124         regions starting at 0k and 8k.
1125        
1126         After each iteration, during which we've identified a region
1127         which is referenced, we free the memory between PROCESSED and
1128         REFED_START and relocate the frame entries between
1129         REFED_START+REFED_LEN.  We then set PROCESSED to
1130         REFED_START+REFED_LEN.  */
1131      uintptr_t processed = 0;
1132    
1133      for (;;)
1134        {
1135          /* Iterate over the frame entries checking to see if they
1136             reference DEALLOC_START+DEALLOC_LEN and modifying
1137             REFED_START+REFED_LEN appropriately.  */
1138          for (struct frame_entry *fe = frame->frame_entries; fe; fe = fe->next)
1139            {
1140              assert (fe->frame == frame);
1141    
1142              /* Don't consider the frame entry we are deallocating.  */
1143              if (fe == frame_entry)
1144                continue;
1145    
1146              if (fe->frame_offset + fe->region.size <= dealloc_start)
1147                /* FE ends before the region to deallocate begins.  */
1148                continue;
1149    
1150              if (fe->frame_offset >= dealloc_start + dealloc_len)
1151                /* FE starts after the region to deallocate ends.  */
1152                continue;
1153    
1154              if (fe->frame_offset < refed_start)
1155                /* FE covers at least part of the region to deallocate and
1156                   starts before what we've found so far.  */
1157                {
1158                  refed_start = fe->frame_offset;
1159                  refed_len = fe->region.size;
1160                }
1161              else if (fe->frame_offset == refed_start
1162                       && fe->region.size > refed_len)
1163                /* FE starts at REFED_START and is larger than
1164                   REFED_LEN.  */
1165                refed_len = fe->region.size;
1166            }
1167    
1168          if (processed < refed_start && processed < dealloc_start)
1169            /* PROCESSED comes before both REFED_START and DEALLOC_START.
1170               If there is memory to be freed, that memory is between
1171               DEALLOC_START and REFED_START.  On the other hand,
1172               REFED_START may come before DEALLOC_START if a frame
1173               straddles DEALLOC_START.  There is no need to gratuitously
1174               split it apart.  */
1175            migrate (processed,
1176                     refed_start < dealloc_start
1177                     ? refed_start - 1 : dealloc_start - 1);
1178    
1179          /* The area between DEALLOC_START and REFED_START is not
1180             referenced.  Free it and adjust the frame entries.  */
1181    
1182          if (dealloc_start < refed_start && l4_address (frame->memory))
1183            {
1184              l4_fpage_t fpages[L4_FPAGE_SPAN_MAX];
1185              int nr_fpages = l4_fpage_span (dealloc_start,
1186                                             refed_start - 1, fpages);
1187    
1188              for (int i = 0; i < nr_fpages; i ++)
1189                {
1190    #ifndef NDEBUG
1191                  memset ((void *) l4_address (frame->memory)
1192                          + l4_address (fpages[i]),
1193                          0xde, l4_size (fpages[i]));
1194    #endif
1195                  zfree (l4_address (frame->memory) + l4_address (fpages[i]),
1196                         l4_size (fpages[i]));
1197                }
1198            }
1199    
1200          if (refed_len > 0)
1201            migrate (refed_start, refed_start + refed_len - 1);
1202          processed = refed_start + refed_len;
1203    
1204          if (refed_start + refed_len >= dealloc_start + dealloc_len)
1205            break;
1206    
1207          dealloc_len -= refed_start + refed_len - dealloc_start;
1208          dealloc_start = refed_start + refed_len;
1209    
1210          refed_start = dealloc_start + dealloc_len;
1211          refed_len = 0;
1212        }
1213    
1214      /* Move any remaining frame entries over.  */
1215      if (processed < l4_size (frame->memory))
1216        migrate (processed, l4_size (frame->memory) - 1);
1217    
1218      /* And destroy the now redundant FRAME_ENTRY.  But don't let it
1219         deallocate the physical memory!  */
1220      frame->memory = l4_fpage (0, l4_size (frame->memory));
1221    
1222      assert (frame->refs == 1);
1223      assert (frame->frame_entries == frame_entry);
1224      assert (! frame->frame_entries->next);
1225      assert (frame_entry->shared_next == frame_entry);
1226      frame_entry_destroy (NULL, frame_entry, true);
1227      frame_entry_free (frame_entry);
1228      return 0;
1229  }  }

Legend:
Removed from v.1.2  
changed lines
  Added in v.1.3

savannah-hackers-public@gnu.org
ViewVC Help
Powered by ViewVC 1.1.26