1 |
/* Slab allocator. |
/* Slab allocator. |
2 |
Copyright 1999, 2000, 2001 Johan Rydberg, jrydberg@opencores.org. |
Copyright 1999, 2000, 2001, 2002 Johan Rydberg, jrydberg@rtmk.org. |
3 |
|
|
4 |
This program is free software; you can redistribute it and/or modify |
This program is free software; you can redistribute it and/or modify |
5 |
it under the terms of the GNU General Public License as published by |
it under the terms of the GNU General Public License as published by |
15 |
along with this program; if not, write to the Free Software |
along with this program; if not, write to the Free Software |
16 |
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ |
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ |
17 |
|
|
18 |
|
/* I'm not very proud code. I should really rewrite it some day soon. */ |
19 |
|
|
20 |
/* ??? remaining to do is to implement shrinking of caches. */ |
/* ??? remaining to do is to implement shrinking of caches. */ |
21 |
/* ??? remaining to do is to fix the large-object stuff. */ |
/* ??? remaining to do is to fix the large-object stuff. */ |
22 |
/* ??? remaining to do is to clean up the code. */ |
/* ??? remaining to do is to clean up the code. */ |
28 |
#include "vm-slab.h" |
#include "vm-slab.h" |
29 |
#include "libkern.h" |
#include "libkern.h" |
30 |
#include "trace.h" |
#include "trace.h" |
31 |
|
#include "vm-map.h" |
32 |
|
#include "vm-kmem.h" |
33 |
|
|
34 |
|
#define kmem_cache_lock(CACHE) thread_lock_write (&(CACHE)->lock) |
35 |
|
#define kmem_cache_unlock(CACHE) thread_lock_unlock (&(CACHE)->lock) |
36 |
|
|
37 |
/* When are are booting the system we allcoate N number of pages |
/* When are are booting the system we allcoate N number of pages |
38 |
so that we can bootstrap the kernel allocator. |
so that we can bootstrap the kernel allocator. |
41 |
get on its feet. ??? may have to increate N pages. */ |
get on its feet. ??? may have to increate N pages. */ |
42 |
|
|
43 |
vm_offset_t kmem_cache_bootstrap_data = 0; |
vm_offset_t kmem_cache_bootstrap_data = 0; |
44 |
vm_size_t kmem_cache_bootstrap_size = 50 * VM_PAGE_SIZE; |
vm_size_t kmem_cache_bootstrap_size = 60 * VM_PAGE_SIZE; |
45 |
|
|
46 |
/* This points to the function that we uses to allocate memory when |
/* This points to the function that we uses to allocate memory when |
47 |
we're increasing the size of a cache. */ |
we're increasing the size of a cache. */ |
57 |
static void * |
static void * |
58 |
allocate_one_page (void) |
allocate_one_page (void) |
59 |
{ |
{ |
60 |
|
vm_offset_t addr; |
61 |
|
|
62 |
if (kmem_cache_bootstrap_size) |
if (kmem_cache_bootstrap_size) |
63 |
{ |
{ |
64 |
vm_offset_t return_address = kmem_cache_bootstrap_data; |
vm_offset_t return_address = kmem_cache_bootstrap_data; |
69 |
return (void *) return_address; |
return (void *) return_address; |
70 |
} |
} |
71 |
|
|
72 |
panic ("out of memory!"); |
addr = kmem_alloc_wired (VM_MAP_KERNEL (), VM_PAGE_SIZE); |
73 |
|
if (! addr) |
74 |
return (*kmem_allocate_fn) (VM_PAGE_SIZE); |
panic ("out of memory!"); |
75 |
|
return (void *) addr; |
76 |
} |
} |
77 |
|
|
78 |
static void * |
static void * |
79 |
allocate_n_pages (int n) |
allocate_n_pages (int n) |
80 |
{ |
{ |
81 |
void *result; |
vm_offset_t addr; |
82 |
|
|
83 |
if (kmem_cache_bootstrap_size >= (n * VM_PAGE_SIZE)) |
if (kmem_cache_bootstrap_size >= (n * VM_PAGE_SIZE)) |
84 |
{ |
{ |
90 |
return (void *) return_address; |
return (void *) return_address; |
91 |
} |
} |
92 |
|
|
93 |
panic ("out of memory"); |
addr = kmem_alloc_wired (VM_MAP_KERNEL (), n * VM_PAGE_SIZE); |
94 |
|
if (! addr) |
95 |
result = (*kmem_allocate_fn) (n * VM_PAGE_SIZE); |
panic ("out of memory!"); |
96 |
return result; |
return (void *) addr; |
97 |
} |
} |
98 |
|
|
99 |
/* Align to word alignment. ??? only 32-bit. */ |
/* Align to word alignment. ??? only 32-bit. */ |
178 |
cache_cache_static.name = "cache cache"; |
cache_cache_static.name = "cache cache"; |
179 |
cache_cache_static.size = ALIGN_TO_WORD (sizeof (struct kmem_cache)); |
cache_cache_static.size = ALIGN_TO_WORD (sizeof (struct kmem_cache)); |
180 |
cache_cache_static.magic = KMEM_CACHE_MAGIC; |
cache_cache_static.magic = KMEM_CACHE_MAGIC; |
181 |
spin_lock_init (&cache_cache_static.lock); |
thread_lock_init (&cache_cache_static.lock, 1, 1); |
182 |
|
|
183 |
cache_cache_static.color_offset = 0; |
cache_cache_static.color_offset = 0; |
184 |
cache_cache_static.color_range |
cache_cache_static.color_range |
282 |
if (cache->flags & CACHE_NEVER_EMPTY_FLAG |
if (cache->flags & CACHE_NEVER_EMPTY_FLAG |
283 |
&& cache->flags & CACHE_GROWING_FLAG) |
&& cache->flags & CACHE_GROWING_FLAG) |
284 |
{ |
{ |
285 |
trace_printf ("growing thingie"); |
trace_printf ("growing thingie: %s", cache->name); |
286 |
return 0; |
return 0; |
287 |
} |
} |
288 |
|
|
347 |
|
|
348 |
if (cache->magic != KMEM_CACHE_MAGIC) |
if (cache->magic != KMEM_CACHE_MAGIC) |
349 |
trace_printf ("cache %p magic %x %s", cache, cache->magic, cache->name); |
trace_printf ("cache %p magic %x %s", cache, cache->magic, cache->name); |
|
|
|
350 |
assert (cache->magic == KMEM_CACHE_MAGIC); |
assert (cache->magic == KMEM_CACHE_MAGIC); |
351 |
|
|
352 |
|
if (cache->flags & CACHE_NEVER_EMPTY_FLAG |
353 |
|
&& cache->flags & CACHE_GROWING_FLAG) |
354 |
|
return 0; |
355 |
|
|
356 |
|
kmem_cache_lock (cache); |
357 |
|
|
358 |
|
force_alloc: |
359 |
if (! cache->freecnt) |
if (! cache->freecnt) |
360 |
{ |
{ |
361 |
int err = kmem_cache_grow (cache); |
int err = kmem_cache_grow (cache); |
363 |
if (err) |
if (err) |
364 |
{ |
{ |
365 |
trace_printf ("kmem_cache_alloc: could not grow cache: %d", err); |
trace_printf ("kmem_cache_alloc: could not grow cache: %d", err); |
366 |
|
kmem_cache_unlock (cache); |
367 |
return 0; |
return 0; |
368 |
} |
} |
369 |
} |
} |
374 |
if (err) |
if (err) |
375 |
{ |
{ |
376 |
trace_printf ("kmem_cache_alloc: could not grow cache: %d", err); |
trace_printf ("kmem_cache_alloc: could not grow cache: %d", err); |
377 |
|
kmem_cache_unlock (cache); |
378 |
return 0; |
return 0; |
379 |
} |
} |
380 |
} |
} |
391 |
/* ??? assert here. this should not happen. */ |
/* ??? assert here. this should not happen. */ |
392 |
if (queue_end (&cache->slabs, &slab->link)) |
if (queue_end (&cache->slabs, &slab->link)) |
393 |
{ |
{ |
394 |
trace_printf ("should not happen\n"); |
trace_printf ("should not happen! freecnt != 0 && no slabs"); |
395 |
return 0; |
trace_printf ("setting freecnt to 0"); |
396 |
|
cache->freecnt = 0; |
397 |
|
goto force_alloc; |
398 |
} |
} |
399 |
|
|
400 |
/* If this cache control large-objects - we create a hash entry |
/* If this cache control large-objects - we create a hash entry |
430 |
|
|
431 |
slab->refcnt++; |
slab->refcnt++; |
432 |
cache->freecnt--; |
cache->freecnt--; |
433 |
|
kmem_cache_unlock (cache); |
434 |
|
|
435 |
bufctl->u.slab = slab; |
bufctl->u.slab = slab; |
436 |
return bufctl->address; |
return bufctl->address; |
443 |
{ |
{ |
444 |
struct kmem_bufctl *bufctl; |
struct kmem_bufctl *bufctl; |
445 |
|
|
446 |
|
kmem_cache_lock (cache); |
447 |
|
|
448 |
if (LARGE_OBJECT_CACHE_P (cache)) |
if (LARGE_OBJECT_CACHE_P (cache)) |
449 |
{ |
{ |
450 |
struct kmem_hash_entry *entry, **prevp; |
struct kmem_hash_entry *entry, **prevp; |
456 |
{ |
{ |
457 |
trace_printf ("bogus free cache %s addr %p", |
trace_printf ("bogus free cache %s addr %p", |
458 |
cache->name, address); |
cache->name, address); |
459 |
|
kmem_cache_unlock (cache); |
460 |
return; |
return; |
461 |
} |
} |
462 |
|
|
472 |
{ |
{ |
473 |
trace_printf ("bogus free (2) cache %s addr %p", |
trace_printf ("bogus free (2) cache %s addr %p", |
474 |
cache->name, address); |
cache->name, address); |
475 |
|
kmem_cache_unlock (cache); |
476 |
return; |
return; |
477 |
} |
} |
478 |
|
|
490 |
|
|
491 |
/* When we get here, bufctl is valid. */ |
/* When we get here, bufctl is valid. */ |
492 |
enqueue_buf_in_slab (cache, bufctl->u.slab, bufctl); |
enqueue_buf_in_slab (cache, bufctl->u.slab, bufctl); |
493 |
|
kmem_cache_unlock (cache); |
494 |
} |
} |
495 |
|
|
496 |
|
|
512 |
cache->size = ALIGN_TO_WORD (size); |
cache->size = ALIGN_TO_WORD (size); |
513 |
cache->flags = flags; |
cache->flags = flags; |
514 |
cache->magic = KMEM_CACHE_MAGIC; |
cache->magic = KMEM_CACHE_MAGIC; |
515 |
spin_lock_init (&cache->lock); |
thread_lock_init (&cache->lock, 1, 1); |
516 |
|
|
517 |
cache->color_offset = 0; |
cache->color_offset = 0; |
518 |
cache->color_range = CACHE_COLOR_RANGE (REAL_OBJECT_SIZE (size)); |
cache->color_range = CACHE_COLOR_RANGE (REAL_OBJECT_SIZE (size)); |
519 |
|
|
520 |
|
|
521 |
if (size > BIG_THRESHOLD) |
if (size > BIG_THRESHOLD) |
522 |
{ |
{ |
523 |
int npages = 1; |
int npages = 1; |