/* * linux/mm/slab.c * Written by Mark Hemment, 1996. * (markhe@nextd.demon.co.uk) */ /* * An implementation of the Slab Allocator as described in outline in; * UNIX Internals: The New Frontiers by Uresh Vahalia * Pub: Prentice Hall ISBN 0-13-101908-2 * or with a little more detail in; * The Slab Allocator: An Object-Caching Kernel Memory Allocator * Jeff Bonwick (Sun Microsystems). * Presented at: USENIX Summer 1994 Technical Conference */ #include #include #include #include #include /* SLAB_MGMT_CHECKS - define to enable extra checks in * kmem_cache_[create|destroy|shrink]. * If you're not messing around with these funcs, then undef this. * SLAB_HIGH_PACK - define to allow 'bufctl's to be stored within objs that do not * have a state. This allows more objs per slab, but removes the * ability to sanity check an addr on release (if the addr is * within any slab, anywhere, kmem_cache_free() will accept it!). * SLAB_DEBUG_SUPPORT - when defined, kmem_cache_create() will honour; SLAB_DEBUG_FREE, * SLAB_DEBUG_INITIAL and SLAB_RED_ZONE. */ #define SLAB_MGMT_CHECKS #undef SLAB_HIGH_PACK #define SLAB_DEBUG_SUPPORT /* undef this when your cache is stable */ #define BYTES_PER_WORD sizeof(void *) /* legal flag mask for kmem_cache_create() */ #if defined(SLAB_DEBUG_SUPPORT) #define SLAB_C_MASK (SLAB_DEBUG_FREE|SLAB_DEBUG_INITIAL|SLAB_HWCACHE_ALIGN|SLAB_RED_ZONE) #else #define SLAB_C_MASK (SLAB_HWCACHE_ALIGN) #endif /* SLAB_DEBUG_SUPPORT */ /* Magic num for red zoning. * Placed in the first word after the end of an obj */ #define SLAB_RED_MAGIC1 0x5A2CF071UL /* when obj is active */ #define SLAB_RED_MAGIC2 0x170FC2A5UL /* when obj is inactive */ /* Used for linking objs within a slab. How much of the struct is * used, and where its placed, depends on the packing used in a cache. * Don't mess with the order! */ typedef struct kmem_bufctl_s { struct kmem_bufctl_s *buf_nextp; struct kmem_slab_s *buf_slabp; void *buf_objp; /* start of obj */ struct kmem_bufctl_s *buf_hnextp; struct kmem_bufctl_s **buf_hashp; } kmem_bufctl_t; /* different portions of the bufctl are used - so need some macros */ #define kmem_bufctl_offset(x) ((unsigned long)&((kmem_bufctl_t *)0)->x) #define kmem_bufctl_short_size (kmem_bufctl_offset(buf_objp)) #define kmem_bufctl_very_short_size (kmem_bufctl_offset(buf_slabp)) /* Slab management struct. * Manages the objs in a slab. Placed either at the end of mem allocated * for the slab, or from an internal obj cache (SLAB_CFLGS_OFF_SLAB). * Slabs are chain into a partially ordered list. The linking ptrs must * be first in the struct! * The size of the struct is important(ish); it should align well on * cache line(s) */ typedef struct kmem_slab_s { struct kmem_slab_s *s_nextp; struct kmem_slab_s *s_prevp; void *s_mem; /* addr of mem allocated for slab */ unsigned long s_jiffies; kmem_bufctl_t *s_freep; /* ptr to first inactive obj in slab */ unsigned long s_flags; unsigned long s_magic; unsigned long s_inuse; /* num of objs active in slab */ } kmem_slab_t; /* to test for end of slab chain */ #define kmem_slab_end(x) ((kmem_slab_t*)&((x)->c_firstp)) /* s_magic */ #define SLAB_MAGIC_ALLOC 0xA5C32F2BUL #define SLAB_MAGIC_UNALLOC 0xB2F23C5AUL /* s_flags */ #define SLAB_SFLGS_DMA 0x000001UL /* slab's mem can do DMA */ /* cache struct - manages a cache. * c_lastp must appear immediately after c_firstp! */ struct kmem_cache_s { kmem_slab_t *c_freep; /* first slab w. free objs */ unsigned long c_flags; unsigned long c_offset; struct kmem_bufctl_s **c_hashp; /* ptr for off-slab bufctls */ kmem_slab_t *c_firstp; /* first slab in chain */ kmem_slab_t *c_lastp; /* last slab in chain */ unsigned long c_hashbits; unsigned long c_num; /* # of objs per slab */ unsigned long c_gfporder; /* order of pgs per slab (2^n) */ unsigned long c_org_size; unsigned long c_magic; unsigned long c_inuse; /* kept at zero */ void (*c_ctor)(void *, int, unsigned long); /* constructor func */ void (*c_dtor)(void *, int, unsigned long); /* de-constructor func */ unsigned long c_align; /* alignment of objs */ unsigned long c_colour; /* cache colouring range */ unsigned long c_colour_next;/* cache colouring */ const char *c_name; struct kmem_cache_s *c_nextp; }; /* magic # for c_magic - used to detect out-of-slabs in __kmem_cache_alloc() */ #define SLAB_C_MAGIC 0x4F17A36DUL /* internal c_flags */ #define SLAB_CFLGS_OFF_SLAB 0x010000UL /* slab mgmt in own cache */ #define SLAB_CFLGS_BUFCTL 0x020000UL /* bufctls in own cache */ #define SLAB_CFLGS_RELEASED 0x040000UL /* cache is/being destroyed */ #if defined(SLAB_HIGH_PACK) #define SLAB_CFLGS_PTR_IN_OBJ 0x080000UL /* free ptr in obj */ #endif #define SLAB_OFF_SLAB(x) ((x) & SLAB_CFLGS_OFF_SLAB) #define SLAB_BUFCTL(x) ((x) & SLAB_CFLGS_BUFCTL) #define SLAB_RELEASED(x) ((x) & SLAB_CFLGS_RELEASED) #if defined(SLAB_HIGH_PACK) #define SLAB_PTR_IN_OBJ(x) ((x) & SLAB_CFLGS_PTR_IN_OBJ) #else #define SLAB_PTR_IN_OBJ(x) (0) #endif /* maximum size of an obj (in 2^order pages) */ #define SLAB_OBJ_MAX_ORDER 5 /* 32 pages */ /* maximum num of pages for a slab (avoids trying to ask for too may contigious pages) */ #define SLAB_MAX_GFP_ORDER 5 /* 32 pages */ /* the 'prefered' minimum num of objs per slab - maybe less for large objs */ #define SLAB_MIN_OBJS_PER_SLAB 4 /* if the num of objs per slab is <= SLAB_MIN_OBJS_PER_SLAB, * then the page order must be less than this before trying the next order */ #define SLAB_BREAK_GFP_ORDER 2 /* size of hash tables for caches which use off-slab bufctls (SLAB_CFLGS_BUFCTL) */ #define KMEM_HASH_SIZE 128 /* size description struct for general-caches */ typedef struct cache_sizes { unsigned long cs_size; kmem_cache_t *cs_cachep; } cache_sizes_t; static cache_sizes_t cache_sizes[] = { #if PAGE_SIZE == 4096 { 32, NULL}, #endif { 64, NULL}, { 128, NULL}, { 256, NULL}, { 512, NULL}, {1024, NULL}, {2048, NULL}, {4096, NULL}, {8192, NULL}, #if PAGE_SIZE == 8192 {16384, NULL}, #endif {0, NULL} }; /* Names for the general-caches. * Not placed into the sizes struct for a good reason; the * string ptr is not needed while searching in kmem_alloc()/ * kmem_free(), and would 'get-in-the-way' - think about it. */ static char *cache_sizes_name[] = { #if PAGE_SIZE == 4096 "cache-32", #endif "cache-64", "cache-128", "cache-256", "cache-512", "cache-1024", "cache-2048", "cache-4096", #if PAGE_SIZE == 4096 "cache-8192" #elif PAGE_SIZE == 8192 "cache-8192", "cache-16384" #else #error Your page size is not supported for the general-caches - please fix #endif }; static void kmem_hash_ctor(void *ptr, int , unsigned long); /* fwd ref */ extern kmem_cache_t cache_cache; /* fwd ref */ /* internal cache of hash objs, only used when bufctls are off-slab */ static kmem_cache_t cache_hash = { /* freep, flags */ kmem_slab_end(&cache_hash), 0, /* offset, hashp */ sizeof(kmem_bufctl_t*)*KMEM_HASH_SIZE, NULL, /* firstp, lastp */ kmem_slab_end(&cache_hash), kmem_slab_end(&cache_hash), /* hashbits, num, gfporder */ 0, 0, 0, /* org_size, magic */ sizeof(kmem_bufctl_t*)*KMEM_HASH_SIZE, SLAB_C_MAGIC, /* inuse, ctor, dtor, align */ 0, kmem_hash_ctor, NULL, L1_CACHE_BYTES, /* colour, colour_next */ 0, 0, /* name, nextp */ "hash_cache", &cache_cache }; /* internal cache of freelist mgmnt objs, only use when bufctls are off-slab */ static kmem_cache_t cache_bufctl = { /* freep, flags */ kmem_slab_end(&cache_bufctl), 0, /* offset, hashp */ sizeof(kmem_bufctl_t), NULL, /* firstp, lastp */ kmem_slab_end(&cache_bufctl), kmem_slab_end(&cache_bufctl), /* hashbits, num, gfporder */ 0, 0, 0, /* org_size, magic */ sizeof(kmem_bufctl_t), SLAB_C_MAGIC, /* inuse, ctor, dtor, align */ 0, NULL, NULL, BYTES_PER_WORD*2, /* colour, colour_next */ 0, 0, /* name, nextp */ "bufctl_cache", &cache_hash }; /* internal cache of slab mngmnt objs, only used when slab mgmt is off-slab */ static kmem_cache_t cache_slab = { /* freep, flags */ kmem_slab_end(&cache_slab), 0, /* offset, hashp */ sizeof(kmem_slab_t), NULL, /* firstp, lastp */ kmem_slab_end(&cache_slab), kmem_slab_end(&cache_slab), /* hashbits, num, gfporder */ 0, 0, 0, /* org_size, magic */ sizeof(kmem_slab_t), SLAB_C_MAGIC, /* inuse, ctor, dtor, align */ 0, NULL, NULL, L1_CACHE_BYTES, /* colour, colour_next */ 0, 0, /* name, nextp */ "slab_cache", &cache_bufctl }; /* internal cache of cache description objs */ static kmem_cache_t cache_cache = { /* freep, flags */ kmem_slab_end(&cache_cache), 0, /* offset, hashp */ sizeof(kmem_cache_t), NULL, /* firstp, lastp */ kmem_slab_end(&cache_cache), kmem_slab_end(&cache_cache), /* hashbits, num, gfporder */ 0, 0, 0, /* org_size, magic */ sizeof(kmem_cache_t), SLAB_C_MAGIC, /* inuse, ctor, dtor, align */ 0, NULL, NULL, L1_CACHE_BYTES, /* colour, colour_next */ 0, 0, /* name */ "kmem_cache", /* nextp */ &cache_slab }; /* constructor for hash tables */ static void kmem_hash_ctor(void *ptr, int size, unsigned long flags) { memset(ptr, 0, sizeof(kmem_bufctl_t*)*KMEM_HASH_SIZE); } /* place maintainer for reaping */ static kmem_cache_t *clock_searchp = &cache_cache; /* Init an internal cache */ static void kmem_own_cache_init(kmem_cache_t *cachep) { unsigned long size, i; if (cachep->c_inuse || cachep->c_magic != SLAB_C_MAGIC) { panic("Bad init of internal cache %s", cachep->c_name); /* NOTREACHED */ } size = cachep->c_offset + kmem_bufctl_short_size; i = size % cachep->c_align; if (i) size += (cachep->c_align-i); cachep->c_offset = size-kmem_bufctl_short_size; i = ((PAGE_SIZE<c_gfporder)-sizeof(kmem_slab_t)); cachep->c_num = i / size; /* num of objs per slab */ /* cache colouring */ cachep->c_colour = 1 + (i-(cachep->c_num*size))/cachep->c_align; cachep->c_colour_next = cachep->c_colour; } /* Initialisation - setup all internal caches */ long kmem_cache_init(long start, long end) { /* sanity */ #define kmem_cache_offset(x) ((unsigned long)&((kmem_cache_t *)0)->x) #define kmem_slab_offset(x) ((unsigned long)&((kmem_slab_t *)0)->x) if (((kmem_cache_offset(c_magic)-kmem_cache_offset(c_firstp)) != kmem_slab_offset(s_magic)) || ((kmem_cache_offset(c_inuse)-kmem_cache_offset(c_firstp)) != kmem_slab_offset(s_inuse))) { /* Offsets to the magic are incorrect, either the structures have * been incorrectly changed, or adjustments are needed for your * architecture. */ panic("kmem_cache_init(): Offsets are different - been messed with!\n"); /* NOTREACHED */ } #undef kmem_cache_offset #undef kmem_slab_offset kmem_own_cache_init(&cache_cache); kmem_own_cache_init(&cache_slab); kmem_own_cache_init(&cache_bufctl); kmem_own_cache_init(&cache_hash); return start; } /* Initialisation - setup general caches */ void kmem_cache_sizes_init(void) { unsigned long i; i = sizeof(cache_sizes)/sizeof(cache_sizes[0])-1; while (i--) cache_sizes[i].cs_cachep = kmem_cache_create(cache_sizes_name[i], cache_sizes[i].cs_size, 0, 0, NULL, NULL); } /* Interface to system's page allocator. * dma pts to non-zero if all of the mem is suitable for DMA */ static inline void * kmem_getpages(const kmem_cache_t *cachep, unsigned long flags, unsigned int *dma) { struct page *page; void *addr; addr = (void*) __get_free_pages(flags & SLAB_LEVEL_MASK, \ cachep->c_gfporder, flags & SLAB_DMA); *dma = 1<c_gfporder; if (!(flags & SLAB_DMA) && addr) { /* need to check if can dma */ page = mem_map + MAP_NR(addr); while ((*dma)--) { if (!PageDMA(page)) { *dma = 0; break; } page++; } } return addr; } /* Interface to system's page release */ static inline void kmem_freepages(kmem_cache_t *cachep, void *addr) { free_pages((unsigned long)addr, cachep->c_gfporder); } /* Hashing function - used for caches with off-slab bufctls */ static inline int kmem_hash(const kmem_cache_t *cachep, const void *objp) { return (((unsigned long)objp >> cachep->c_hashbits) & (KMEM_HASH_SIZE-1)); } /* Link bufctl into a hash table - used for caches with off-slab bufctls * - called with ints disabled */ static inline void * kmem_add_to_hash(kmem_cache_t *cachep, kmem_bufctl_t *bufp) { kmem_bufctl_t **bufpp = bufp->buf_hashp; bufp->buf_hnextp = *bufpp; return (*bufpp = bufp)->buf_objp; } /* Find bufcntl for given obj addr, and unlink. * - called with ints disabled */ static inline kmem_bufctl_t * kmem_remove_from_hash(kmem_cache_t *cachep, const void *objp) { kmem_bufctl_t *bufp; kmem_bufctl_t **bufpp = &cachep->c_hashp[kmem_hash(cachep, objp)]; for (;*bufpp; bufpp = &(*bufpp)->buf_hnextp) { if ((*bufpp)->buf_objp != objp) continue; bufp = *bufpp; *bufpp = bufp->buf_hnextp; return bufp; } return NULL; } /* Three slab chain funcs - all called with ints disabled */ static inline void kmem_slab_unlink(kmem_slab_t *slabp) { kmem_slab_t *prevp = slabp->s_prevp; kmem_slab_t *nextp = slabp->s_nextp; prevp->s_nextp = nextp; nextp->s_prevp = prevp; } static inline void kmem_slab_link_end(kmem_cache_t *cachep, kmem_slab_t *slabp) { slabp->s_nextp = kmem_slab_end(cachep); slabp->s_prevp = cachep->c_lastp; kmem_slab_end(cachep)->s_prevp = slabp; slabp->s_prevp->s_nextp = slabp; } static inline void kmem_slab_link_free(kmem_cache_t *cachep, kmem_slab_t *slabp) { kmem_slab_t *nextp = cachep->c_freep; slabp->s_nextp = nextp; cachep->c_freep = slabp; slabp->s_prevp = nextp->s_prevp; nextp->s_prevp = slabp; slabp->s_prevp->s_nextp = slabp; } /* Cal the num objs, wastage, and bytes left over for a given slab size */ static int kmem_cache_cal_waste(unsigned long gfporder, unsigned long size, unsigned long extra, unsigned long flags, unsigned long *left_over, unsigned long *num) { unsigned long wastage; wastage = PAGE_SIZE << gfporder; gfporder = 0; if (!SLAB_OFF_SLAB(flags)) gfporder = sizeof(kmem_slab_t); wastage -= gfporder; *num = wastage / size; wastage -= (*num * size); *left_over = wastage; wastage += (extra * *num); wastage += gfporder; return wastage; } /* Create a cache * Returns a ptr to the cache on success, NULL on failure. * Cannot be called within a int, but can be interrupted. * NOTE: The 'name' is assumed to be memory that is _not_ going to disappear. */ kmem_cache_t * kmem_cache_create(const char *name, unsigned long size, unsigned long align, unsigned long flags, void (*ctor)(void*, int, unsigned long), void (*dtor)(void*, int, unsigned long)) { const char *func_nm="kmem_create: "; kmem_cache_t *searchp, *cachep; unsigned long words, i; unsigned long num, left_over; /* sanity checks */ #if defined(SLAB_MGMT_CHECKS) if (!name) { printk(KERN_ERR "%sNULL ptr\n", func_nm); return NULL; } if (0 && intr_count) { printk(KERN_ERR "%sCalled during int - %s\n", func_nm, name); return NULL; } if (size < kmem_bufctl_very_short_size) { printk(KERN_WARNING "%sSize too small %lu - %s\n", func_nm, size, name); size = kmem_bufctl_very_short_size; } if (size > ((1<= size) { printk(KERN_WARNING "%sAlign weired %lu - %s\n", func_nm, align, name); align = 0; } if (dtor && !ctor) { /* Descon, but no con - doesn't make sense */ printk(KERN_ERR "%sDecon but no con - %s\n", func_nm, name); return NULL; } if ((flags & SLAB_DEBUG_INITIAL) && !ctor) { /* No constructor, but inital state check requested */ printk(KERN_WARNING "%sNo con, but init state check requested - %s\n", func_nm, name); flags &= ~SLAB_DEBUG_INITIAL; } #endif /* SLAB_MGMT_CHECKS */ /* get cache's description obj */ cachep = (kmem_cache_t *) kmem_cache_alloc(&cache_cache, SLAB_KERNEL); if (!cachep) goto opps; /* remember original size, so can be passed to a constructor or decon. * Allows the same con/decon to be used for caches of similar objs * that have a different size data buffer assoicated with them */ cachep->c_org_size = size; #if defined(SLAB_DEBUG_SUPPORT) if (flags & SLAB_RED_ZONE) size += BYTES_PER_WORD; /* word for redzone */ #endif /* SLAB_DEBUG_SUPPORT */ /* Make a guess if slab mngmnt obj and/or bufctls are 'on' or 'off' slab */ i = kmem_bufctl_short_size; if (size < (PAGE_SIZE>>3)) { /* Size is small(ish). Use format where bufctl size per * obj is low, and slab mngmnt is on-slab */ if (!ctor && !dtor && !(flags & SLAB_RED_ZONE)) { /* the objs in this cache have no state - can store * store freelist ptr within obj. (redzoning is a state) */ #if defined(SLAB_HIGH_PACK) i=0; flags |= SLAB_CFLGS_PTR_IN_OBJ; #else i = kmem_bufctl_very_short_size; #endif } } else { /* Size is large, assume best to place the slab mngmnt obj * off-slab (should allow better packing of objs) */ flags |= SLAB_CFLGS_OFF_SLAB; if (!(size & ~PAGE_MASK) || size == (PAGE_SIZE+PAGE_SIZE/2) || size == (PAGE_SIZE/2) || size == (PAGE_SIZE/4) || size == (PAGE_SIZE/8)) { /* to avoid waste the bufctls are off-slab */ flags |= SLAB_CFLGS_BUFCTL; /* get hash table for cache */ cachep->c_hashp = kmem_cache_alloc(&cache_hash, SLAB_KERNEL); if (cachep->c_hashp == NULL) { kmem_cache_free(&cache_cache, cachep); goto opps; } i = 0; cachep->c_hashbits = PAGE_SHIFT; if (size <= (PAGE_SIZE/2)) { cachep->c_hashbits--; if (size <= (PAGE_SIZE/4)) cachep->c_hashbits--; if (size <= (PAGE_SIZE/8)) cachep->c_hashbits -= 2; } } /* else slab mngmnt is off-slab, but freelist ptrs are on */ } size += i; /* Adjust the mem used for objs so they will align correctly. * Force objs to start on word boundaries, but caller may specify * h/w cache line boundaries. This 'alignment' is slightly different * to the 'align' argument. Objs may be requested to start on h/w * lines (as that is how the members of the obj have been organised), * but the 'align' may be quite high (say 64) as the first 64 bytes * are commonly accessed/modified within a loop (stops h/w line * thrashing). The 'align' is the slab colouring. */ words = BYTES_PER_WORD; if (flags & SLAB_HWCACHE_ALIGN) words = L1_CACHE_BYTES; words--; size += words; size = size & ~words; /* alignment might not be a factor of the boundary alignment - fix-up */ align += words; align = align & ~words; /* Cal size (in pages) of slabs, and the num of objs per slab. * This could be made much more intelligent. */ cachep->c_gfporder=0; do { unsigned long wastage; wastage = kmem_cache_cal_waste(cachep->c_gfporder, size, i, flags, &left_over, &num); if (!num) goto next; if (SLAB_PTR_IN_OBJ(flags)) break; if (cachep->c_gfporder == SLAB_MAX_GFP_ORDER) break; /* large num of objs is good, but v. large slabs are bad for the * VM sub-system */ if (num <= SLAB_MIN_OBJS_PER_SLAB) { if (cachep->c_gfporder < SLAB_BREAK_GFP_ORDER) goto next; } /* stop caches with small objs having a large num of pages */ if (left_over <= sizeof(kmem_slab_t)) break; if ((wastage*8) <= (PAGE_SIZE<c_gfporder)) break; /* acceptable wastage */ next: cachep->c_gfporder++; } while (1); cachep->c_num = num; /* try with requested alignment, but reduce it if that will * allow at least some alignment words */ words++; if (left_over < align) align = (left_over / words) * words; else if (!align && words <= left_over) { /* no alignment given, but space enough - give one */ align = words; if (words == BYTES_PER_WORD) { if (BYTES_PER_WORD*4 <= left_over) align += align; if (BYTES_PER_WORD*8 <= left_over) align += align; } } cachep->c_align = align; #if 0 printk("Size:%lu Orig:%lu Left:%lu Align %lu Pages:%d - %s\n", size, cachep->c_org_size, left_over, align, 1<c_gfporder, name); if (SLAB_OFF_SLAB(flags)) printk("OFF SLAB\n"); if (SLAB_BUFCTL(flags)) printk("BUFCTL PTRS\n"); #endif /* if the bufctl's are on-slab, c_offset does not inc the size of the bufctl */ if (!SLAB_BUFCTL(flags)) size -= kmem_bufctl_short_size; cachep->c_freep = kmem_slab_end(cachep); cachep->c_flags = flags; cachep->c_offset = size; cachep->c_firstp = kmem_slab_end(cachep); cachep->c_lastp = kmem_slab_end(cachep); cachep->c_ctor = ctor; cachep->c_dtor = dtor; cachep->c_magic = SLAB_C_MAGIC; cachep->c_inuse = 0; /* always zero */ cachep->c_name = name; /* simply point to the name */ cachep->c_colour = 1; if (align) cachep->c_colour += (left_over/align); cachep->c_colour_next = cachep->c_colour; /* warn on dup cache names */ searchp = &cache_cache; do { if (!strcmp(searchp->c_name, name)) { printk(KERN_WARNING "%sDup name - %s\n", func_nm, name); break; } searchp = searchp->c_nextp; } while (searchp != &cache_cache); cachep->c_nextp = cache_cache.c_nextp; cache_cache.c_nextp = cachep; return cachep; opps: printk(KERN_WARNING "%sOut of mem creating cache %s\n", func_nm, name); return NULL; } /* Destroy all the objs in a slab, and release the mem back to the system. * Before calling the slab must have been unlinked */ static void kmem_slab_destroy(kmem_cache_t *cachep, kmem_slab_t *slabp, unsigned long flags) { if (cachep->c_dtor || SLAB_BUFCTL(cachep->c_flags)) { kmem_bufctl_t *bufp = slabp->s_freep; /* for each obj in slab... */ while (bufp) { kmem_bufctl_t *freep; if (cachep->c_dtor) { void *objp = ((void*)bufp)-cachep->c_offset; if (SLAB_BUFCTL(cachep->c_flags)) objp = bufp->buf_objp; (cachep->c_dtor)(objp, cachep->c_org_size, flags); } freep = bufp; bufp = bufp->buf_nextp; if (SLAB_BUFCTL(cachep->c_flags)) kmem_cache_free(&cache_bufctl, freep); } } slabp->s_magic = SLAB_MAGIC_UNALLOC; kmem_freepages(cachep, slabp->s_mem); if (SLAB_OFF_SLAB(cachep->c_flags)) kmem_cache_free(&cache_slab, slabp); } /* Destroy (remove) a cache. * All objs in the cache should be inactive */ int kmem_cache_destroy(kmem_cache_t *cachep) { kmem_cache_t **searchp; kmem_slab_t *slabp; unsigned long save_flags; #if defined(SLAB_MGMT_CHECKS) if (!cachep) { printk(KERN_ERR "kmem_dest: NULL ptr\n"); goto err_end; } if (0 && intr_count) { printk(KERN_ERR "kmem_dest: Called during int - %s\n", cachep->c_name); err_end: return 1; } #endif /* SLAB_MGMT_CHECKS */ /* unlink the cache from the chain of active caches. * Note: the chain is never modified during an int */ searchp = &(cache_cache.c_nextp); for (;*searchp != &cache_cache; searchp = &((*searchp)->c_nextp)) { if (*searchp != cachep) continue; goto good_cache; } printk(KERN_ERR "kmem_dest: Invalid cache addr %p\n", cachep); return 1; good_cache: /* disable cache so attempts to allocated from an int can * be caught. */ save_flags(save_flags); cli(); if (cachep->c_freep != kmem_slab_end(cachep)) { restore_flags(save_flags); printk(KERN_ERR "kmem_dest: active cache - %s\n", cachep->c_name); return 2; } *searchp = cachep->c_nextp; /* remove from cache chain */ cachep->c_flags |= SLAB_CFLGS_RELEASED; cachep->c_freep = kmem_slab_end(cachep); if (cachep == clock_searchp) clock_searchp = cachep->c_nextp; restore_flags(save_flags); while ((slabp = cachep->c_firstp) != kmem_slab_end(cachep)) { kmem_slab_unlink(slabp); kmem_slab_destroy(cachep, slabp, 0); } if (SLAB_BUFCTL(cachep->c_flags)) kmem_cache_free(&cache_hash, cachep->c_hashp); kmem_cache_free(&cache_cache, cachep); return 0; } /* Shrink a cache, ie. remove _all_ inactive slabs. * Can be called when a user of a cache knows they are not going to be * needing any new objs for a while. * NOTE: This func is probably going to disappear - let me know if you * are using it! */ int kmem_cache_shrink(kmem_cache_t *cachep, int wait) { kmem_slab_t *slabp; unsigned long dtor_flags; unsigned long save_flags, num_freed=0; #if defined(SLAB_MGMT_CHECKS) if (!cachep) { printk(KERN_ERR "kmem_shrink: NULL ptr\n"); goto end; } if (0 && intr_count) { printk(KERN_ERR "kmem_shrink: Called during int - %s\n", cachep->c_name); goto end; } #endif /* SLAB_MGMT_CHECKS */ dtor_flags = 0; if (!wait) /* not allowed to wait */ dtor_flags = SLAB_DTOR_ATOMIC; save_flags(save_flags); while (0) { cli(); slabp = cachep->c_lastp; if (slabp == kmem_slab_end(cachep) || slabp->s_inuse) { restore_flags(save_flags); goto end; } kmem_slab_unlink(slabp); if (cachep->c_freep == slabp) cachep->c_freep = kmem_slab_end(cachep); restore_flags(save_flags); num_freed++; kmem_slab_destroy(cachep, slabp, dtor_flags); } end: return num_freed; } /* Search for a slab whose objs are suitable for DMA. * Note: since testing the first free slab (in __kmem_cache_alloc()), * ints must not have been enabled! */ static inline kmem_slab_t * kmem_cache_search_dma(kmem_cache_t *cachep) { kmem_slab_t *slabp = cachep->c_freep->s_nextp; for (; slabp != kmem_slab_end(cachep); slabp = slabp->s_nextp) { if (!(slabp->s_flags & SLAB_SFLGS_DMA)) continue; kmem_slab_unlink(slabp); kmem_slab_link_free(cachep, slabp); return slabp; } return NULL; } /* get the mem for a slab mgmt obj */ static inline kmem_slab_t * kmem_cache_slabmgmt(kmem_cache_t *cachep, void *objp, unsigned long local_flags, unsigned long offset) { kmem_slab_t *slabp; if (SLAB_OFF_SLAB(cachep->c_flags)) { /* slab mngmnt obj is off-slab */ if (!(slabp = kmem_cache_alloc(&cache_slab, local_flags))) return NULL; } else { /* slab mngmnt at end of slab mem */ slabp = objp + (PAGE_SIZE << cachep->c_gfporder); slabp--; if (!SLAB_PTR_IN_OBJ(cachep->c_flags)) { /* A bit of extra help for the L1 cache; try to position the slab * mgmnt struct at different offsets within the gap at the end * of a slab. This helps avoid thrashing the h/w cache lines, * that map to the end of a page, too much... */ unsigned long gap = cachep->c_offset; if (!SLAB_BUFCTL(cachep->c_flags)) gap += kmem_bufctl_short_size; gap = (PAGE_SIZE << cachep->c_gfporder)-((gap*cachep->c_num)+offset+sizeof(*slabp)); gap /= (sizeof(*slabp)/2); gap *= (sizeof(*slabp)/2); slabp = (((void*)slabp)-gap); } } slabp->s_flags = slabp->s_inuse = slabp->s_jiffies = 0; return slabp; } static inline int kmem_cache_init_objs(kmem_cache_t *cachep, kmem_slab_t *slabp, void *objp, unsigned long local_flags, unsigned long ctor_flags) { kmem_bufctl_t **bufpp = &slabp->s_freep; unsigned long num = cachep->c_num; do { if (SLAB_BUFCTL(cachep->c_flags)) { if (!(*bufpp = kmem_cache_alloc(&cache_bufctl, local_flags))) { kmem_slab_destroy(cachep, slabp, 0); return 1; } (*bufpp)->buf_objp = objp; (*bufpp)->buf_hashp = &cachep->c_hashp[kmem_hash(cachep, objp)]; } if (cachep->c_ctor) cachep->c_ctor(objp, cachep->c_org_size, ctor_flags); #if defined(SLAB_DEBUG_SUPPORT) if (cachep->c_flags & SLAB_RED_ZONE) *((unsigned long*)(objp+cachep->c_org_size)) = SLAB_RED_MAGIC1; #endif /* SLAB_DEBUG_SUPPORT */ objp += cachep->c_offset; if (!SLAB_BUFCTL(cachep->c_flags)) { *bufpp = objp; objp += kmem_bufctl_short_size; } if (!SLAB_PTR_IN_OBJ(cachep->c_flags)) (*bufpp)->buf_slabp = slabp; bufpp = &(*bufpp)->buf_nextp; } while (--num); *bufpp = NULL; return 0; } /* Grow (by 1) the number of slabs within a cache. * This is called by kmem_cache_alloc() when there are no * inactive objs left in a cache */ static void kmem_cache_grow(kmem_cache_t *cachep, unsigned long flags) { kmem_slab_t *slabp; void *objp; unsigned int offset, dma; unsigned long ctor_flags, local_flags, save_flags; if (flags & SLAB_NO_GROW) return; /* caller doesn't want us to grow */ save_flags(save_flags); /* The test for missing atomic flag is performed here, rather than * the more obvious place, simply to reduce the critical path length * in kmem_cache_alloc(). If a caller is slightly mis-behaving, * will eventually be caught here (where it matters) */ if (0 && intr_count && (flags & SLAB_LEVEL_MASK) != SLAB_ATOMIC) { static int count = 0; if (count < 8) { printk(KERN_ERR "kmem_grow: Called nonatomically from " "int - %s\n", cachep->c_name); count++; } flags &= ~SLAB_LEVEL_MASK; flags |= SLAB_ATOMIC; } local_flags = (flags & SLAB_LEVEL_MASK); ctor_flags = SLAB_CTOR_CONSTRUCTOR; if ((flags & SLAB_LEVEL_MASK) == SLAB_ATOMIC) { /* Not allowed to sleep. * Need to tell a constructor about this - it * might need to know.... */ ctor_flags |= SLAB_CTOR_ATOMIC; } slabp = NULL; /* get mem for the objs */ if (!(objp = kmem_getpages(cachep, flags, &dma))) goto opps1; /* get colour for the slab, and cal the next value */ cli(); if (!(offset = --(cachep->c_colour_next))) cachep->c_colour_next = cachep->c_colour; restore_flags(save_flags); offset *= cachep->c_align; /* get slab mgmt */ if (!(slabp = kmem_cache_slabmgmt(cachep, objp, local_flags, offset))) goto opps2; if (dma) slabp->s_flags = SLAB_SFLGS_DMA; slabp->s_mem = objp; objp += offset; /* address of first object */ /* For on-slab bufctls, c_offset is the distance between the start of * an obj and its related bufctl. For off-slab bufctls, c_offset is * the distance between objs in the slab. * Reason for bufctl at end of obj (when on slab), as opposed to the front; * if stored within the obj (has no state), and the obj is 'used' after being * freed then (normally) most activity occurs at the beginning of the obj. * By keeping the bufctl ptr away from the front, should reduce the chance of * corruption. Also, allows easier alignment of objs onto cache lines when * bufctl is not stored with the objs. * Downsize; if, while an obj is active, a write is made past its end, then the * bufctl will be corrupted :( */ if (kmem_cache_init_objs(cachep, slabp, objp, local_flags, ctor_flags)) goto no_objs; cli(); /* make slab active */ slabp->s_magic = SLAB_MAGIC_ALLOC; kmem_slab_link_end(cachep, slabp); if (cachep->c_freep == kmem_slab_end(cachep)) cachep->c_freep = slabp; restore_flags(save_flags); return; no_objs: kmem_freepages(cachep, slabp->s_mem); opps2: kmem_freepages(cachep, objp); opps1: if (slabp && SLAB_OFF_SLAB(cachep->c_flags)) kmem_cache_free(&cache_slab, slabp); /* printk("kmem_alloc: Out of mem - %s\n", cachep->c_name); */ return; } #if defined(SLAB_DEBUG_SUPPORT) /* Perform extra freeing checks. * Currently, this check is only for caches that use bufctl structures * within the slab. Those which use bufctl's from the internal cache * have a reasonable check when the address is searched for. */ static void * kmem_extra_free_checks(const kmem_cache_t *cachep, kmem_bufctl_t *search_bufp, const kmem_bufctl_t *bufp, void * objp) { if (SLAB_BUFCTL(cachep->c_flags)) goto end; /* check slab's freelist to see if this obj is there */ for (; search_bufp; search_bufp = search_bufp->buf_nextp) { if (search_bufp != bufp) continue; printk(KERN_ERR "kmem_free: Double free detected during checking " "%p - %s\n", objp, cachep->c_name); return NULL; } end: return objp; } #endif /* SLAB_DEBUG_SUPPORT */ static inline void kmem_cache_full_free(kmem_cache_t *cachep, kmem_slab_t *slabp) { if (!slabp->s_nextp->s_inuse) return; /* at correct position */ slabp->s_jiffies = jiffies; /* set release time */ if (cachep->c_freep == slabp) cachep->c_freep = slabp->s_nextp; kmem_slab_unlink(slabp); kmem_slab_link_end(cachep, slabp); return; } static inline void kmem_cache_one_free(kmem_cache_t *cachep, kmem_slab_t *slabp) { if (slabp->s_nextp->s_inuse != cachep->c_num) { cachep->c_freep = slabp; return; } kmem_slab_unlink(slabp); kmem_slab_link_free(cachep, slabp); return; } /* Returns a ptr to an obj in the given cache. * The obj is in the initial state (if there is one) */ static inline void * __kmem_cache_alloc(kmem_cache_t *cachep, unsigned long flags) { kmem_slab_t *slabp; kmem_bufctl_t *bufp; void *objp; unsigned long save_flags; /* sanity check */ if (!cachep) goto nul_ptr; save_flags(save_flags); cli(); /* get slab alloc is to come from */ slabp = cachep->c_freep; /* magic is a sanity check _and_ says if we need a new slab */ if (slabp->s_magic != SLAB_MAGIC_ALLOC) goto alloc_new_slab; try_again: /* DMA allocations are 'rare' - keep out of critical path */ if (flags & SLAB_DMA) goto search_dma; try_again_dma: slabp->s_inuse++; bufp = slabp->s_freep; slabp->s_freep = bufp->buf_nextp; if (!SLAB_BUFCTL(cachep->c_flags)) { /* Nasty - we want the 'if' to be taken in the common case */ if (slabp->s_freep) { short_finished: objp = ((void*)bufp) - cachep->c_offset; restore_flags(save_flags); #if defined(SLAB_DEBUG_SUPPORT) if (cachep->c_flags & SLAB_RED_ZONE) goto red_zone; #endif /* SLAB_DEBUG_SUPPORT */ return objp; } else { cachep->c_freep = slabp->s_nextp; goto short_finished; } } if (!slabp->s_freep) cachep->c_freep = slabp->s_nextp; /* link into hash chain */ objp = kmem_add_to_hash(cachep, bufp); restore_flags(save_flags); #if defined(SLAB_DEBUG_SUPPORT) if (!(cachep->c_flags & SLAB_RED_ZONE)) #endif /* SLAB_DEBUG_SUPPORT */ return objp; #if defined(SLAB_DEBUG_SUPPORT) red_zone: /* set alloc red-zone, and check old one */ if (xchg((unsigned long *)(objp+cachep->c_org_size), SLAB_RED_MAGIC2) != SLAB_RED_MAGIC1) printk(KERN_ERR "kmem_alloc: Bad redzone %p - %s\n", objp, cachep->c_name); return objp; #endif /* SLAB_DEBUG_SUPPORT */ search_dma: if (slabp->s_flags & SLAB_SFLGS_DMA) goto try_again_dma; /* need to search... */ if ((slabp = kmem_cache_search_dma(cachep))) goto try_again_dma; alloc_new_slab: /* Either out of slabs, or magic number corruption */ if (slabp != kmem_slab_end(cachep)) goto bad_slab; /* need a new slab */ restore_flags(save_flags); if (SLAB_RELEASED(cachep->c_flags)) { printk(KERN_ERR "kmem_alloc: destroyed cache\n"); goto end; } /* Be lazy and only check for valid flags * here (keeping it out of the critical path above) */ if (flags & ~(SLAB_DMA|SLAB_LEVEL_MASK|SLAB_NO_GROW)) { printk(KERN_ERR "kmem_alloc: Illegal flgs %lX (correcting) - %s\n", flags, cachep->c_name); flags &= (SLAB_DMA|SLAB_LEVEL_MASK|SLAB_NO_GROW); } kmem_cache_grow(cachep, flags); cli(); if ((slabp=cachep->c_freep) != kmem_slab_end(cachep)) goto try_again; restore_flags(save_flags); end: return NULL; bad_slab: /* v. serious error - maybe panic() here? */ printk(KERN_ERR "kmem_alloc: Bad slab magic (corruption) - %s\n", cachep->c_name); goto end; nul_ptr: printk(KERN_ERR "kmem_alloc: NULL ptr\n"); goto end; } /* Release an obj back to its cache. * If the obj has a constructed state, it should be * in this state _before_ it is released. */ static inline void __kmem_cache_free(kmem_cache_t *cachep, void *objp) { kmem_slab_t *slabp; kmem_bufctl_t *bufp; unsigned long save_flags; /* basic sanity checks */ if (!cachep) goto nul_cache; if (!objp) goto nul_obj; save_flags(save_flags); #if defined(SLAB_DEBUG_SUPPORT) if (cachep->c_flags & SLAB_DEBUG_INITIAL) goto init_state_check; finished_initial: #endif /* SLAB_DEBUG_SUPPORT */ if (SLAB_BUFCTL(cachep->c_flags)) goto bufctl; bufp = (kmem_bufctl_t *)(objp+cachep->c_offset); /* get slab for the obj */ if (SLAB_PTR_IN_OBJ(cachep->c_flags)) { /* if SLAB_HIGH_PACK is undef, the below is optimised away */ slabp = (kmem_slab_t *)((((unsigned long)objp)&PAGE_MASK)+PAGE_SIZE); slabp--; } else slabp = (kmem_slab_t *) bufp->buf_slabp; if (slabp->s_magic != SLAB_MAGIC_ALLOC) /* sanity check */ goto bad_obj; cli(); #if defined(SLAB_DEBUG_SUPPORT) if (cachep->c_flags & (SLAB_DEBUG_FREE|SLAB_RED_ZONE)) goto extra_checks; #endif /* SLAB_DEBUG_SUPPORT */ passed_extra: if (!slabp->s_inuse) /* sanity check */ goto too_many; bufp->buf_nextp = slabp->s_freep; slabp->s_freep = bufp; if (--(slabp->s_inuse)) { if (bufp->buf_nextp) { restore_flags(save_flags); return; } kmem_cache_one_free(cachep, slabp); restore_flags(save_flags); return; } kmem_cache_full_free(cachep, slabp); restore_flags(save_flags); return; bufctl: /* Off-slab bufctls. Need to search hash for bufctl, and hence the slab. * No 'extra' checks are performed for objs stored this way, finding * the obj a check enough */ cli(); if ((bufp = kmem_remove_from_hash(cachep, objp))) { slabp = (kmem_slab_t *) bufp->buf_slabp; #if defined(SLAB_DEBUG_SUPPORT) if (cachep->c_flags & SLAB_RED_ZONE) goto red_zone; #endif /* SLAB_DEBUG_SUPPORT */ goto passed_extra; } restore_flags(save_flags); printk(KERN_ERR "kmem_free: Either bad obj addr or double free: %p - %s\n", objp, cachep->c_name); return; #if defined(SLAB_DEBUG_SUPPORT) red_zone: if (xchg((unsigned long *)(objp+cachep->c_org_size), SLAB_RED_MAGIC1) != SLAB_RED_MAGIC2) { /* Either write past end of the object, or a double free */ printk(KERN_ERR "kmem_free: Bad redzone %p - %s\n", objp, cachep->c_name); } goto passed_extra; init_state_check: /* Need to call the slab's constructor so that * the caller can perform a verify of its state (debugging) */ cachep->c_ctor(objp, cachep->c_org_size, SLAB_CTOR_CONSTRUCTOR|SLAB_CTOR_VERIFY); goto finished_initial; extra_checks: if ((cachep->c_flags & SLAB_DEBUG_FREE) && (objp != kmem_extra_free_checks(cachep, slabp->s_freep, bufp, objp))) { restore_flags(save_flags); return; } if (cachep->c_flags & SLAB_RED_ZONE) goto red_zone; goto passed_extra; #endif /* SLAB_DEBUG_SUPPORT */ bad_obj: /* The addr of the slab doesn't contain the correct * magic num */ if (slabp->s_magic == SLAB_MAGIC_UNALLOC) { /* magic num says this is an unalloc slab */ printk(KERN_ERR "kmem_free: obj %p from destroyed slab - %s\n", objp, cachep->c_name); return; } printk(KERN_ERR "kmem_free: Bad obj %p - %s\n", objp, cachep->c_name); return; too_many: /* don't add to freelist */ restore_flags(save_flags); printk(KERN_ERR "kmem_free: obj free for slab with no active objs - %s\n", cachep->c_name); return; nul_obj: printk(KERN_ERR "kmem_free: NULL obj - %s\n", cachep->c_name); return; nul_cache: printk(KERN_ERR "kmem_free: NULL cache ptr\n"); return; } void * kmem_cache_alloc(kmem_cache_t *cachep, unsigned long flags) { return __kmem_cache_alloc(cachep, flags); } void kmem_cache_free(kmem_cache_t *cachep, void *objp) { __kmem_cache_free(cachep, objp); } void * kmem_alloc(unsigned long size, unsigned long flags) { cache_sizes_t *cachep = cache_sizes; for (; cachep->cs_size; cachep++) { if (size > cachep->cs_size) continue; /* should the inline version be used here? */ return kmem_cache_alloc(cachep->cs_cachep, flags); } printk(KERN_ERR "kmem_alloc: Size (%lu) too large\n", size); return NULL; } void kmem_free(void *objp, unsigned long size) { cache_sizes_t *cachep = cache_sizes; for (; cachep->cs_size; cachep++) { if (size > cachep->cs_size) continue; /* should the inline version be used here? */ kmem_cache_free(cachep->cs_cachep, objp); return; } printk(KERN_ERR "kmem_free: Size (%lu) too large - strange\n", size); } /* Called from try_to_free_page(). * Ideal solution would have a weight for each cache, based on; * o num of fully free slabs * o if the objs have a constructor/deconstructor * o length of time slabs have been fully free (ie. ageing) * This function _cannot_ be called within a int, but it * can be interrupted. */ int kmem_cache_reap(int pri, int dma, int wait) { unsigned long dtor_flags = 0; unsigned long best_jiffie; unsigned long now; int count = 8; kmem_slab_t *best_slabp = NULL; kmem_cache_t *best_cachep = NULL; kmem_slab_t *slabp; kmem_cache_t *searchp; unsigned long save_flags; /* 'pri' maps to the number of caches to examine, not the number of slabs. * This avoids only checking the jiffies for slabs in one cache at the * expensive spending more cycles */ pri = (9 - pri); if (!wait) /* not allowed to wait */ dtor_flags = SLAB_DTOR_ATOMIC; searchp = clock_searchp; save_flags(save_flags); now = jiffies; best_jiffie = now - (2*HZ); /* 2secs - avoid heavy thrashing */ while (pri--) { kmem_slab_t *local_slabp; unsigned long local_jiffie; if (searchp == &cache_cache) goto next; /* sanity check for corruption */ if (searchp->c_inuse || searchp->c_magic != SLAB_C_MAGIC) { printk(KERN_ERR "kmem_reap: Corrupted cache struct for %s\n", searchp->c_name); goto next; } local_slabp = NULL; local_jiffie = now - (2*HZ); cli(); /* As the fully free slabs, within a cache, have no particular * order, we need to test them all. Infact, we only check 'count' * slabs. */ slabp = searchp->c_lastp; for (;count && slabp != kmem_slab_end(searchp) && !slabp->s_inuse; slabp = slabp->s_prevp, count--) { if (slabp->s_jiffies >= local_jiffie) continue; /* weight caches with a con/decon */ if ((searchp->c_ctor || searchp->c_dtor) && slabp->s_jiffies >= (local_jiffie - (2*HZ))) continue; /* weight caches with high page orders. Avoids stressing the * VM sub-system by reducing the frequency requests for a large * num of contigious pages */ if (searchp->c_gfporder > 1 && slabp->s_jiffies >= (local_jiffie - (4*HZ))) continue; local_jiffie = slabp->s_jiffies; local_slabp = slabp; if (!searchp->c_gfporder && (now-local_jiffie) >= (300*HZ)) { /* an old, one page slab. Make a quick get away... */ pri = 0; break; } } if (local_slabp) { if (!count || local_jiffie < best_jiffie) { best_slabp = local_slabp; best_jiffie = local_jiffie; best_cachep = searchp; if (!count) break; } } restore_flags(save_flags); next: searchp = searchp->c_nextp; if (searchp == clock_searchp) break; count = 8; /* # of slabs at which we force a reap */ } /* only move along with we didn't find an over allocated cache */ if (count) clock_searchp = clock_searchp->c_nextp; if (!best_slabp) return 0; cli(); if (best_slabp->s_inuse) { /* an object in our selected slab has been * allocated. This souldn't happen v. often, so we * simply fail - which isn't ideal but will do. * NOTE: No test for the case where an obj has been * allocated from the slab, and then freed. While * this would change our idea of the best slab to * reap, it's not worth the re-calculation effort. */ restore_flags(save_flags); return 0; } if (best_cachep->c_freep == best_slabp) best_cachep->c_freep = best_slabp->s_nextp; kmem_slab_unlink(best_slabp); restore_flags(save_flags); kmem_slab_destroy(best_cachep, best_slabp, dtor_flags); return 1; } /* /proc/slabinfo * cache-name num-active-objs total-objs num-active-slabs total-slabs num-pages-per-slab */ int get_slabinfo(char *buf) { kmem_cache_t *cachep; kmem_slab_t *slabp; unsigned long active_objs; unsigned long num_slabs, active_slabs; unsigned long save_flags; int len=0; /* output format version, so at least we can change it without _too_ * many complaints */ len = sprintf(buf, "slabinfo - version: 1.0\n"); save_flags(save_flags); cachep = &cache_cache; do { active_slabs = num_slabs = active_objs = 0; cli(); for (slabp = cachep->c_firstp; slabp != kmem_slab_end(cachep); slabp = slabp->s_nextp) { num_slabs++; active_objs += slabp->s_inuse; if (slabp->s_inuse) active_slabs++; } restore_flags(save_flags); len += sprintf(buf+len, "%-20s%lu %lu %lu %lu %d\n", cachep->c_name, active_objs, cachep->c_num*num_slabs, active_slabs, num_slabs, 1<c_gfporder); } while ((cachep = cachep->c_nextp) != &cache_cache); return len; }