2006-12-13 09:34:23 +01:00
|
|
|
#ifndef _LINUX_SLAB_DEF_H
|
|
|
|
#define _LINUX_SLAB_DEF_H
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Definitions unique to the original Linux SLAB allocator.
|
|
|
|
*
|
|
|
|
* What we provide here is a way to optimize the frequent kmalloc
|
|
|
|
* calls in the kernel by selecting the appropriate general cache
|
|
|
|
* if kmalloc was called with a size that can be established at
|
|
|
|
* compile time.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <asm/page.h> /* kmalloc_sizes.h needs PAGE_SIZE */
|
|
|
|
#include <asm/cache.h> /* kmalloc_sizes.h needs L1_CACHE_BYTES */
|
|
|
|
#include <linux/compiler.h>
|
2009-04-10 08:26:18 +02:00
|
|
|
#include <linux/kmemtrace.h>
|
2006-12-13 09:34:23 +01:00
|
|
|
|
2008-05-09 20:32:44 +02:00
|
|
|
/*
|
|
|
|
* struct kmem_cache
|
|
|
|
*
|
|
|
|
* manages a cache.
|
|
|
|
*/
|
|
|
|
|
|
|
|
struct kmem_cache {
|
|
|
|
/* 1) per-cpu data, touched during every alloc/free */
|
|
|
|
struct array_cache *array[NR_CPUS];
|
|
|
|
/* 2) Cache tunables. Protected by cache_chain_mutex */
|
|
|
|
unsigned int batchcount;
|
|
|
|
unsigned int limit;
|
|
|
|
unsigned int shared;
|
|
|
|
|
|
|
|
unsigned int buffer_size;
|
|
|
|
u32 reciprocal_buffer_size;
|
|
|
|
/* 3) touched by every alloc & free from the backend */
|
|
|
|
|
|
|
|
unsigned int flags; /* constant flags */
|
|
|
|
unsigned int num; /* # of objs per slab */
|
|
|
|
|
|
|
|
/* 4) cache_grow/shrink */
|
|
|
|
/* order of pgs per slab (2^n) */
|
|
|
|
unsigned int gfporder;
|
|
|
|
|
|
|
|
/* force GFP flags, e.g. GFP_DMA */
|
|
|
|
gfp_t gfpflags;
|
|
|
|
|
|
|
|
size_t colour; /* cache colouring range */
|
|
|
|
unsigned int colour_off; /* colour offset */
|
|
|
|
struct kmem_cache *slabp_cache;
|
|
|
|
unsigned int slab_size;
|
|
|
|
unsigned int dflags; /* dynamic flags */
|
|
|
|
|
|
|
|
/* constructor func */
|
|
|
|
void (*ctor)(void *obj);
|
|
|
|
|
|
|
|
/* 5) cache creation/removal */
|
|
|
|
const char *name;
|
|
|
|
struct list_head next;
|
|
|
|
|
|
|
|
/* 6) statistics */
|
|
|
|
#ifdef CONFIG_DEBUG_SLAB
|
|
|
|
unsigned long num_active;
|
|
|
|
unsigned long num_allocations;
|
|
|
|
unsigned long high_mark;
|
|
|
|
unsigned long grown;
|
|
|
|
unsigned long reaped;
|
|
|
|
unsigned long errors;
|
|
|
|
unsigned long max_freeable;
|
|
|
|
unsigned long node_allocs;
|
|
|
|
unsigned long node_frees;
|
|
|
|
unsigned long node_overflow;
|
|
|
|
atomic_t allochit;
|
|
|
|
atomic_t allocmiss;
|
|
|
|
atomic_t freehit;
|
|
|
|
atomic_t freemiss;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If debugging is enabled, then the allocator can add additional
|
|
|
|
* fields and/or padding to every object. buffer_size contains the total
|
|
|
|
* object size including these internal fields, the following two
|
|
|
|
* variables contain the offset to the user object and its size.
|
|
|
|
*/
|
|
|
|
int obj_offset;
|
|
|
|
int obj_size;
|
|
|
|
#endif /* CONFIG_DEBUG_SLAB */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We put nodelists[] at the end of kmem_cache, because we want to size
|
|
|
|
* this array to nr_node_ids slots instead of MAX_NUMNODES
|
|
|
|
* (see kmem_cache_init())
|
|
|
|
* We still use [MAX_NUMNODES] and not [1] or [0] because cache_cache
|
|
|
|
* is statically defined, so we reserve the max number of nodes.
|
|
|
|
*/
|
|
|
|
struct kmem_list3 *nodelists[MAX_NUMNODES];
|
|
|
|
/*
|
|
|
|
* Do not add fields after nodelists[]
|
|
|
|
*/
|
|
|
|
};
|
|
|
|
|
2006-12-13 09:34:23 +01:00
|
|
|
/* Size description struct for general caches. */
|
|
|
|
struct cache_sizes {
|
|
|
|
size_t cs_size;
|
|
|
|
struct kmem_cache *cs_cachep;
|
2007-02-10 10:43:10 +01:00
|
|
|
#ifdef CONFIG_ZONE_DMA
|
2006-12-13 09:34:23 +01:00
|
|
|
struct kmem_cache *cs_dmacachep;
|
2007-02-10 10:43:10 +01:00
|
|
|
#endif
|
2006-12-13 09:34:23 +01:00
|
|
|
};
|
|
|
|
extern struct cache_sizes malloc_sizes[];
|
|
|
|
|
slob: initial NUMA support
This adds preliminary NUMA support to SLOB, primarily aimed at systems with
small nodes (tested all the way down to a 128kB SRAM block), whether
asymmetric or otherwise.
We follow the same conventions as SLAB/SLUB, preferring current node
placement for new pages, or with explicit placement, if a node has been
specified. Presently on UP NUMA this has the side-effect of preferring
node#0 allocations (since numa_node_id() == 0, though this could be
reworked if we could hand off a pfn to determine node placement), so
single-CPU NUMA systems will want to place smaller nodes further out in
terms of node id. Once a page has been bound to a node (via explicit node
id typing), we only do block allocations from partial free pages that have
a matching node id in the page flags.
The current implementation does have some scalability problems, in that all
partial free pages are tracked in the global freelist (with contention due
to the single spinlock). However, these are things that are being reworked
for SMP scalability first, while things like per-node freelists can easily
be built on top of this sort of functionality once it's been added.
More background can be found in:
http://marc.info/?l=linux-mm&m=118117916022379&w=2
http://marc.info/?l=linux-mm&m=118170446306199&w=2
http://marc.info/?l=linux-mm&m=118187859420048&w=2
and subsequent threads.
Acked-by: Christoph Lameter <clameter@sgi.com>
Acked-by: Matt Mackall <mpm@selenic.com>
Signed-off-by: Paul Mundt <lethal@linux-sh.org>
Acked-by: Nick Piggin <nickpiggin@yahoo.com.au>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-16 08:38:22 +02:00
|
|
|
void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
|
|
|
|
void *__kmalloc(size_t size, gfp_t flags);
|
|
|
|
|
2008-08-10 19:14:05 +02:00
|
|
|
#ifdef CONFIG_KMEMTRACE
|
|
|
|
extern void *kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags);
|
|
|
|
extern size_t slab_buffer_size(struct kmem_cache *cachep);
|
|
|
|
#else
|
|
|
|
static __always_inline void *
|
|
|
|
kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags)
|
2006-12-13 09:34:23 +01:00
|
|
|
{
|
2008-08-10 19:14:05 +02:00
|
|
|
return kmem_cache_alloc(cachep, flags);
|
|
|
|
}
|
|
|
|
static inline size_t slab_buffer_size(struct kmem_cache *cachep)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static __always_inline void *kmalloc(size_t size, gfp_t flags)
|
|
|
|
{
|
|
|
|
struct kmem_cache *cachep;
|
|
|
|
void *ret;
|
|
|
|
|
2006-12-13 09:34:23 +01:00
|
|
|
if (__builtin_constant_p(size)) {
|
|
|
|
int i = 0;
|
2007-07-17 13:03:22 +02:00
|
|
|
|
|
|
|
if (!size)
|
|
|
|
return ZERO_SIZE_PTR;
|
|
|
|
|
2006-12-13 09:34:23 +01:00
|
|
|
#define CACHE(x) \
|
|
|
|
if (size <= x) \
|
|
|
|
goto found; \
|
|
|
|
else \
|
|
|
|
i++;
|
2008-03-05 22:58:17 +01:00
|
|
|
#include <linux/kmalloc_sizes.h>
|
2006-12-13 09:34:23 +01:00
|
|
|
#undef CACHE
|
2009-01-27 22:48:59 +01:00
|
|
|
return NULL;
|
2006-12-13 09:34:23 +01:00
|
|
|
found:
|
2007-02-10 10:43:10 +01:00
|
|
|
#ifdef CONFIG_ZONE_DMA
|
|
|
|
if (flags & GFP_DMA)
|
2008-08-10 19:14:05 +02:00
|
|
|
cachep = malloc_sizes[i].cs_dmacachep;
|
|
|
|
else
|
2007-02-10 10:43:10 +01:00
|
|
|
#endif
|
2008-08-10 19:14:05 +02:00
|
|
|
cachep = malloc_sizes[i].cs_cachep;
|
|
|
|
|
|
|
|
ret = kmem_cache_alloc_notrace(cachep, flags);
|
|
|
|
|
2009-03-23 14:12:24 +01:00
|
|
|
trace_kmalloc(_THIS_IP_, ret,
|
|
|
|
size, slab_buffer_size(cachep), flags);
|
2008-08-10 19:14:05 +02:00
|
|
|
|
|
|
|
return ret;
|
2006-12-13 09:34:23 +01:00
|
|
|
}
|
|
|
|
return __kmalloc(size, flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_NUMA
|
|
|
|
extern void *__kmalloc_node(size_t size, gfp_t flags, int node);
|
slob: initial NUMA support
This adds preliminary NUMA support to SLOB, primarily aimed at systems with
small nodes (tested all the way down to a 128kB SRAM block), whether
asymmetric or otherwise.
We follow the same conventions as SLAB/SLUB, preferring current node
placement for new pages, or with explicit placement, if a node has been
specified. Presently on UP NUMA this has the side-effect of preferring
node#0 allocations (since numa_node_id() == 0, though this could be
reworked if we could hand off a pfn to determine node placement), so
single-CPU NUMA systems will want to place smaller nodes further out in
terms of node id. Once a page has been bound to a node (via explicit node
id typing), we only do block allocations from partial free pages that have
a matching node id in the page flags.
The current implementation does have some scalability problems, in that all
partial free pages are tracked in the global freelist (with contention due
to the single spinlock). However, these are things that are being reworked
for SMP scalability first, while things like per-node freelists can easily
be built on top of this sort of functionality once it's been added.
More background can be found in:
http://marc.info/?l=linux-mm&m=118117916022379&w=2
http://marc.info/?l=linux-mm&m=118170446306199&w=2
http://marc.info/?l=linux-mm&m=118187859420048&w=2
and subsequent threads.
Acked-by: Christoph Lameter <clameter@sgi.com>
Acked-by: Matt Mackall <mpm@selenic.com>
Signed-off-by: Paul Mundt <lethal@linux-sh.org>
Acked-by: Nick Piggin <nickpiggin@yahoo.com.au>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-16 08:38:22 +02:00
|
|
|
extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
|
2006-12-13 09:34:23 +01:00
|
|
|
|
2008-08-10 19:14:05 +02:00
|
|
|
#ifdef CONFIG_KMEMTRACE
|
|
|
|
extern void *kmem_cache_alloc_node_notrace(struct kmem_cache *cachep,
|
|
|
|
gfp_t flags,
|
|
|
|
int nodeid);
|
|
|
|
#else
|
|
|
|
static __always_inline void *
|
|
|
|
kmem_cache_alloc_node_notrace(struct kmem_cache *cachep,
|
|
|
|
gfp_t flags,
|
|
|
|
int nodeid)
|
|
|
|
{
|
|
|
|
return kmem_cache_alloc_node(cachep, flags, nodeid);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
|
2006-12-13 09:34:23 +01:00
|
|
|
{
|
2008-08-10 19:14:05 +02:00
|
|
|
struct kmem_cache *cachep;
|
|
|
|
void *ret;
|
|
|
|
|
2006-12-13 09:34:23 +01:00
|
|
|
if (__builtin_constant_p(size)) {
|
|
|
|
int i = 0;
|
2007-07-17 13:03:22 +02:00
|
|
|
|
|
|
|
if (!size)
|
|
|
|
return ZERO_SIZE_PTR;
|
|
|
|
|
2006-12-13 09:34:23 +01:00
|
|
|
#define CACHE(x) \
|
|
|
|
if (size <= x) \
|
|
|
|
goto found; \
|
|
|
|
else \
|
|
|
|
i++;
|
2008-03-05 22:58:17 +01:00
|
|
|
#include <linux/kmalloc_sizes.h>
|
2006-12-13 09:34:23 +01:00
|
|
|
#undef CACHE
|
2009-01-27 22:48:59 +01:00
|
|
|
return NULL;
|
2006-12-13 09:34:23 +01:00
|
|
|
found:
|
2007-02-10 10:43:10 +01:00
|
|
|
#ifdef CONFIG_ZONE_DMA
|
|
|
|
if (flags & GFP_DMA)
|
2008-08-10 19:14:05 +02:00
|
|
|
cachep = malloc_sizes[i].cs_dmacachep;
|
|
|
|
else
|
2007-02-10 10:43:10 +01:00
|
|
|
#endif
|
2008-08-10 19:14:05 +02:00
|
|
|
cachep = malloc_sizes[i].cs_cachep;
|
|
|
|
|
|
|
|
ret = kmem_cache_alloc_node_notrace(cachep, flags, node);
|
|
|
|
|
2009-03-23 14:12:24 +01:00
|
|
|
trace_kmalloc_node(_THIS_IP_, ret,
|
|
|
|
size, slab_buffer_size(cachep),
|
|
|
|
flags, node);
|
2008-08-10 19:14:05 +02:00
|
|
|
|
|
|
|
return ret;
|
2006-12-13 09:34:23 +01:00
|
|
|
}
|
|
|
|
return __kmalloc_node(size, flags, node);
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif /* CONFIG_NUMA */
|
|
|
|
|
|
|
|
#endif /* _LINUX_SLAB_DEF_H */
|