about 1 results (0.00 seconds)

话说Python内存管理机制篇一之结构

by LauCyun Jul 24,2017 15:54:00 14,280 views

在Python的内部系统中,它的内存管理结构是以金子塔结构呈现的,如下:

    _____   ______   ______       ________
   [ int ] [ dict ] [ list ] ... [ string ]       Python core         |
+3 | <----- Object-specific memory -----> | <-- Non-object memory --> |
    _______________________________       |                           |
   [   Python's object allocator   ]      |                           |
+2 | ####### Object memory ####### | <------ Internal buffers ------> |
    ______________________________________________________________    |
   [          Python's raw memory allocator (PyMem_ API)          ]   |
+1 | <----- Python memory (under PyMem manager's control) ------> |   |
    __________________________________________________________________
   [    Underlying general-purpose allocator (ex: C library malloc)   ]
 0 | <------ Virtual memory allocated for the python process -------> |

   =========================================================================
    _______________________________________________________________________
   [                OS-specific Virtual Memory Manager (VMM)               ]
-1 | <--- Kernel dynamic storage allocation & management (page-based) ---> |
    __________________________________   __________________________________
   [                                  ] [                                  ]
-2 | <-- Physical memory: ROM/RAM --> | | <-- Secondary storage (swap) --> |

Layer-2、Layer-1:

这两层是跟操作系统来负责的,就不表了。

Layer0:

操作系统提供的内存管理接口,比如C运行时所提供的mallocfree接口。这层由操作系统实现并管理,Python不能干涉这一层的行为。

Layer1:

Python基于第0层操作系统的内存管理接口封装而成的,为Python提供一层统一的raw memory的管理接口,防止操作系统的差异。以PyMem_为前缀的函数族来实现。

Include/pymem.h中,声明了如下函数:

PyAPI_FUNC(void *) PyMem_Malloc(size_t size);
PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize);
PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size);
PyAPI_FUNC(void) PyMem_Free(void *ptr);

然后在Object/obmalloc.c中,使用了一个数据结构PyMemAllocatorEx里面定义了上下文及四个函数:

typedef struct {
    /* user context passed as the first argument to the 4 functions */
    void *ctx;

    /* allocate a memory block */
    void* (*malloc) (void *ctx, size_t size);

    /* allocate a memory block initialized by zeros */
    void* (*calloc) (void *ctx, size_t nelem, size_t elsize);

    /* allocate or resize a memory block */
    void* (*realloc) (void *ctx, void *ptr, size_t new_size);

    /* release a memory block */
    void (*free) (void *ctx, void *ptr);
} PyMemAllocatorEx;

void
PyMem_SetupDebugHooks(void)
{
#ifdef PYMALLOC_DEBUG
    PyMemAllocatorEx alloc;

    alloc.malloc = _PyMem_DebugMalloc;
    alloc.calloc = _PyMem_DebugCalloc;
    alloc.realloc = _PyMem_DebugRealloc;
    alloc.free = _PyMem_DebugFree;

    if (_PyMem_Raw.malloc != _PyMem_DebugMalloc) {
        alloc.ctx = &_PyMem_Debug.raw;
        PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &_PyMem_Debug.raw.alloc);
        PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc);
    }

    if (_PyMem.malloc != _PyMem_DebugMalloc) {
        alloc.ctx = &_PyMem_Debug.mem;
        PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &_PyMem_Debug.mem.alloc);
        PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc);
    }

    if (_PyObject.malloc != _PyMem_DebugMalloc) {
        alloc.ctx = &_PyMem_Debug.obj;
        PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &_PyMem_Debug.obj.alloc);
        PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc);
    }
#endif
}

void *
PyMem_Malloc(size_t size)
{
    /* see PyMem_RawMalloc() */
    if (size > (size_t)PY_SSIZE_T_MAX)
        return NULL;
    return _PyMem.malloc(_PyMem.ctx, size);
}

void *
PyMem_Calloc(size_t nelem, size_t elsize)
{
    /* see PyMem_RawMalloc() */
    if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
        return NULL;
    return _PyMem.calloc(_PyMem.ctx, nelem, elsize);
}

void *
PyMem_Realloc(void *ptr, size_t new_size)
{
    /* see PyMem_RawMalloc() */
    if (new_size > (size_t)PY_SSIZE_T_MAX)
        return NULL;
    return _PyMem.realloc(_PyMem.ctx, ptr, new_size);
}

void
PyMem_Free(void *ptr)
{
    _PyMem.free(_PyMem.ctx, ptr);
}

在第一层中,Python还提供了面向Python中类型的内存分配器

/*
 * Include/pymem.h
 */

/*
 * Type-oriented memory interface
 * ==============================
 *
 * Allocate memory for n objects of the given type.  Returns a new pointer
 * or NULL if the request was too large or memory allocation failed.  Use
 * these macros rather than doing the multiplication yourself so that proper
 * overflow checking is always done.
 */

#define PyMem_New(type, n) \
  ( ((size_t)(n) > PY_SSIZE_T_MAX / sizeof(type)) ? NULL :	\
	( (type *) PyMem_Malloc((n) * sizeof(type)) ) )
#define PyMem_NEW(type, n) \
  ( ((size_t)(n) > PY_SSIZE_T_MAX / sizeof(type)) ? NULL :	\
	( (type *) PyMem_MALLOC((n) * sizeof(type)) ) )

/*
 * The value of (p) is always clobbered by this macro regardless of success.
 * The caller MUST check if (p) is NULL afterwards and deal with the memory
 * error if so.  This means the original value of (p) MUST be saved for the
 * caller's memory error handler to not lose track of it.
 */
#define PyMem_Resize(p, type, n) \
  ( (p) = ((size_t)(n) > PY_SSIZE_T_MAX / sizeof(type)) ? NULL :	\
	(type *) PyMem_Realloc((p), (n) * sizeof(type)) )
#define PyMem_RESIZE(p, type, n) \
  ( (p) = ((size_t)(n) > PY_SSIZE_T_MAX / sizeof(type)) ? NULL :	\
	(type *) PyMem_REALLOC((p), (n) * sizeof(type)) )

/* PyMem{Del,DEL} are left over from ancient days, and shouldn't be used
 * anymore.  They're just confusing aliases for PyMem_{Free,FREE} now.
 */
#define PyMem_Del		PyMem_Free
#define PyMem_DEL		PyMem_FREE

PyMem_New中,只要提供类型和数量,Python会自动侦测其所需的内存空间大小。

Layer2:

PyObj_为前缀的函数族,主要提供了创建Python对象的接口。这一套函数族又被唤作Pymalloc机制。

在第二层内存管理机制之上,对于Python中的一些常用对象,比如整数对象、字符串对象等,Python又构建了更高抽象层次的内存管理策略。

真正在Python中发挥巨大作用、同时也是GC的藏身之处的内存管理机制所在层次。

Layer3:

第三层的内存管理策略,主要就是对象缓冲池机制。

(全文完)

...

Tags Read More..