【走进RDS】之MySQL内存分配与管理（上篇）

#ifdef UNIV_PFS_MEMORY
#define UT_NEW(expr, key) ::new (ut_allocator<decltype(expr)>(key).allocate(1, NULL, key, false, false)) expr
...
#define ut_malloc(n_bytes, key) static_cast<void *>(ut_allocator<byte>(key).allocate(n_bytes, NULL, UT_NEW_THIS_FILE_PSI_KEY, false, false))
...


#else /* UNIV_PFS_MEMORY */
#define UT_NEW(expr, key) ::new (std::nothrow) expr
...
#define ut_malloc(n_bytes, key) ::malloc(n_bytes)
...
#endif

// 比实际申请多出一块pfx的内存
total_bytes+=sizeof(ut_new_pfx_t)
// 申请内存
...
// 返回实际内存开始的地址
return (reinterpret_cast<pointer>(pfx + 1));

for (size_t retries = 1;; retries++) {
  // 内存分配malloc/calloc
  malloc(); // calloc()...
  if (ptr != nullptr || retries >= alloc_max_retries) break;
  std::this_thread::sleep_for(std::chrono::seconds(1));
}

deallocate_trace(pfx);
free(pfx);

pointer ptr = reinterpret_cast<pointer>(os_mem_alloc_large(&n_bytes));
    |->mmap()/shmget()、shmat()、shmctl()
...
allocate_trace(n_bytes, PSI_NOT_INSTRUMENTED, pfx);

deallocate_trace(pfx);
os_mem_free_large(ptr, pfx->m_size); 
 |->munmap()/shmdt()

class mem_heap_allocator {
...
  pointer allocate(size_type n, const_pointer hint = nullptr) {
      return (reinterpret_cast<pointer>(mem_heap_alloc(m_heap, n * sizeof(T)))); // 内存申请调用mem_heap_alloc
  }
  void deallocate(pointer p, size_type n) {}; // 内存释放等为空操作
...
}

typedef struct mem_block_info_t mem_block_t;
typedef mem_block_t mem_heap_t;


...
/** The info structure stored at the beginning of a heap block */
struct mem_block_info_t {
...
  UT_LIST_BASE_NODE_T(mem_block_t) base; /* 链表基节点，只在第一个block定义 */
  UT_LIST_NODE_T(mem_block_t) list;   /* block链表 */
  ulint len;        /*!< 当前block大小 */
  ulint total_size; /*!< 所有block总大小 */
  ulint type;       /*!< 分配类型 */
  ulint free;       /*!< 当前block的可用位置 */
  ulint start;      /*!< block构建时free的起始位置（没看到较多的用途） */
  void *free_block; /* 包含有 MEM_HEAP_BTR_SEARCH 类型的heap中,
                        heap root挂着free_block用以申请更多的空间，其他类型该指针为空 */
  void *buf_block;  /* 内存从buffer pool申请，保存buf_block_t指针，否则为空 */
};

#define MEM_HEAP_DYNAMIC 0 /* 原始申请，调用innodb内存申请ut_allocator相关 */
#define MEM_HEAP_BUFFER 1 /* 从buffer_pool获取内存 */
#define MEM_HEAP_BTR_SEARCH 2/* 使用free_block中的内存 */       

/** Different type of heaps in terms of which data structure is using them */
#define MEM_HEAP_FOR_BTR_SEARCH (MEM_HEAP_BTR_SEARCH | MEM_HEAP_BUFFER)
#define MEM_HEAP_FOR_PAGE_HASH (MEM_HEAP_DYNAMIC)
#define MEM_HEAP_FOR_RECV_SYS (MEM_HEAP_BUFFER)
#define MEM_HEAP_FOR_LOCK_HEAP (MEM_HEAP_BUFFER)

mem_heap_t *mem_heap_create_func(ulint size, ulint type) {
  mem_block_t *block;
  
  if (!size) {
      size = MEM_BLOCK_START_SIZE;
  }
  // 创建mem_heap的第一个block，传入的第一个参数是nullptr
  block = mem_heap_create_block(nullptr, size, type, file_name, line);
  // 在MEM_HEAP_BTR_SEARCH模式下，存在构建失败的可能性，返回空指针
  if (block == nullptr) {
      return (nullptr);
  }
                  
   由于BP resize的可能性，因此第一个block不能从BP中获取
  ut_ad(block->buf_block == nullptr);
                      
  // 初始化链表基节点（base不为空，标志该节点为基节点）
  UT_LIST_INIT(block->base, &mem_block_t::list);
  UT_LIST_ADD_FIRST(block->base, block);
                                
  return (block);
}

void mem_heap_free(mem_heap_t *heap) {
  ...
// 获取链表中最后一个节点
 block = UT_LIST_GET_LAST(heap->base);
     
// 释放free_block节点（MEM_HEAP_BTR_SEARCH模式创建）
 if (heap->free_block) {
     mem_heap_free_block_free(heap);
 }
     
 // 由后往前逐个释放block
   while (block != nullptr) {
     /* Store the contents of info before freeing current block
     (it is erased in freeing) */
     prev_block = UT_LIST_GET_PREV(list, block);
     mem_heap_block_free(heap, block);
     block = prev_block;
 }
}

// case 1
if (type == MEM_HEAP_DYNAMIC || len < UNIV_PAGE_SIZE / 2) {
    ut_ad(type == MEM_HEAP_DYNAMIC || n <= MEM_MAX_ALLOC_IN_BUF);
  
    block = static_cast<mem_block_t *>(ut_malloc_nokey(len));
} else {
   len = UNIV_PAGE_SIZE;
                
// case 2
 if ((type & MEM_HEAP_BTR_SEARCH) && heap) {
    // 从heap root的free_block获取内存
    buf_block = static_cast<buf_block_t *>(heap->free_block);
    heap->free_block = nullptr;
                      
    if (UNIV_UNLIKELY(!buf_block)) {
        return (nullptr);
    }
} else {
// case 3             
   buf_block = buf_block_alloc(nullptr);
  }


 block = (mem_block_t *)buf_block->frame;
}

UNIV_MEM_FREE(block, len);
UNIV_MEM_ALLOC(block, MEM_BLOCK_HEADER_SIZE);


block->buf_block = buf_block;
block->free_block = nullptr;

// 移动构造函数
MEM_ROOT(MEM_ROOT &&other)
  noexcept
      : m_xxx(other.m_cxxx),
        ...{
    other.m_xxx = nullptr/0/origin_value;
  ...
  }
// 移动赋值
MEM_ROOT &operator=(MEM_ROOT &&other) noexcept {
   Clear();
   ::new (this) MEM_ROOT(std::move(other));
   return *this;
  }

void *MEM_ROOT::AllocSlow(size_t length) {
  // 本次申请的内存很大或是要求是独占一块内存的形式
  if (length >= m_block_size || MEM_ROOT_SINGLE_CHUNKS) {
    Block *new_block =
        AllocBlock(/*wanted_length=*/length, /*minimum_length=*/length);
    if (new_block == nullptr) return nullptr;
    if (m_current_block == nullptr) {
      new_block->prev = nullptr;
      m_current_block = new_block;
      m_current_free_end = new_block->end;
      m_current_free_start = m_current_free_end;
    } else {
      // Insert the new block in the second-to-last position.
      new_block->prev = m_current_block->prev;
      m_current_block->prev = new_block;
    }
    return pointer_cast<char *>(new_block) + ALIGN_SIZE(sizeof(*new_block));
  } else { // 常规情况
    if (ForceNewBlock(/*minimum_length=*/length)) {
      return nullptr;
    }
    char *new_mem = m_current_free_start;
    m_current_free_start += length;
    return new_mem;
  }
}

THD::THD(bool enable_plugins)
    : Query_arena(&main_mem_root, STMT_REGULAR_EXECUTION),
      ...
      lex_returning(new im::Lex_returning(false, &main_mem_root)),
      ... {
  main_lex->reset();
  set_psi(nullptr);
  mdl_context.init(this);
  init_sql_alloc(key_memory_thd_main_mem_root, &main_mem_root,
                  global_system_variables.query_alloc_block_size,
                  global_system_variables.query_prealloc_size);
  ...
 }

THD::THD(bool enable_plugins)
    : Query_arena(&main_mem_root, STMT_REGULAR_EXECUTION),
 ...
 
MEM_ROOT* old_mem_root = thd->mem_root; // 保存原来的mem_root(main_mem_root)
thd->mem_root = xxx_mem_root; // mem_root大多是临时性的MEM_ROOT
// do something using memory
...
thd->mem_root = old_mem_root; // 恢复成原来的mem_root(main_mem_root)

// sql/dd_table_share.cc
open_table_def() 


// sql/sp_head.cc
sp_parser_data::start_parsing_sp_body() &&
sp_parser_data::finish_parsing_sp_body()


// sql/sp_instr.cc PSI_NOT_INSTRUMENTED
LEX *sp_lex_instr::parse_expr() 


// sql/sql_cursor.cc
Query_result_materialize::start_execution()


// sql/sql_table.cc
rm_table_do_discovery_and_lock_fk_tables()
drop_base_table()
lock_check_constraint_names()


// sql/thd_raii.h 该类及其调用之处(sql/auth/sql_auth_cache.cc:grant_load())
class Swap_mem_root_guard; 


// sql/auth/sql_authorization.cc
mysql_table_grant() // 存储表级、行级权限
mysql_routine_grant() // 存储routine级权限


/* sql/dd/upgrade_57/global.h  storage/ndb/pligin/ndb_dd_upgrade_table.cc 
    该类及其调用之处 */
class Thd_mem_root_guard