From 799b21f0b0d152e9f2a4abc3a61c5c34ed65a467 Mon Sep 17 00:00:00 2001 From: jooho812 Date: Tue, 26 Sep 2017 14:34:18 +0900 Subject: [PATCH] FEATURE : Add block allocator for more efficient memory management - This feature is applied only on btree collection --- engines/COMMON/mblock_allocator.c | 86 +++-- engines/COMMON/mblock_allocator.h | 4 +- engines/default/default_engine.c | 47 +++ engines/default/items.c | 249 +++++++++++- engines/default/items.h | 21 ++ engines/demo/demo_engine.c | 22 ++ include/memcached/engine.h | 23 ++ include/memcached/types.h | 40 +- memcached.c | 604 +++++++++++++++++++++++++++++- 9 files changed, 1040 insertions(+), 56 deletions(-) diff --git a/engines/COMMON/mblock_allocator.c b/engines/COMMON/mblock_allocator.c index 7faae52b8..e643c2e31 100644 --- a/engines/COMMON/mblock_allocator.c +++ b/engines/COMMON/mblock_allocator.c @@ -39,6 +39,17 @@ static void do_mblock_allocator_free_all() { pool_tail = NULL; } +static void prepare_eblk_add_elem(eblock_result_t *result) { + if (result->tail_blk == NULL) { + result->tail_blk = result->head_blk; + result->elem_cnt = 0; + } else { + assert(result->elem_cnt > 0); + if (result->elem_cnt % EITEMS_PER_BLOCK == 0) + result->tail_blk = result->tail_blk->next; + } +} + int mblock_allocator_init(size_t nblocks) { mem_block_t *helper = NULL; int i; @@ -49,7 +60,7 @@ int mblock_allocator_init(size_t nblocks) { else nblk = nblocks; for (i = 0; i < nblk; i++) { - helper = (mem_block_t *)malloc(sizeof(mem_block_t)); + helper = (mem_block_t *)malloc(EITEMS_BLCK_SIZE); if (helper == NULL) break; helper->next = NULL; @@ -105,7 +116,7 @@ void *allocate_single_block() { // This malloc() inside mutex may raise some performance issue, // Is there any way to execute malloc and counter adjustment // outside the mutex lock? - ret = (mem_block_t *)malloc(sizeof(mem_block_t)); + ret = (mem_block_t *)malloc(EITEMS_BLCK_SIZE); if (ret != NULL) { total_blocks++; ret->next = NULL; @@ -157,7 +168,7 @@ bool mblock_list_alloc(uint32_t blck_cnt, mem_block_t **head_blk, mem_block_t ** mem_block_t *new_blk = NULL; uint32_t new_cnt = 0; while (alloc_cnt < blck_cnt) { - if ((new_blk = (mem_block_t *)malloc(sizeof(mem_block_t))) == NULL) break; + if ((new_blk = (mem_block_t *)malloc(EITEMS_BLCK_SIZE)) == NULL) break; new_blk->next = NULL; if (*head_blk) (*tail_blk)->next = new_blk; @@ -171,7 +182,7 @@ bool mblock_list_alloc(uint32_t blck_cnt, mem_block_t **head_blk, mem_block_t ** total_mblocks += new_cnt; //pthread_mutex_unlock(&pool_mutex); if (alloc_cnt < blck_cnt) { - mblock_list_free(alloc_cnt, *head_blk, *tail_blk); + mblock_list_free(alloc_cnt, head_blk, tail_blk); return false; } } @@ -179,24 +190,25 @@ bool mblock_list_alloc(uint32_t blck_cnt, mem_block_t **head_blk, mem_block_t ** return true; } -void mblock_list_free(uint32_t blck_cnt, mem_block_t *head_blk, mem_block_t *tail_blk) { +void mblock_list_free(uint32_t blck_cnt, mem_block_t **head_blk, mem_block_t **tail_blk) { //mem_block_t *bye = NULL; //mem_block_t *bye_helper = NULL; //pthread_mutex_lock(&pool_mutex); - if (head_blk == NULL || blck_cnt == 0) + if (*head_blk == NULL || blck_cnt == 0) return; assert(pool_tail == NULL || pool_tail->next == NULL); - assert(tail_blk->next == NULL); + assert((*tail_blk)->next == NULL); if (pool_head == NULL) { - pool_head = head_blk; + pool_head = *head_blk; } else { - pool_tail->next = head_blk; + pool_tail->next = *head_blk; } - pool_tail = tail_blk; + pool_tail = *tail_blk; + *head_blk = *tail_blk = NULL; free_mblocks += blck_cnt; assert(free_mblocks <= total_mblocks); @@ -223,17 +235,35 @@ void mblock_list_free(uint32_t blck_cnt, mem_block_t *head_blk, mem_block_t *tai free(bye_helper); }*/ } + bool eblk_prepare(eblock_result_t *result, uint32_t elem_count) { assert(elem_count > 0); - uint32_t blkcnt = ((elem_count - 1) / EITEMS_PER_BLOCK) + 1; - if (!mblock_list_alloc(blkcnt, &result->head_blk, &result->last_blk)) { - result->elem_cnt = 0; - return false; + uint32_t blkcnt; + if (result->head_blk == NULL) { // empty block + blkcnt = ((elem_count - 1) / EITEMS_PER_BLOCK) + 1; + if (!mblock_list_alloc(blkcnt, &result->head_blk, &result->last_blk)) { + result->elem_cnt = 0; + return false; + } + result->tail_blk = NULL; + } else { + mem_block_t *head; + mem_block_t *last; + uint32_t curr_blkcnt = result->blck_cnt; + int alloc_blkcnt; + blkcnt = ((result->elem_cnt + elem_count - 1) / EITEMS_PER_BLOCK) + 1; + alloc_blkcnt = blkcnt - curr_blkcnt; + if (alloc_blkcnt > 0) { // need append block + if (!mblock_list_alloc((alloc_blkcnt), &head, &last)) + return false; + result->last_blk->next = head; + result->last_blk = last; + } } - result->tail_blk = NULL; result->blck_cnt = blkcnt; return true; } + void eblk_truncate(eblock_result_t *result) { assert(result->last_blk->next == NULL); /* returns empty blocklist */ @@ -244,27 +274,33 @@ void eblk_truncate(eblock_result_t *result) { uint32_t used_nblks = ((result->elem_cnt - 1) / EITEMS_PER_BLOCK) + 1; uint32_t free_nblks = result->blck_cnt - used_nblks; - mblock_list_free(free_nblks, free_head, free_tail); + mblock_list_free(free_nblks, &free_head, &free_tail); result->tail_blk->next = NULL; result->last_blk = result->tail_blk; result->blck_cnt -= free_nblks; } } else { /* ENGINE_ELEM_ENOENT case */ - mblock_list_free(result->blck_cnt, result->head_blk, result->last_blk); + mblock_list_free(result->blck_cnt, &result->head_blk, &result->last_blk); result->head_blk = result->tail_blk = result->last_blk = NULL; result->elem_cnt = result->blck_cnt = 0; } } void eblk_add_elem(eblock_result_t *result, eitem *elem) { - if (result->tail_blk == NULL) { - result->tail_blk = result->head_blk; - result->elem_cnt = 0; - } else { - assert(result->elem_cnt > 0); - if (result->elem_cnt % EITEMS_PER_BLOCK == 0) - result->tail_blk = result->tail_blk->next; + prepare_eblk_add_elem(result); + result->tail_blk->items[result->elem_cnt++ % EITEMS_PER_BLOCK] = (eitem *)elem; +} + +void eblk_add_elem_with_posi(eblock_result_t *result, eitem *elem, int posi) { + mem_block_t *curr_blk = result->head_blk; + int move_block_count = (posi / EITEMS_PER_BLOCK); + + while (move_block_count > 0) { + curr_blk = curr_blk->next; + move_block_count--; } - result->tail_blk->items[result->elem_cnt++ % EITEMS_PER_BLOCK] = (eitem *)elem; + prepare_eblk_add_elem(result); + curr_blk->items[posi % EITEMS_PER_BLOCK] = (eitem *)elem; + result->elem_cnt++; } diff --git a/engines/COMMON/mblock_allocator.h b/engines/COMMON/mblock_allocator.h index 4b16c52ec..cab22f266 100644 --- a/engines/COMMON/mblock_allocator.h +++ b/engines/COMMON/mblock_allocator.h @@ -29,10 +29,10 @@ void mblock_allocator_destroy(void); void mblock_allocator_stats(mblock_stats *blk_stat); bool mblock_list_alloc(uint32_t blck_cnt, mem_block_t **head_blk, mem_block_t **tail_blk); -void mblock_list_free(uint32_t blck_cnt, mem_block_t *head_blk, mem_block_t *tail_blk); +void mblock_list_free(uint32_t blck_cnt, mem_block_t **head_blk, mem_block_t **tail_blk); bool eblk_prepare(eblock_result_t *result, uint32_t elem_count); void eblk_truncate(eblock_result_t *result); void eblk_add_elem(eblock_result_t *result, eitem *elem); - +void eblk_add_elem_with_posi(eblock_result_t *result, eitem *elem, int posi); #endif diff --git a/engines/default/default_engine.c b/engines/default/default_engine.c index 9f69b45c3..541a82773 100644 --- a/engines/default/default_engine.c +++ b/engines/default/default_engine.c @@ -777,6 +777,15 @@ default_btree_elem_alloc(ENGINE_HANDLE* handle, const void* cookie, return ret; } +#ifdef USE_EBLOCK_RESULT +static void +default_btree_elem_release(ENGINE_HANDLE* handle, const void *cookie, + eitem *eitem, EITEM_TYPE type) +{ + struct default_engine *engine = get_handle(handle); + btree_elem_release(engine, eitem, type); +} +#else static void default_btree_elem_release(ENGINE_HANDLE* handle, const void *cookie, eitem **eitem_array, const int eitem_count) @@ -784,6 +793,7 @@ default_btree_elem_release(ENGINE_HANDLE* handle, const void *cookie, struct default_engine *engine = get_handle(handle); btree_elem_release(engine, (btree_elem_item**)eitem_array, eitem_count); } +#endif static ENGINE_ERROR_CODE default_btree_elem_insert(ENGINE_HANDLE* handle, const void* cookie, @@ -879,7 +889,11 @@ default_btree_elem_get(ENGINE_HANDLE* handle, const void* cookie, const bkey_range *bkrange, const eflag_filter *efilter, const uint32_t offset, const uint32_t req_count, const bool delete, const bool drop_if_empty, +#ifdef USE_EBLOCK_RESULT + eblock_result_t *eblk_ret, +#else eitem** eitem_array, uint32_t* eitem_count, +#endif uint32_t *access_count, uint32_t* flags, bool* dropped_trimmed, uint16_t vbucket) { @@ -890,7 +904,11 @@ default_btree_elem_get(ENGINE_HANDLE* handle, const void* cookie, if (delete) ACTION_BEFORE_WRITE(cookie, key, nkey); ret = btree_elem_get(engine, key, nkey, bkrange, efilter, offset, req_count, delete, drop_if_empty, +#ifdef USE_EBLOCK_RESULT + eblk_ret, +#else (btree_elem_item**)eitem_array, eitem_count, +#endif access_count, flags, dropped_trimmed); if (delete) ACTION_AFTER_WRITE(cookie, ret); return ret; @@ -932,8 +950,13 @@ default_btree_posi_find_with_get(ENGINE_HANDLE* handle, const void* cookie, const char *key, const size_t nkey, const bkey_range *bkrange, ENGINE_BTREE_ORDER order, const uint32_t count, +#ifdef USE_EBLOCK_RESULT + int *position, eblock_result_t *eblk_ret, + uint32_t *eitem_index, +#else int *position, eitem **eitem_array, uint32_t *eitem_count, uint32_t *eitem_index, +#endif uint32_t *flags, uint16_t vbucket) { struct default_engine *engine = get_handle(handle); @@ -941,8 +964,13 @@ default_btree_posi_find_with_get(ENGINE_HANDLE* handle, const void* cookie, VBUCKET_GUARD(engine, vbucket); ret = btree_posi_find_with_get(engine, key, nkey, bkrange, order, count, +#ifdef USE_EBLOCK_RESULT + position, eblk_ret, + eitem_index, flags); +#else position, (btree_elem_item**)eitem_array, eitem_count, eitem_index, flags); +#endif return ret; } @@ -951,7 +979,11 @@ default_btree_elem_get_by_posi(ENGINE_HANDLE* handle, const void* cookie, const char *key, const size_t nkey, ENGINE_BTREE_ORDER order, uint32_t from_posi, uint32_t to_posi, +#ifdef USE_EBLOCK_RESULT + eblock_result_t *eblk_ret, +#else eitem **eitem_array, uint32_t *eitem_count, +#endif uint32_t *flags, uint16_t vbucket) { struct default_engine *engine = get_handle(handle); @@ -959,7 +991,11 @@ default_btree_elem_get_by_posi(ENGINE_HANDLE* handle, const void* cookie, VBUCKET_GUARD(engine, vbucket); ret = btree_elem_get_by_posi(engine, key, nkey, order, from_posi, to_posi, +#ifdef USE_EBLOCK_RESULT + eblk_ret, +#else (btree_elem_item**)eitem_array, eitem_count, +#endif flags); return ret; } @@ -973,10 +1009,16 @@ default_btree_elem_smget_old(ENGINE_HANDLE* handle, const void* cookie, const bkey_range *bkrange, const eflag_filter *efilter, const uint32_t offset, const uint32_t count, +#ifdef USE_EBLOCK_RESULT + eblock_result_t *eblk_ret, + uint32_t* kfnd_array, + uint32_t* flag_array, +#else eitem** eitem_array, uint32_t* kfnd_array, uint32_t* flag_array, uint32_t* eitem_count, +#endif uint32_t* missed_key_array, uint32_t* missed_key_count, bool *trimmed, bool *duplicated, @@ -987,8 +1029,13 @@ default_btree_elem_smget_old(ENGINE_HANDLE* handle, const void* cookie, VBUCKET_GUARD(engine, vbucket); ret = btree_elem_smget_old(engine, karray, kcount, bkrange, efilter, +#ifdef USE_EBLOCK_RESULT + offset, count, eblk_ret, + kfnd_array, flag_array, +#else offset, count, (btree_elem_item**)eitem_array, kfnd_array, flag_array, eitem_count, +#endif missed_key_array, missed_key_count, trimmed, duplicated); return ret; diff --git a/engines/default/items.c b/engines/default/items.c index 4b9c4b95c..faf9af868 100644 --- a/engines/default/items.c +++ b/engines/default/items.c @@ -2517,6 +2517,20 @@ static void do_btree_elem_free(struct default_engine *engine, btree_elem_item *e do_mem_slot_free(engine, elem, ntotal); } +#ifdef USE_EBLOCK_RESULT +static void do_btree_elem_release(struct default_engine *engine, eitem *eitem) +{ + /* assert(elem->status != BTREE_ITEM_STATUS_FREE); */ + btree_elem_item *elem = (btree_elem_item *)eitem; + if (elem->refcount != 0) { + elem->refcount--; + } + if (elem->refcount == 0 && elem->status == BTREE_ITEM_STATUS_UNLINK) { + elem->status = BTREE_ITEM_STATUS_FREE; + do_btree_elem_free(engine, elem); + } +} +#else static void do_btree_elem_release(struct default_engine *engine, btree_elem_item *elem) { /* assert(elem->status != BTREE_ITEM_STATUS_FREE); */ @@ -2528,6 +2542,7 @@ static void do_btree_elem_release(struct default_engine *engine, btree_elem_item do_btree_elem_free(engine, elem); } } +#endif static inline btree_elem_item *do_btree_get_first_elem(btree_indx_node *node) { @@ -4488,7 +4503,11 @@ static bool do_btree_overlapped_with_trimmed_space(btree_meta_info *info, static ENGINE_ERROR_CODE do_btree_elem_get(struct default_engine *engine, btree_meta_info *info, const int bkrtype, const bkey_range *bkrange, const eflag_filter *efilter, const uint32_t offset, const uint32_t count, const bool delete, +#ifdef USE_EBLOCK_RESULT + eblock_result_t *eblk_ret, +#else btree_elem_item **elem_array, uint32_t *elem_count, +#endif uint32_t *access_count, bool *potentialbkeytrim) { btree_elem_posi path[BTREE_MAX_DEPTH]; @@ -4504,6 +4523,11 @@ static ENGINE_ERROR_CODE do_btree_elem_get(struct default_engine *engine, btree_ return ENGINE_ELEM_ENOENT; } +#ifdef USE_EBLOCK_RESULT + if (!eblk_prepare(eblk_ret, (count > 0 && count < info->ccnt) ? count : info->ccnt)) + return ENGINE_ENOMEM; +#endif + assert(info->root->ndepth < BTREE_MAX_DEPTH); elem = do_btree_find_first(info->root, bkrtype, bkrange, path, delete); if (elem != NULL) { @@ -4513,7 +4537,12 @@ static ENGINE_ERROR_CODE do_btree_elem_get(struct default_engine *engine, btree_ if (offset == 0) { if (efilter == NULL || do_btree_elem_filter(elem, efilter)) { elem->refcount++; +#ifdef USE_EBLOCK_RESULT + eblk_add_elem(eblk_ret, elem); + tot_found++; +#else elem_array[tot_found++] = elem; +#endif if (delete) { do_btree_elem_unlink(engine, info, path, ELEM_DELETE_NORMAL); } @@ -4554,7 +4583,11 @@ static ENGINE_ERROR_CODE do_btree_elem_get(struct default_engine *engine, btree_ skip_cnt++; } else { elem->refcount++; +#ifdef USE_EBLOCK_RESULT + eblk_add_elem(eblk_ret, elem); +#else elem_array[tot_found+cur_found] = elem; +#endif if (delete) { stotal += slabs_space_size(engine, do_btree_elem_ntotal(elem)); elem->status = BTREE_ITEM_STATUS_UNLINK; @@ -4633,7 +4666,11 @@ static ENGINE_ERROR_CODE do_btree_elem_get(struct default_engine *engine, btree_ if (access_count) *access_count = tot_access; +#ifdef USE_EBLOCK_RESULT + eblk_truncate(eblk_ret); +#else *elem_count = tot_found; +#endif if (tot_found > 0) { return ENGINE_SUCCESS; } else { @@ -4864,7 +4901,11 @@ static int do_btree_posi_find(btree_meta_info *info, static int do_btree_elem_batch_get(btree_elem_posi posi, const int count, const bool forward, const bool reverse, +#ifdef USE_EBLOCK_RESULT + eblock_result_t *eblk_ret) +#else btree_elem_item **elem_array) +#endif { btree_elem_item *elem; int nfound = 0; @@ -4875,24 +4916,41 @@ static int do_btree_elem_batch_get(btree_elem_posi posi, const int count, elem = BTREE_GET_ELEM_ITEM(posi.node, posi.indx); elem->refcount++; +#ifdef USE_EBLOCK_RESULT + if (reverse) eblk_add_elem_with_posi(eblk_ret, elem, count-nfound-1); + else eblk_add_elem(eblk_ret, elem); +#else if (reverse) elem_array[count-nfound-1] = elem; else elem_array[nfound] = elem; +#endif nfound += 1; } return nfound; } +#ifdef USE_EBLOCK_RESULT +static ENGINE_ERROR_CODE do_btree_posi_find_with_get(btree_meta_info *info, + const int bkrtype, const bkey_range *bkrange, + ENGINE_BTREE_ORDER order, const int count, + eblock_result_t *eblk_ret, int *position, + uint32_t *elem_index) +#else static int do_btree_posi_find_with_get(btree_meta_info *info, const int bkrtype, const bkey_range *bkrange, ENGINE_BTREE_ORDER order, const int count, btree_elem_item **elem_array, uint32_t *elem_count, uint32_t *elem_index) +#endif { btree_elem_posi path[BTREE_MAX_DEPTH]; btree_elem_item *elem; int bpos, ecnt, eidx; +#ifdef USE_EBLOCK_RESULT + if (info->root == NULL) return ENGINE_ELEM_ENOENT; +#else if (info->root == NULL) return -1; /* not found */ +#endif elem = do_btree_find_first(info->root, bkrtype, bkrange, path, true); if (elem != NULL) { @@ -4902,6 +4960,23 @@ static int do_btree_posi_find_with_get(btree_meta_info *info, ecnt = 1; /* elem count */ eidx = (bpos < count) ? bpos : count; /* elem index in elem array */ +#ifdef USE_EBLOCK_RESULT + if (!eblk_prepare(eblk_ret, (eidx + count + 1))) + return ENGINE_ENOMEM; + + elem->refcount++; + eblk_add_elem_with_posi(eblk_ret, elem, eidx); + if (order == BTREE_ORDER_ASC) { + ecnt += do_btree_elem_batch_get(path[0], eidx, false, true, eblk_ret); + assert((ecnt-1) == eidx); + ecnt += do_btree_elem_batch_get(path[0], count, true, false, eblk_ret); + } else { + ecnt += do_btree_elem_batch_get(path[0], eidx, true, true, eblk_ret); + assert((ecnt-1) == eidx); + ecnt += do_btree_elem_batch_get(path[0], count, false, false, eblk_ret); + } + eblk_truncate(eblk_ret); +#else elem->refcount++; elem_array[eidx] = elem; @@ -4915,16 +4990,27 @@ static int do_btree_posi_find_with_get(btree_meta_info *info, ecnt += do_btree_elem_batch_get(path[0], count, false, false, &elem_array[eidx+1]); } *elem_count = (uint32_t)ecnt; +#endif *elem_index = (uint32_t)eidx; } else { bpos = -1; /* not found */ } +#ifdef USE_EBLOCK_RESULT + *position = bpos; + if (bpos < 0) return ENGINE_ELEM_ENOENT; + else return ENGINE_SUCCESS; +#else return bpos; /* btree_position */ +#endif } static ENGINE_ERROR_CODE do_btree_elem_get_by_posi(btree_meta_info *info, const int index, const uint32_t count, const bool forward, +#ifdef USE_EBLOCK_RESULT + eblock_result_t *eblk_ret) +#else btree_elem_item **elem_array, uint32_t *elem_count) +#endif { btree_elem_posi posi; btree_indx_node *node; @@ -4934,6 +5020,10 @@ static ENGINE_ERROR_CODE do_btree_elem_get_by_posi(btree_meta_info *info, if (info->root == NULL) return ENGINE_ELEM_ENOENT; +#ifdef USE_EBLOCK_RESULT + if (!eblk_prepare(eblk_ret, (count < info->ccnt) ? count : info->ccnt)) + return ENGINE_ENOMEM; +#endif node = info->root; tot_ecnt = 0; while (node->ndepth > 0) { @@ -4951,12 +5041,23 @@ static ENGINE_ERROR_CODE do_btree_elem_get_by_posi(btree_meta_info *info, elem = BTREE_GET_ELEM_ITEM(posi.node, posi.indx); elem->refcount++; +#ifdef USE_EBLOCK_RESULT + eblk_add_elem(eblk_ret, elem); + nfound = 1; + nfound += do_btree_elem_batch_get(posi, count-1, forward, false, eblk_ret); +#else elem_array[0] = elem; nfound = 1; nfound += do_btree_elem_batch_get(posi, count-1, forward, false, &elem_array[nfound]); +#endif +#ifdef USE_EBLOCK_RESULT + eblk_truncate(eblk_ret); + if (nfound > 0) { +#else *elem_count = nfound; if (*elem_count > 0) { +#endif return ENGINE_SUCCESS; } else { return ENGINE_ELEM_ENOENT; @@ -5065,7 +5166,11 @@ static bool do_btree_smget_check_trim(smget_result_t *smres) static void do_btree_smget_adjust_trim(smget_result_t *smres) { +#ifdef USE_EBLOCK_RESULT + eitem **new_trim_elems = smres->trim_elems; +#else eitem **new_trim_elems = &smres->elem_array[smres->elem_count]; +#endif smget_emis_t *new_trim_kinfo = &smres->miss_kinfo[smres->miss_count]; uint32_t new_trim_count = 0; btree_elem_item *tail_elem = NULL; @@ -5080,7 +5185,11 @@ static void do_btree_smget_adjust_trim(smget_result_t *smres) * we might trim the trimmed keys if the bkey-before-trim is behind * the bkey of the last found element. */ +#ifdef USE_EBLOCK_RESULT + tail_elem = (btree_elem_item *)EBLOCK_ELEM_LAST(smres->eblk_ret); +#else tail_elem = smres->elem_array[smres->elem_count-1]; +#endif } for (idx = smres->trim_count-1; idx >= 0; idx--) @@ -5141,6 +5250,10 @@ static void do_btree_smget_adjust_trim(smget_result_t *smres) smres->trim_elems = new_trim_elems; smres->trim_kinfo = new_trim_kinfo; smres->trim_count = new_trim_count; +#ifdef USE_EBLOCK_RESULT + for (i = 0; i < smres->trim_count; i++) + eblk_add_elem(smres->eblk_ret, smres->trim_elems[i]); +#endif } #ifdef JHPARK_OLD_SMGET_INTERFACE @@ -5626,8 +5739,13 @@ do_btree_smget_scan_sort(struct default_engine *engine, static ENGINE_ERROR_CODE do_btree_smget_elem_sort_old(btree_scan_info *btree_scan_buf, uint16_t *sort_sindx_buf, const int sort_sindx_cnt, const int bkrtype, const bkey_range *bkrange, const eflag_filter *efilter, +#ifdef USE_EBLOCK_RESULT + const uint32_t offset, const uint32_t count, eblock_result_t *eblk_ret, + uint32_t *kfnd_array, uint32_t *flag_array, +#else const uint32_t offset, const uint32_t count, btree_elem_item **elem_array, uint32_t *kfnd_array, uint32_t *flag_array, uint32_t *elem_count, +#endif bool *potentialbkeytrim, bool *bkey_duplicated) { btree_meta_info *info; @@ -5641,13 +5759,21 @@ static ENGINE_ERROR_CODE do_btree_smget_elem_sort_old(btree_scan_info *btree_sca int i, cmp_res; int mid, left, right; int skip_count = 0; +#ifdef USE_EBLOCK_RESULT + int elem_count = 0; +#endif int sort_count = sort_sindx_cnt; bool ascending = (bkrtype != BKEY_RANGE_TYPE_DSC ? true : false); #ifdef FIX_SMGET_BUG bool key_trim_found = false; bool dup_bkey_found; #endif +#ifdef USE_EBLOCK_RESULT + if (!eblk_prepare(eblk_ret, count)) + return ENGINE_ENOMEM; +#else *elem_count = 0; +#endif while (sort_count > 0) { curr_idx = sort_sindx_buf[first_idx]; @@ -5670,16 +5796,28 @@ static ENGINE_ERROR_CODE do_btree_smget_elem_sort_old(btree_scan_info *btree_sca skip_count++; } else { /* skip_count == offset */ #ifdef FIX_SMGET_BUG +#ifdef USE_EBLOCK_RESULT + if (elem_count > 0 && dup_bkey_found) { +#else if (*elem_count > 0 && dup_bkey_found) { +#endif *bkey_duplicated = true; } #endif elem->refcount++; +#ifdef USE_EBLOCK_RESULT + eblk_add_elem(eblk_ret, elem); + kfnd_array[elem_count] = btree_scan_buf[curr_idx].kidx; + flag_array[elem_count] = btree_scan_buf[curr_idx].it->flags; + elem_count += 1; + if (elem_count >= count) break; +#else elem_array[*elem_count] = elem; kfnd_array[*elem_count] = btree_scan_buf[curr_idx].kidx; flag_array[*elem_count] = btree_scan_buf[curr_idx].it->flags; *elem_count += 1; if (*elem_count >= count) break; +#endif } scan_next: @@ -5749,10 +5887,17 @@ static ENGINE_ERROR_CODE do_btree_smget_elem_sort_old(btree_scan_info *btree_sca sort_sindx_buf[right] = curr_idx; } #ifdef FIX_SMGET_BUG +#ifdef USE_EBLOCK_RESULT + if (key_trim_found && elem_count < count) { +#else if (key_trim_found && *elem_count < count) { +#endif *potentialbkeytrim = true; } #endif +#ifdef USE_EBLOCK_RESULT + eblk_truncate(eblk_ret); +#endif return ENGINE_SUCCESS; } @@ -5785,6 +5930,10 @@ do_btree_smget_elem_sort(btree_scan_info *btree_scan_buf, #ifdef FIX_SMGET_BUG bool dup_bkey_found; #endif +#ifdef USE_EBLOCK_RESULT + if (!eblk_prepare(smres->eblk_ret, smres->elem_arrsz + smres->keys_arrsz)) /* elem_count + trim_count */ + return ENGINE_ENOMEM; +#endif while (sort_count > 0) { curr_idx = sort_sindx_buf[first_idx]; @@ -5812,7 +5961,11 @@ do_btree_smget_elem_sort(btree_scan_info *btree_scan_buf, smres->duplicated = true; } #endif +#ifdef USE_EBLOCK_RESULT + eblk_add_elem(smres->eblk_ret, elem); +#else smres->elem_array[smres->elem_count] = elem; +#endif smres->elem_kinfo[smres->elem_count].kidx = btree_scan_buf[curr_idx].kidx; smres->elem_kinfo[smres->elem_count].flag = btree_scan_buf[curr_idx].it->flags; smres->elem_count += 1; @@ -5906,6 +6059,9 @@ do_btree_smget_elem_sort(btree_scan_info *btree_scan_buf, do_btree_smget_adjust_trim(smres); } } +#ifdef USE_EBLOCK_RESULT + eblk_truncate(smres->eblk_ret); +#endif return ret; } #endif @@ -7027,6 +7183,13 @@ btree_elem_item *btree_elem_alloc(struct default_engine *engine, return elem; } +#ifdef USE_EBLOCK_RESULT +void btree_elem_release(struct default_engine *engine, + eitem *eitem, EITEM_TYPE type) +{ + do_coll_elem_release(engine, eitem, type, do_btree_elem_release); +} +#else void btree_elem_release(struct default_engine *engine, btree_elem_item **elem_array, const int elem_count) { @@ -7041,6 +7204,7 @@ void btree_elem_release(struct default_engine *engine, } pthread_mutex_unlock(&engine->cache_lock); } +#endif ENGINE_ERROR_CODE btree_elem_insert(struct default_engine *engine, const char *key, const size_t nkey, @@ -7219,7 +7383,11 @@ ENGINE_ERROR_CODE btree_elem_get(struct default_engine *engine, const bkey_range *bkrange, const eflag_filter *efilter, const uint32_t offset, const uint32_t req_count, const bool delete, const bool drop_if_empty, +#ifdef USE_EBLOCK_RESULT + eblock_result_t *eblk_ret, +#else btree_elem_item **elem_array, uint32_t *elem_count, +#endif uint32_t *access_count, uint32_t *flags, bool *dropped_trimmed) { @@ -7244,9 +7412,15 @@ ENGINE_ERROR_CODE btree_elem_get(struct default_engine *engine, (info->bktype == BKEY_TYPE_BINARY && bkrange->from_nbkey == 0)) { ret = ENGINE_EBADBKEY; break; } +#ifdef USE_EBLOCK_RESULT + ret = do_btree_elem_get(engine, info, bkrtype, bkrange, efilter, + offset, req_count, delete, + eblk_ret, access_count, &potentialbkeytrim); +#else ret = do_btree_elem_get(engine, info, bkrtype, bkrange, efilter, offset, req_count, delete, elem_array, elem_count, access_count, &potentialbkeytrim); +#endif if (ret == ENGINE_SUCCESS) { if (delete) { if (info->ccnt == 0 && drop_if_empty) { @@ -7263,7 +7437,11 @@ ENGINE_ERROR_CODE btree_elem_get(struct default_engine *engine, } else { if (potentialbkeytrim == true) ret = ENGINE_EBKEYOOR; +#ifdef USE_EBLOCK_RESULT + /* ret = ENGINE_ENOENT or ENGINE_ELEM_ENOENT; */ +#else /* ret = ENGINE_ELEM_ENOENT; */ +#endif } } while (0); do_item_release(engine, it); @@ -7344,7 +7522,11 @@ ENGINE_ERROR_CODE btree_posi_find_with_get(struct default_engine *engine, const char *key, const size_t nkey, const bkey_range *bkrange, ENGINE_BTREE_ORDER order, const int count, int *position, +#ifdef USE_EBLOCK_RESULT + eblock_result_t *eblk_ret, +#else btree_elem_item **elem_array, uint32_t *elem_count, +#endif uint32_t *elem_index, uint32_t *flags) { hash_item *it; @@ -7369,11 +7551,17 @@ ENGINE_ERROR_CODE btree_posi_find_with_get(struct default_engine *engine, (info->bktype == BKEY_TYPE_BINARY && bkrange->from_nbkey == 0)) { ret = ENGINE_EBADBKEY; break; } +#ifdef USE_EBLOCK_RESULT + ret = do_btree_posi_find_with_get(info, bkrtype, bkrange, order, count, + eblk_ret, position, elem_index); + if (ret != ENGINE_SUCCESS) break; /* ENGINE_ELEM_ENOENT or ENGINE_ENOMEM */ +#else *position = do_btree_posi_find_with_get(info, bkrtype, bkrange, order, count, elem_array, elem_count, elem_index); if (*position < 0) { ret = ENGINE_ELEM_ENOENT; break; } +#endif *flags = it->flags; } while (0); do_item_release(engine, it); @@ -7385,7 +7573,11 @@ ENGINE_ERROR_CODE btree_posi_find_with_get(struct default_engine *engine, ENGINE_ERROR_CODE btree_elem_get_by_posi(struct default_engine *engine, const char *key, const size_t nkey, ENGINE_BTREE_ORDER order, uint32_t from_posi, uint32_t to_posi, +#ifdef USE_EBLOCK_RESULT + eblock_result_t *eblk_ret, uint32_t *flags) +#else btree_elem_item **elem_array, uint32_t *elem_count, uint32_t *flags) +#endif { hash_item *it; btree_meta_info *info; @@ -7421,9 +7613,15 @@ ENGINE_ERROR_CODE btree_elem_get_by_posi(struct default_engine *engine, forward = false; rqcount = from_posi - to_posi + 1; } +#ifdef USE_EBLOCK_RESULT + ret = do_btree_elem_get_by_posi(info, from_posi, rqcount, forward, eblk_ret); + if (ret != ENGINE_SUCCESS) /* ret = ENGINE_ELEM_ENOENT or ENGINE_ELEM_ENOENT */ + break; +#else ret = do_btree_elem_get_by_posi(info, from_posi, rqcount, forward, elem_array, elem_count); if (ret != ENGINE_SUCCESS) /* ret == ENGINE_ELEM_ENOENT */ break; +#endif *flags = it->flags; } while (0); do_item_release(engine, it); @@ -7438,8 +7636,13 @@ ENGINE_ERROR_CODE btree_elem_smget_old(struct default_engine *engine, token_t *key_array, const int key_count, const bkey_range *bkrange, const eflag_filter *efilter, const uint32_t offset, const uint32_t count, +#ifdef USE_EBLOCK_RESULT + eblock_result_t *eblk_ret, uint32_t *kfnd_array, + uint32_t *flag_array, +#else btree_elem_item **elem_array, uint32_t *kfnd_array, uint32_t *flag_array, uint32_t *elem_count, +#endif uint32_t *missed_key_array, uint32_t *missed_key_count, bool *trimmed, bool *duplicated) { @@ -7473,10 +7676,17 @@ ENGINE_ERROR_CODE btree_elem_smget_old(struct default_engine *engine, #endif if (ret == ENGINE_SUCCESS) { /* the 2nd phase: get the sorted elems */ +#ifdef USE_EBLOCK_RESULT + ret = do_btree_smget_elem_sort_old(btree_scan_buf, sort_sindx_buf, sort_sindx_cnt, + bkrtype, bkrange, efilter, offset, count, + eblk_ret, kfnd_array, flag_array, + trimmed, duplicated); +#else ret = do_btree_smget_elem_sort_old(btree_scan_buf, sort_sindx_buf, sort_sindx_cnt, bkrtype, bkrange, efilter, offset, count, elem_array, kfnd_array, flag_array, elem_count, trimmed, duplicated); +#endif for (i = 0; i <= (offset+count); i++) { if (btree_scan_buf[i].it != NULL) do_item_release(engine, btree_scan_buf[i].it); @@ -7508,10 +7718,15 @@ ENGINE_ERROR_CODE btree_elem_smget(struct default_engine *engine, } /* initialize smget result structure */ +#ifdef USE_EBLOCK_RESULT + /* trim_elems, elem_kinfo, miss_kinfo, already init */ + assert(result->trim_elems != NULL); +#else assert(result->elem_array != NULL); result->trim_elems = (eitem *)&result->elem_array[count]; result->elem_kinfo = (smget_ehit_t *)&result->elem_array[count + key_count]; result->miss_kinfo = (smget_emis_t *)&result->elem_kinfo[count]; +#endif result->trim_kinfo = result->miss_kinfo; result->elem_count = 0; result->miss_count = 0; @@ -7840,25 +8055,27 @@ static void do_coll_elem_release(struct default_engine *engine, eitem *item, EIT pthread_mutex_unlock(&engine->cache_lock); } else if (type == EITEM_TYPE_BLOCK) { eblock_result_t *eblk_ret = (eblock_result_t *)item; - uint32_t elem_count = EBLOCK_ELEM_COUNT(eblk_ret); - uint32_t lock_count, i, j; - eblock_scan_t eblk_sc; - eitem *elem; - - EBLOCK_SCAN_INIT(eblk_ret, &eblk_sc); - for (i = 0; i < elem_count; i += lock_count) { - lock_count = (elem_count-i) < 100 - ? (elem_count-i) : 100; - pthread_mutex_lock(&engine->cache_lock); - for (j = 0; j < lock_count; j++) { - EBLOCK_SCAN_NEXT(&eblk_sc, elem); - elem_release_func(engine, elem); + if (eblk_ret->head_blk != NULL) { //validate check + uint32_t elem_count = EBLOCK_ELEM_COUNT(eblk_ret); + uint32_t lock_count, i, j; + eblock_scan_t eblk_sc; + eitem *elem; + + EBLOCK_SCAN_INIT(eblk_ret, &eblk_sc); + for (i = 0; i < elem_count; i += lock_count) { + lock_count = (elem_count-i) < 100 + ? (elem_count-i) : 100; + pthread_mutex_lock(&engine->cache_lock); + for (j = 0; j < lock_count; j++) { + EBLOCK_SCAN_NEXT(&eblk_sc, elem); + elem_release_func(engine, elem); + } + pthread_mutex_unlock(&engine->cache_lock); } + pthread_mutex_lock(&engine->cache_lock); + mblock_list_free(eblk_ret->blck_cnt, &eblk_ret->head_blk, &eblk_ret->tail_blk); pthread_mutex_unlock(&engine->cache_lock); } - pthread_mutex_lock(&engine->cache_lock); - mblock_list_free(eblk_ret->blck_cnt, eblk_ret->head_blk, eblk_ret->tail_blk); - pthread_mutex_unlock(&engine->cache_lock); } } #endif diff --git a/engines/default/items.h b/engines/default/items.h index 5929e687d..e215102db 100644 --- a/engines/default/items.h +++ b/engines/default/items.h @@ -530,7 +530,11 @@ btree_elem_item *btree_elem_alloc(struct default_engine *engine, const void *cookie); void btree_elem_release(struct default_engine *engine, +#ifdef USE_EBLOCK_RESULT + eitem *eitem, EITEM_TYPE type); +#else btree_elem_item **elem_array, const int elem_count); +#endif ENGINE_ERROR_CODE btree_elem_insert(struct default_engine *engine, const char *key, const size_t nkey, @@ -562,7 +566,11 @@ ENGINE_ERROR_CODE btree_elem_get(struct default_engine *engine, const bkey_range *bkrange, const eflag_filter *efilter, const uint32_t offset, const uint32_t req_count, const bool delete, const bool drop_if_empty, +#ifdef USE_EBLOCK_RESULT + eblock_result_t *eblk_ret, +#else btree_elem_item **elem_array, uint32_t *elem_count, +#endif uint32_t *access_count, uint32_t *flags, bool *dropped_trimmed); @@ -579,13 +587,21 @@ ENGINE_ERROR_CODE btree_posi_find_with_get(struct default_engine *engine, const char *key, const size_t nkey, const bkey_range *bkrange, ENGINE_BTREE_ORDER order, const int count, int *position, +#ifdef USE_EBLOCK_RESULT + eblock_result_t *eblk_ret, +#else btree_elem_item **elem_array, uint32_t *elem_count, +#endif uint32_t *elem_index, uint32_t *flags); ENGINE_ERROR_CODE btree_elem_get_by_posi(struct default_engine *engine, const char *key, const size_t nkey, ENGINE_BTREE_ORDER order, uint32_t from_posi, uint32_t to_posi, +#ifdef USE_EBLOCK_RESULT + eblock_result_t *eblk_ret, uint32_t *flags); +#else btree_elem_item **elem_array, uint32_t *elem_count, uint32_t *flags); +#endif #ifdef SUPPORT_BOP_SMGET #ifdef JHPARK_OLD_SMGET_INTERFACE @@ -594,8 +610,13 @@ ENGINE_ERROR_CODE btree_elem_smget_old(struct default_engine *engine, token_t *key_array, const int key_count, const bkey_range *bkrange, const eflag_filter *efilter, const uint32_t offset, const uint32_t count, +#ifdef USE_EBLOCK_RESULT + eblock_result_t *eblk_ret, uint32_t *kfnd_array, + uint32_t *flag_array, +#else btree_elem_item **elem_array, uint32_t *kfnd_array, uint32_t *flag_array, uint32_t *elem_count, +#endif uint32_t *missed_key_array, uint32_t *missed_key_count, bool *trimmed, bool *duplicated); #endif diff --git a/engines/demo/demo_engine.c b/engines/demo/demo_engine.c index 3643f9651..59d265e4f 100644 --- a/engines/demo/demo_engine.c +++ b/engines/demo/demo_engine.c @@ -511,12 +511,21 @@ Demo_btree_elem_alloc(ENGINE_HANDLE* handle, const void* cookie, return ENGINE_ENOTSUP; } +#ifdef USE_EBLOCK_RESULT +static void +Demo_btree_elem_release(ENGINE_HANDLE* handle, const void *cookie, + eitem *eitem, EITEM_TYPE type) +{ + return; +} +#else static void Demo_btree_elem_release(ENGINE_HANDLE* handle, const void *cookie, eitem **eitem_array, const int eitem_count) { return; } +#endif static ENGINE_ERROR_CODE Demo_btree_elem_insert(ENGINE_HANDLE* handle, const void* cookie, @@ -567,7 +576,11 @@ Demo_btree_elem_get(ENGINE_HANDLE* handle, const void* cookie, const bkey_range *bkrange, const eflag_filter *efilter, const uint32_t offset, const uint32_t req_count, const bool delete, const bool drop_if_empty, +#ifdef USE_EBLOCK_RESULT + eblock_result_t *eblk_ret, +#else eitem** eitem_array, uint32_t* eitem_count, +#endif uint32_t *access_count, uint32_t* flags, bool* dropped_trimmed, uint16_t vbucket) { @@ -599,8 +612,13 @@ Demo_btree_posi_find_with_get(ENGINE_HANDLE* handle, const void* cookie, const char *key, const size_t nkey, const bkey_range *bkrange, ENGINE_BTREE_ORDER order, const uint32_t count, +#ifdef USE_EBLOCK_RESULT + int *position, eblock_result_t *eblk_ret, + uint32_t *eitem_index, +#else int *position, eitem **eitem_array, uint32_t *eitem_count, uint32_t *eitem_index, +#endif uint32_t *flags, uint16_t vbucket) { return ENGINE_ENOTSUP; @@ -611,7 +629,11 @@ Demo_btree_elem_get_by_posi(ENGINE_HANDLE* handle, const void* cookie, const char *key, const size_t nkey, ENGINE_BTREE_ORDER order, uint32_t from_posi, uint32_t to_posi, +#ifdef USE_EBLOCK_RESULT + eblock_result_t *eblk_ret, +#else eitem **eitem_array, uint32_t *eitem_count, +#endif uint32_t *flags, uint16_t vbucket) { return ENGINE_ENOTSUP; diff --git a/include/memcached/engine.h b/include/memcached/engine.h index 3381f4f7a..507adf73a 100644 --- a/include/memcached/engine.h +++ b/include/memcached/engine.h @@ -510,7 +510,11 @@ extern "C" { const size_t nbytes, eitem** eitem); void (*btree_elem_release)(ENGINE_HANDLE* handle, const void *cookie, +#ifdef USE_EBLOCK_RESULT + eitem *eitem, EITEM_TYPE type); +#else eitem **eitem_array, const int eitem_count); +#endif ENGINE_ERROR_CODE (*btree_elem_insert)(ENGINE_HANDLE* handle, const void* cookie, const void* key, const int nkey, @@ -551,7 +555,11 @@ extern "C" { const uint32_t offset, const uint32_t req_count, const bool delete, const bool drop_if_empty, +#ifdef USE_EBLOCK_RESULT + eblock_result_t *eblk_ret, +#else eitem** eitem_array, uint32_t* eitem_count, +#endif uint32_t* access_count, uint32_t* flags, bool* dropped_trimmed, uint16_t vbucket); @@ -573,15 +581,24 @@ extern "C" { const bkey_range *bkrange, ENGINE_BTREE_ORDER order, const uint32_t count, +#ifdef USE_EBLOCK_RESULT + int *position, eblock_result_t *eblk_ret, + uint32_t *eitem_index, +#else int *position, eitem **eitem_array, uint32_t *eitem_count, uint32_t *eitem_index, +#endif uint32_t *flags, uint16_t vbucket); ENGINE_ERROR_CODE (*btree_elem_get_by_posi)(ENGINE_HANDLE *handle, const void* cookie, const char *key, const size_t nkey, ENGINE_BTREE_ORDER order, uint32_t from_posi, uint32_t to_posi, +#ifdef USE_EBLOCK_RESULT + eblock_result_t *eblk_ret, +#else eitem **eitem_array, uint32_t *eitem_count, +#endif uint32_t *flags, uint16_t vbucket); #ifdef SUPPORT_BOP_SMGET @@ -592,10 +609,16 @@ extern "C" { const bkey_range *bkrange, const eflag_filter *efilter, const uint32_t offset, const uint32_t count, +#ifdef USE_EBLOCK_RESULT + eblock_result_t *eblk_ret, + uint32_t* kfnd_array, + uint32_t* flag_array, +#else eitem** eitem_array, uint32_t* kfnd_array, uint32_t* flag_array, uint32_t* eitem_count, +#endif uint32_t* missed_key_array, uint32_t* missed_key_count, bool *trimmed, bool *duplicated, diff --git a/include/memcached/types.h b/include/memcached/types.h index e1d74547a..5ec026cca 100644 --- a/include/memcached/types.h +++ b/include/memcached/types.h @@ -326,6 +326,16 @@ extern "C" { uint32_t flag; /* item flags */ } smget_ehit_t; +#ifdef USE_EBLOCK_RESULT + typedef struct _eblock_result_t { + struct _mem_block_t *head_blk; /* head block pointer */ + struct _mem_block_t *tail_blk; /* tail block pointer */ + struct _mem_block_t *last_blk; /* last block pointer */ + uint32_t elem_cnt; /* total element count */ + uint32_t blck_cnt; /* block count */ + } eblock_result_t; +#endif + /* Key info of the missed/trimmed keys in smget */ typedef struct { uint16_t kidx; /* key index in keys array */ @@ -334,7 +344,11 @@ extern "C" { /* smget result structure */ typedef struct { +#ifdef USE_EBLOCK_RESULT + eblock_result_t *eblk_ret; /* found elements in smget */ +#else eitem **elem_array; /* found elements in smget */ +#endif smget_ehit_t *elem_kinfo; /* key info of found elements */ smget_emis_t *miss_kinfo; /* key info of missed keys */ smget_emis_t *trim_kinfo; /* key info of trimmed keys */ @@ -382,8 +396,11 @@ extern "C" { #ifdef USE_EBLOCK_RESULT #define EITEMS_PER_BLOCK 1023 +#define EITEMS_BLCK_SIZE sizeof(mem_block_t) +#define EBLOCK_ELEM_LAST(b) ((b)->tail_blk->items[(EBLOCK_ELEM_COUNT(b) - 1) % EITEMS_PER_BLOCK]) #define EBLOCK_ELEM_COUNT(b) ((b)->elem_cnt) + #define EBLOCK_SCAN_INIT(b, s) \ do { \ (s)->blk = (b)->head_blk; \ @@ -391,12 +408,23 @@ extern "C" { (s)->idx = 0; \ } while(0) \ +#define EBLOCK_SCAN_RESET(f, b, s) \ + do { \ + if ((f) == false) { \ + (s)->blk = (b)->head_blk; \ + (s)->idx = 0; \ + (f) = true; \ + } \ + (s)->tot = (b)->elem_cnt; \ + } while(0) \ + #define EBLOCK_SCAN_NEXT(s, e) \ do { \ if ((s)->idx < (s)->tot) { \ - (e) = ((s)->blk)->items[(s)->idx % EITEMS_PER_BLOCK]; \ - if ((((s)->idx)++ % EITEMS_PER_BLOCK) == (EITEMS_PER_BLOCK - 1)) \ + if (((s)->idx % EITEMS_PER_BLOCK) == 0 && ((s)->idx != 0)) { \ (s)->blk = ((s)->blk)->next; \ + } \ + (e) = ((s)->blk)->items[(s)->idx++ % EITEMS_PER_BLOCK]; \ } else { \ (e) = NULL; \ } \ @@ -412,14 +440,6 @@ extern "C" { struct _mem_block_t *next; } mem_block_t; - typedef struct _eblock_result_t { - struct _mem_block_t *head_blk; /* head block pointer */ - struct _mem_block_t *tail_blk; /* tail block pointer */ - struct _mem_block_t *last_blk; /* last block pointer */ - uint32_t elem_cnt; /* element count */ - uint32_t blck_cnt; /* block count */ - } eblock_result_t; - typedef struct _eblock_scan_t { mem_block_t *blk; /* current block pointer */ uint32_t tot; /* total element count */ diff --git a/memcached.c b/memcached.c index 73b4fe403..e999b497b 100644 --- a/memcached.c +++ b/memcached.c @@ -807,7 +807,11 @@ static void conn_coll_eitem_free(conn *c) { /* bop */ case OPERATION_BOP_INSERT: case OPERATION_BOP_UPSERT: +#ifdef USE_EBLOCK_RESULT + mc_engine.v1->btree_elem_release(mc_engine.v0, c, c->coll_eitem, EITEM_TYPE_SINGLE); +#else mc_engine.v1->btree_elem_release(mc_engine.v0, c, &c->coll_eitem, 1); +#endif break; case OPERATION_BOP_UPDATE: if (c->coll_eitem != NULL) @@ -816,8 +820,12 @@ static void conn_coll_eitem_free(conn *c) { case OPERATION_BOP_GET: case OPERATION_BOP_PWG: /* position with get */ case OPERATION_BOP_GBP: /* get by position */ +#ifdef USE_EBLOCK_RESULT + mc_engine.v1->btree_elem_release(mc_engine.v0, c, c->coll_eitem, EITEM_TYPE_BLOCK); +#else mc_engine.v1->btree_elem_release(mc_engine.v0, c, c->coll_eitem, c->coll_ecount); free(c->coll_eitem); +#endif if (c->coll_resps != NULL) { free(c->coll_resps); c->coll_resps = NULL; } @@ -829,8 +837,15 @@ static void conn_coll_eitem_free(conn *c) { #ifdef SUPPORT_BOP_SMGET case OPERATION_BOP_SMGET: #endif +#ifdef USE_EBLOCK_RESULT + mc_engine.v1->btree_elem_release(mc_engine.v0, c, c->coll_eitem, EITEM_TYPE_BLOCK); + if (c->coll_resps != NULL) { + free(c->coll_resps); c->coll_resps = NULL; + } +#else mc_engine.v1->btree_elem_release(mc_engine.v0, c, c->coll_eitem, c->coll_ecount); free(c->coll_eitem); +#endif break; #endif default: @@ -2119,9 +2134,9 @@ static void process_mop_get_complete(conn *c) bool drop_if_empty = c->coll_drop; bool dropped; #ifdef USE_EBLOCK_RESULT - int i, need_size = 0; + int i; #else - int need_size = 0; + int i, need_size = 0; if (c->coll_numkeys <= 0 || c->coll_numkeys > MAX_MAP_SIZE) { need_size = MAX_MAP_SIZE * sizeof(eitem*); @@ -2371,7 +2386,11 @@ static void process_bop_insert_complete(conn *c) { if (strncmp((char*)info.value + info.nbytes - 2, "\r\n", 2) != 0) { // release the btree element +#ifdef USE_EBLOCK_RESULT + mc_engine.v1->btree_elem_release(mc_engine.v0, c, c->coll_eitem, EITEM_TYPE_SINGLE); +#else mc_engine.v1->btree_elem_release(mc_engine.v0, c, &c->coll_eitem, 1); +#endif c->coll_eitem = NULL; out_string(c, "CLIENT_ERROR bad data chunk"); } else { @@ -2391,7 +2410,11 @@ static void process_bop_insert_complete(conn *c) { } // release the btree element inserted. +#ifdef USE_EBLOCK_RESULT + mc_engine.v1->btree_elem_release(mc_engine.v0, c, c->coll_eitem, EITEM_TYPE_SINGLE); +#else mc_engine.v1->btree_elem_release(mc_engine.v0, c, &c->coll_eitem, 1); +#endif c->coll_eitem = NULL; if (settings.detail_enabled) { @@ -2421,15 +2444,23 @@ static void process_bop_insert_complete(conn *c) { (add_iov(c, info.value, info.nbytes) != 0) || (add_iov(c, "TRIMMED\r\n", strlen("TRIMMED\r\n")) != 0)) { +#ifdef USE_EBLOCK_RESULT + mc_engine.v1->btree_elem_release(mc_engine.v0, c, + &trim_result.elems, EITEM_TYPE_SINGLE); +#else mc_engine.v1->btree_elem_release(mc_engine.v0, c, &trim_result.elems, trim_result.count); +#endif if (c->ewouldblock) c->ewouldblock = false; out_string(c, "SERVER_ERROR out of memory writing get response"); } else { /* prepare for writing response */ c->coll_eitem = trim_result.elems; +#ifdef USE_EBLOCK_RESULT +#else c->coll_ecount = trim_result.count; /* trim_result.count == 1 */ +#endif conn_set_state(c, conn_mwrite); } } else { @@ -2536,10 +2567,17 @@ static void process_bop_update_complete(conn *c) #ifdef SUPPORT_BOP_MGET static void process_bop_mget_complete(conn *c) { assert(c->coll_op == OPERATION_BOP_MGET); +#ifdef USE_EBLOCK_RESULT + assert(c->coll_resps != NULL); +#else assert(c->coll_eitem != NULL); +#endif ENGINE_ERROR_CODE ret = ENGINE_SUCCESS; +#ifdef USE_EBLOCK_RESULT +#else eitem **elem_array = (eitem **)c->coll_eitem; +#endif uint32_t tot_elem_count = 0; uint32_t tot_access_count = 0; char delimiter = ' '; @@ -2583,7 +2621,14 @@ static void process_bop_mget_complete(conn *c) { bool trimmed; eitem_info info; char *resultptr; +#ifdef USE_EBLOCK_RESULT + char *valuestrp = c->coll_resps; + bool eblk_scan_init = false; + eitem *elem; + eblock_scan_t eblk_sc; +#else char *valuestrp = (char*)elem_array + (c->coll_numkeys * c->coll_rcount * sizeof(eitem*)); +#endif int resultlen; int nvaluestr; @@ -2596,7 +2641,20 @@ static void process_bop_mget_complete(conn *c) { c->coll_bkrange.to_nbkey = c->coll_bkrange.from_nbkey; } +#ifdef USE_EBLOCK_RESULT + EBLOCK_SCAN_INIT(&c->eblk_ret, &eblk_sc); +#endif for (k = 0; k < c->coll_numkeys; k++) { +#ifdef USE_EBLOCK_RESULT + ret = mc_engine.v1->btree_elem_get(mc_engine.v0, c, + key_tokens[k].value, key_tokens[k].length, + &c->coll_bkrange, + (c->coll_efilter.ncompval==0 ? NULL : &c->coll_efilter), + c->coll_roffset, c->coll_rcount, + false, false, + &c->eblk_ret, + &cur_access_count, &flags, &trimmed, 0); +#else ret = mc_engine.v1->btree_elem_get(mc_engine.v0, c, key_tokens[k].value, key_tokens[k].length, &c->coll_bkrange, @@ -2605,6 +2663,7 @@ static void process_bop_mget_complete(conn *c) { false, false, &elem_array[tot_elem_count], &cur_elem_count, &cur_access_count, &flags, &trimmed, 0); +#endif if (settings.detail_enabled) { stats_prefix_record_bop_get(key_tokens[k].value, key_tokens[k].length, @@ -2613,6 +2672,9 @@ static void process_bop_mget_complete(conn *c) { if (ret == ENGINE_SUCCESS) { do { +#ifdef USE_EBLOCK_RESULT + cur_elem_count = EBLOCK_ELEM_COUNT(&c->eblk_ret) - tot_elem_count; +#endif sprintf(resultptr, " %s %u %u\r\n", (trimmed==false ? "OK" : "TRIMMED"), htonl(flags), cur_elem_count); if ((add_iov(c, valuestrp, nvaluestr) != 0) || @@ -2623,9 +2685,18 @@ static void process_bop_mget_complete(conn *c) { } resultptr += strlen(resultptr); +#ifdef USE_EBLOCK_RESULT + EBLOCK_SCAN_RESET(eblk_scan_init, &c->eblk_ret, &eblk_sc); +#endif for (e = 0; e < cur_elem_count; e++) { +#ifdef USE_EBLOCK_RESULT + EBLOCK_SCAN_NEXT(&eblk_sc, elem); + mc_engine.v1->get_elem_info(mc_engine.v0, c, ITEM_TYPE_BTREE, + elem, &info); +#else mc_engine.v1->get_elem_info(mc_engine.v0, c, ITEM_TYPE_BTREE, elem_array[tot_elem_count+e], &info); +#endif sprintf(resultptr, "ELEMENT "); resultlen = strlen(resultptr); resultlen += make_bop_elem_response(resultptr + resultlen, &info); @@ -2691,7 +2762,11 @@ static void process_bop_mget_complete(conn *c) { if (ret != ENGINE_SUCCESS) { /* ENGINE_ENOMEM or ENGINE_DISCONNECT or SEVERE error */ /* release elements */ +#ifdef USE_EBLOCK_RESULT + mc_engine.v1->btree_elem_release(mc_engine.v0, c, &c->eblk_ret, EITEM_TYPE_BLOCK); +#else mc_engine.v1->btree_elem_release(mc_engine.v0, c, elem_array, tot_elem_count); +#endif } } @@ -2699,8 +2774,13 @@ static void process_bop_mget_complete(conn *c) { case ENGINE_SUCCESS: STATS_NOKEY2(c, cmd_bop_mget, bop_mget_oks); /* Remember this command so we can garbage collect it later */ +#ifdef USE_EBLOCK_RESULT + /* c->coll_resps = c->eblk_ret.cur_ecnt; */ + c->coll_eitem = (void *)&c->eblk_ret; +#else /* c->coll_eitem = (void *)elem_array; */ c->coll_ecount = tot_elem_count; +#endif c->coll_op = OPERATION_BOP_MGET; conn_set_state(c, conn_mwrite); c->msgcurr = 0; @@ -2734,10 +2814,17 @@ static void process_bop_mget_complete(conn *c) { #endif c->coll_strkeys = NULL; } +#ifdef USE_EBLOCK_RESULT + if (c->coll_resps != NULL) { + free((void *)c->coll_resps); + c->coll_resps = NULL; + } +#else if (c->coll_eitem != NULL) { free((void *)c->coll_eitem); c->coll_eitem = NULL; } +#endif } } #endif @@ -2790,8 +2877,12 @@ static void process_bop_smget_complete_old(conn *c) { uint32_t kmis_count = 0; uint32_t elem_count = 0; +#ifdef USE_EBLOCK_RESULT + uint32_t *kfnd_array = (uint32_t*)c->coll_resps; +#else eitem **elem_array = (eitem **)c->coll_eitem; uint32_t *kfnd_array = (uint32_t*)((char*)elem_array + (smget_count*sizeof(eitem*))); +#endif uint32_t *flag_array = (uint32_t*)((char*)kfnd_array + (smget_count*sizeof(uint32_t))); uint32_t *kmis_array = (uint32_t*)((char*)flag_array + (smget_count*sizeof(uint32_t))); bool trimmed; @@ -2839,14 +2930,25 @@ static void process_bop_smget_complete_old(conn *c) { &c->coll_bkrange, (c->coll_efilter.ncompval==0 ? NULL : &c->coll_efilter), c->coll_roffset, c->coll_rcount, +#ifdef USE_EBLOCK_RESULT + &c->eblk_ret, kfnd_array, flag_array, +#else elem_array, kfnd_array, flag_array, &elem_count, +#endif kmis_array, &kmis_count, &trimmed, &duplicated, 0); } +#ifdef USE_EBLOCK_RESULT + elem_count = EBLOCK_ELEM_COUNT(&c->eblk_ret); +#endif switch (ret) { case ENGINE_SUCCESS: { eitem_info info[elem_count+1]; /* elem_count might be 0. */ +#ifdef USE_EBLOCK_RESULT + eitem *elem; + eblock_scan_t eblk_sc; +#endif do { sprintf(respptr, "VALUE %u\r\n", elem_count); @@ -2855,13 +2957,22 @@ static void process_bop_smget_complete_old(conn *c) { } respptr += strlen(respptr); +#ifdef USE_EBLOCK_RESULT + EBLOCK_SCAN_INIT(&c->eblk_ret, &eblk_sc); +#endif for (i = 0; i < elem_count; i++) { idx = kfnd_array[i]; if (add_iov(c, keys_array[idx].value, keys_array[idx].length) != 0) { ret = ENGINE_ENOMEM; break; } +#ifdef USE_EBLOCK_RESULT + EBLOCK_SCAN_NEXT(&eblk_sc, elem); + mc_engine.v1->get_elem_info(mc_engine.v0, c, ITEM_TYPE_BTREE, + elem, &info[i]); +#else mc_engine.v1->get_elem_info(mc_engine.v0, c, ITEM_TYPE_BTREE, elem_array[i], &info[i]); +#endif /* flags */ sprintf(respptr, " %u ", htonl(flag_array[i])); resplen = strlen(respptr); @@ -2908,14 +3019,23 @@ static void process_bop_smget_complete_old(conn *c) { if (ret == ENGINE_SUCCESS) { STATS_NOKEY2(c, cmd_bop_smget, bop_smget_oks); /* Remember this command so we can garbage collect it later */ +#ifdef USE_EBLOCK_RESULT + /* c->coll_resps = (void *)kfnd_array; */ + c->coll_eitem = (void *)&c->eblk_ret; +#else /* c->coll_eitem = (void *)elem_array; */ c->coll_ecount = elem_count; +#endif c->coll_op = OPERATION_BOP_SMGET; conn_set_state(c, conn_mwrite); c->msgcurr = 0; } else { STATS_NOKEY(c, cmd_bop_smget); +#ifdef USE_EBLOCK_RESULT + mc_engine.v1->btree_elem_release(mc_engine.v0, c, &c->eblk_ret, EITEM_TYPE_BLOCK); +#else mc_engine.v1->btree_elem_release(mc_engine.v0, c, elem_array, elem_count); +#endif out_string(c, "SERVER_ERROR out of memory writing get response"); } } @@ -2923,6 +3043,11 @@ static void process_bop_smget_complete_old(conn *c) { case ENGINE_DISCONNECT: c->state = conn_closing; break; +#ifdef USE_EBLOCK_RESULT + case ENGINE_ENOMEM: + out_string(c, "SERVER_ERROR out of memory getting elements"); + break; +#endif default: STATS_NOKEY(c, cmd_bop_smget); if (ret == ENGINE_EBADVALUE) out_string(c, "CLIENT_ERROR bad data chunk"); @@ -2955,17 +3080,27 @@ static void process_bop_smget_complete_old(conn *c) { #endif c->coll_strkeys = NULL; } +#ifdef USE_EBLOCK_RESULT + if (kfnd_array != NULL) { + free((void *)kfnd_array); + } +#else if (c->coll_eitem != NULL) { free((void *)c->coll_eitem); c->coll_eitem = NULL; } +#endif } } #endif static void process_bop_smget_complete(conn *c) { assert(c->coll_op == OPERATION_BOP_SMGET); +#ifdef USE_EBLOCK_RESULT + assert(c->coll_resps != NULL); +#else assert(c->coll_eitem != NULL); +#endif #ifdef JHPARK_OLD_SMGET_INTERFACE if (c->coll_smgmode == 0) { process_bop_smget_complete_old(c); @@ -2988,8 +3123,14 @@ static void process_bop_smget_complete(conn *c) { int resplen; smget_result_t smres; +#ifdef USE_EBLOCK_RESULT + smres.eblk_ret = &c->eblk_ret; + smres.trim_elems = (eitem **)c->coll_resps; + smres.elem_kinfo = (smget_ehit_t *)&smres.trim_elems[c->coll_numkeys]; +#else smres.elem_array = (eitem **)c->coll_eitem; smres.elem_kinfo = (smget_ehit_t *)&smres.elem_array[c->coll_rcount+c->coll_numkeys]; +#endif smres.miss_kinfo = (smget_emis_t *)&smres.elem_kinfo[c->coll_rcount]; respptr = (char *)&smres.miss_kinfo[c->coll_numkeys]; @@ -3046,6 +3187,10 @@ static void process_bop_smget_complete(conn *c) { case ENGINE_SUCCESS: { eitem_info info[smres.elem_count+1]; /* elem_count might be 0. */ +#ifdef USE_EBLOCK_RESULT + eitem *elem; + eblock_scan_t eblk_sc; +#endif do { /* Change smget response head string: VALUE => ELEMENTS. @@ -3057,9 +3202,18 @@ static void process_bop_smget_complete(conn *c) { } respptr += strlen(respptr); +#ifdef USE_EBLOCK_RESULT + EBLOCK_SCAN_INIT(smres.eblk_ret, &eblk_sc); +#endif for (i = 0; i < smres.elem_count; i++) { +#ifdef USE_EBLOCK_RESULT + EBLOCK_SCAN_NEXT(&eblk_sc, elem); + mc_engine.v1->get_elem_info(mc_engine.v0, c, ITEM_TYPE_BTREE, + elem, &info[i]); +#else mc_engine.v1->get_elem_info(mc_engine.v0, c, ITEM_TYPE_BTREE, smres.elem_array[i], &info[i]); +#endif sprintf(respptr, " %u ", htonl(smres.elem_kinfo[i].flag)); resplen = strlen(respptr); resplen += make_bop_elem_response(respptr + resplen, &info[i]); @@ -3125,15 +3279,24 @@ static void process_bop_smget_complete(conn *c) { if (ret == ENGINE_SUCCESS) { STATS_NOKEY2(c, cmd_bop_smget, bop_smget_oks); /* Remember this command so we can garbage collect it later */ +#ifdef USE_EBLOCK_RESULT + /* c->coll_resps = (void *)smres.trim_elems; */ + c->coll_eitem = (void *)smres.eblk_ret; +#else /* c->coll_eitem = (void *)elem_array; */ c->coll_ecount = smres.elem_count+smres.trim_count; +#endif c->coll_op = OPERATION_BOP_SMGET; conn_set_state(c, conn_mwrite); c->msgcurr = 0; } else { STATS_NOKEY(c, cmd_bop_smget); +#ifdef USE_EBLOCK_RESULT + mc_engine.v1->btree_elem_release(mc_engine.v0, c, smres.eblk_ret, EITEM_TYPE_BLOCK); +#else mc_engine.v1->btree_elem_release(mc_engine.v0, c, smres.elem_array, smres.elem_count+smres.trim_count); +#endif out_string(c, "SERVER_ERROR out of memory writing get response"); } } @@ -3141,6 +3304,11 @@ static void process_bop_smget_complete(conn *c) { case ENGINE_DISCONNECT: c->state = conn_closing; break; +#ifdef USE_EBLOCK_RESULT + case ENGINE_ENOMEM: + out_string(c,"SERVER_ERROR out of memory getting elements"); + break; +#endif default: STATS_NOKEY(c, cmd_bop_smget); if (ret == ENGINE_EBADVALUE) out_string(c, "CLIENT_ERROR bad data chunk"); @@ -3175,10 +3343,17 @@ static void process_bop_smget_complete(conn *c) { #endif c->coll_strkeys = NULL; } +#ifdef USE_EBLOCK_RESULT + if (c->coll_resps != NULL) { + free((void *)c->coll_resps); + c->coll_resps = NULL; + } +#else if (c->coll_eitem != NULL) { free((void *)c->coll_eitem); c->coll_eitem = NULL; } +#endif } } #endif @@ -4757,7 +4932,10 @@ static void process_bin_lop_prepare_nread(conn *c) { c->ritem = (char *)info.value; c->rlbytes = vlen; c->coll_eitem = (void *)elem; +#ifdef USE_EBLOCK_RESULT +#else c->coll_ecount = 1; +#endif c->coll_op = OPERATION_LOP_INSERT; c->coll_key = key; c->coll_nkey = nkey; @@ -5259,7 +5437,10 @@ static void process_bin_sop_prepare_nread(conn *c) { c->rlbytes = vlen; } c->coll_eitem = (void *)elem; +#ifdef USE_EBLOCK_RESULT +#else c->coll_ecount = 1; +#endif c->coll_key = key; c->coll_nkey = nkey; if (c->cmd == PROTOCOL_BINARY_CMD_SOP_INSERT) { @@ -5831,7 +6012,10 @@ static void process_bin_bop_prepare_nread(conn *c) { c->ritem = (char *)info.value; c->rlbytes = vlen; c->coll_eitem = (void *)elem; +#ifdef USE_EBLOCK_RESULT +#else c->coll_ecount = 1; +#endif c->coll_op = (c->cmd == PROTOCOL_BINARY_CMD_BOP_INSERT ? OPERATION_BOP_INSERT : OPERATION_BOP_UPSERT); c->coll_key = key; @@ -5931,7 +6115,11 @@ static void process_bin_bop_insert_complete(conn *c) { } /* release the c->coll_eitem reference */ +#ifdef USE_EBLOCK_RESULT + mc_engine.v1->btree_elem_release(mc_engine.v0, c, c->coll_eitem, EITEM_TYPE_SINGLE); +#else mc_engine.v1->btree_elem_release(mc_engine.v0, c, &c->coll_eitem, 1); +#endif c->coll_eitem = NULL; } @@ -6064,7 +6252,10 @@ static void process_bin_bop_update_prepare_nread(conn *c) { if (req->message.body.novalue) { c->coll_eitem = NULL; +#ifdef USE_EBLOCK_RESULT +#else c->coll_ecount = 0; +#endif c->coll_key = key; c->coll_nkey = nkey; c->coll_op = OPERATION_BOP_UPDATE; @@ -6093,7 +6284,10 @@ static void process_bin_bop_update_prepare_nread(conn *c) { c->ritem = ((elem_value *)elem)->value; c->rlbytes = vlen; c->coll_eitem = (void *)elem; +#ifdef USE_EBLOCK_RESULT +#else c->coll_ecount = 1; +#endif c->coll_key = key; c->coll_nkey = nkey; c->coll_op = OPERATION_BOP_UPDATE; @@ -6231,13 +6425,19 @@ static void process_bin_bop_get(conn *c) { (req->message.body.delete ? "true" : "false")); } +#ifdef USE_EBLOCK_RESULT +#else eitem **elem_array = NULL; +#endif uint32_t elem_count; uint32_t access_count; uint32_t flags, i; bool dropped_trimmed; int est_count; +#ifdef USE_EBLOCK_RESULT +#else int need_size; +#endif ENGINE_ERROR_CODE ret = ENGINE_SUCCESS; @@ -6246,6 +6446,18 @@ static void process_bin_bop_get(conn *c) { est_count = req->message.body.count; if (est_count % 2) est_count += 1; } +#ifdef USE_EBLOCK_RESULT + ret = mc_engine.v1->btree_elem_get(mc_engine.v0, c, key, nkey, + bkrange, efilter, + req->message.body.offset, + req->message.body.count, + (bool)req->message.body.delete, + (bool)req->message.body.drop, + &c->eblk_ret, &access_count, + &flags, &dropped_trimmed, + c->binary_header.request.vbucket); + elem_count = EBLOCK_ELEM_COUNT(&c->eblk_ret); +#else need_size = est_count * (sizeof(eitem*)+MAX_BKEY_LENG+MAX_EFLAG_LENG+sizeof(uint32_t)); if ((elem_array = (eitem **)malloc(need_size)) == NULL) { write_bin_packet(c, PROTOCOL_BINARY_RESPONSE_ENOMEM, 0); @@ -6261,6 +6473,7 @@ static void process_bin_bop_get(conn *c) { elem_array, &elem_count, &access_count, &flags, &dropped_trimmed, c->binary_header.request.vbucket); +#endif if (ret == ENGINE_EWOULDBLOCK) { c->ewouldblock = true; ret = ENGINE_SUCCESS; @@ -6275,16 +6488,37 @@ static void process_bin_bop_get(conn *c) { case ENGINE_SUCCESS: { protocol_binary_response_bop_get* rsp = (protocol_binary_response_bop_get*)c->wbuf; + eitem_info info[elem_count]; +#ifdef USE_EBLOCK_RESULT + eitem *elem; + eblock_scan_t eblk_sc; + uint64_t *bkeyptr; + uint32_t *vlenptr; +#else uint64_t *bkeyptr = ((elem_count % 2) == 0 /* for 8 byte align */ ? (uint64_t *)&elem_array[elem_count] : (uint64_t *)&elem_array[elem_count+1]); uint32_t *vlenptr = (uint32_t *)((char*)bkeyptr + (sizeof(uint64_t) * elem_count)); +#endif uint32_t bodylen; +#ifdef USE_EBLOCK_RESULT + int need_size = est_count * (MAX_BKEY_LENG+MAX_EFLAG_LENG+sizeof(uint32_t)); - eitem_info info[elem_count]; + if ((bkeyptr = (uint64_t *)malloc(need_size)) == NULL) { + ret = ENGINE_ENOMEM; + } else { + vlenptr = (uint32_t *)((char*)bkeyptr + (sizeof(uint64_t) * elem_count)); + EBLOCK_SCAN_INIT(&c->eblk_ret, &eblk_sc); +#endif for (i = 0; i < elem_count; i++) { +#ifdef USE_EBLOCK_RESULT + EBLOCK_SCAN_NEXT(&eblk_sc, elem); + mc_engine.v1->get_elem_info(mc_engine.v0, c, ITEM_TYPE_BTREE, + elem, &info[i]); +#else mc_engine.v1->get_elem_info(mc_engine.v0, c, ITEM_TYPE_BTREE, elem_array[i], &info[i]); +#endif } bodylen = sizeof(rsp->message.body) + (elem_count * (sizeof(uint64_t)+sizeof(uint32_t))); @@ -6312,17 +6546,30 @@ static void process_bin_bop_get(conn *c) { break; } } +#ifdef USE_EBLOCK_RESULT + } +#endif if (ret == ENGINE_SUCCESS) { STATS_ELEM_HITS(c, bop_get, key, nkey); /* Remember this command so we can garbage collect it later */ +#ifdef USE_EBLOCK_RESULT + c->coll_eitem = (void *)&c->eblk_ret; + c->coll_resps = (void *)bkeyptr; +#else c->coll_eitem = (void *)elem_array; c->coll_ecount = elem_count; +#endif c->coll_op = OPERATION_BOP_GET; conn_set_state(c, conn_mwrite); } else { STATS_NOKEY(c, cmd_bop_get); +#ifdef USE_EBLOCK_RESULT + mc_engine.v1->btree_elem_release(mc_engine.v0, c, &c->eblk_ret, EITEM_TYPE_BLOCK); + free((void *)bkeyptr); +#else mc_engine.v1->btree_elem_release(mc_engine.v0, c, elem_array, elem_count); +#endif if (c->ewouldblock) c->ewouldblock = false; write_bin_packet(c, PROTOCOL_BINARY_RESPONSE_ENOMEM, 0); @@ -6347,6 +6594,11 @@ static void process_bin_bop_get(conn *c) { else write_bin_packet(c, PROTOCOL_BINARY_RESPONSE_UNREADABLE, 0); break; +#ifdef USE_EBLOCK_RESULT + case ENGINE_ENOMEM: + write_bin_packet(c, PROTOCOL_BINARY_RESPONSE_ENOMEM, 0); + break; +#endif default: STATS_NOKEY(c, cmd_bop_get); if (ret == ENGINE_EBADTYPE) @@ -6357,9 +6609,12 @@ static void process_bin_bop_get(conn *c) { write_bin_packet(c, PROTOCOL_BINARY_RESPONSE_EINTERNAL, 0); } +#ifdef USE_EBLOCK_RESULT +#else if (ret != ENGINE_SUCCESS && elem_array != NULL) { free((void *)elem_array); } +#endif } static void process_bin_bop_count(conn *c) { @@ -6509,6 +6764,52 @@ static void process_bin_bop_prepare_nread_keys(conn *c) { need_size = elem_array_size; } #endif +#ifdef USE_EBLOCK_RESULT +#ifdef SUPPORT_BOP_SMGET + if (c->cmd == PROTOCOL_BINARY_CMD_BOP_SMGET) { +#ifdef JHPARK_OLD_SMGET_INTERFACE + if (c->coll_smgmode == 0) { + int smget_count; + int kfnd_array_size; + int kmis_array_size; /* key index array where the missed key indexes are to be saved */ + int elem_rshdr_size; /* the size of result header about the found elems */ + int kmis_rshdr_size; /* the size of result header about the missed keys */ + + if (req->message.body.key_count > MAX_SMGET_KEY_COUNT || + (req->message.body.req_offset + req->message.body.req_count) > MAX_SMGET_REQ_COUNT) { + ret = ENGINE_EBADVALUE; break; + } + smget_count = req->message.body.req_offset + req->message.body.req_count; + kfnd_array_size = smget_count * (2*sizeof(uint32_t)); + kmis_array_size = req->message.body.key_count * sizeof(uint32_t); + elem_rshdr_size = smget_count * (sizeof(uint64_t) + (3*sizeof(uint32_t))); + kmis_rshdr_size = req->message.body.key_count * sizeof(uint32_t); + need_size = kfnd_array_size + kmis_array_size + elem_rshdr_size + kmis_rshdr_size; + } else { +#endif + int trim_array_size; /* smget trim element array size */ + int ehit_array_size; /* smget hitted elem array size */ + int emis_array_size; /* element missed keys array size */ + int elem_rshdr_size; /* the size of result header about the found elems */ + int emis_rshdr_size; /* the size of result header about the missed keys */ + + if (req->message.body.key_count > MAX_SMGET_KEY_COUNT || + (req->message.body.req_offset + req->message.body.req_count) > MAX_SMGET_REQ_COUNT) { + ret = ENGINE_EBADVALUE; break; + } + trim_array_size = req->message.body.key_count * sizeof(eitem*); + ehit_array_size = req->message.body.req_count * sizeof(smget_ehit_t); + emis_array_size = req->message.body.key_count * sizeof(smget_emis_t); + elem_rshdr_size = req->message.body.req_count * (sizeof(uint64_t) + (3*sizeof(uint32_t))); + emis_rshdr_size = req->message.body.key_count * sizeof(uint32_t); + need_size = trim_array_size + ehit_array_size + emis_array_size + + elem_rshdr_size + emis_rshdr_size; +#ifdef JHPARK_OLD_SMGET_INTERFACE + } +#endif + } +#endif +#else /* USE_EBLOCK_RESULT */ #ifdef SUPPORT_BOP_SMGET if (c->cmd == PROTOCOL_BINARY_CMD_BOP_SMGET) { #ifdef JHPARK_OLD_SMGET_INTERFACE @@ -6553,6 +6854,7 @@ static void process_bin_bop_prepare_nread_keys(conn *c) { #endif } #endif +#endif /* USE_EBLOCK_RESULT */ assert(need_size > 0); if ((elem = (eitem *)malloc(need_size)) == NULL) { @@ -6592,8 +6894,12 @@ static void process_bin_bop_prepare_nread_keys(conn *c) { c->ritem = (char *)c->coll_strkeys; c->rlbytes = vlen; #endif +#ifdef USE_EBLOCK_RESULT + c->coll_resps = (void *)elem; +#else c->coll_eitem = (void *)elem; c->coll_ecount = 0; +#endif c->coll_op = (c->cmd==PROTOCOL_BINARY_CMD_BOP_MGET ? OPERATION_BOP_MGET : OPERATION_BOP_SMGET); conn_set_state(c, conn_nread); c->substate = bin_reading_bop_nread_keys_complete; @@ -6614,7 +6920,11 @@ static void process_bin_bop_prepare_nread_keys(conn *c) { #ifdef SUPPORT_BOP_MGET static void process_bin_bop_mget_complete(conn *c) { assert(c->coll_op == OPERATION_BOP_MGET); +#ifdef USE_EBLOCK_RESULT + assert(c->coll_resps != NULL); +#else assert(c->coll_eitem != NULL); +#endif ENGINE_ERROR_CODE ret = ENGINE_ENOTSUP; write_bin_packet(c, PROTOCOL_BINARY_RESPONSE_NOT_SUPPORTED, 0); @@ -6630,10 +6940,17 @@ static void process_bin_bop_mget_complete(conn *c) { #endif c->coll_strkeys = NULL; } +#ifdef USE_EBLOCK_RESULT + if (c->coll_resps != NULL) { + free((void *)c->coll_resps); + c->coll_resps = NULL; + } +#else if (c->coll_eitem != NULL) { free((void *)c->coll_eitem); c->coll_eitem = NULL; } +#endif } } #endif @@ -6667,8 +6984,12 @@ static void process_bin_bop_smget_complete_old(conn *c) { uint32_t kmis_count = 0; uint32_t elem_count = 0; +#ifdef USE_EBLOCK_RESULT + uint32_t *kfnd_array = (uint32_t*)c->coll_resps; +#else eitem **elem_array = (eitem **)c->coll_eitem; uint32_t *kfnd_array = (uint32_t*)((char*)elem_array + (smget_count*sizeof(eitem*))); +#endif uint32_t *flag_array = (uint32_t*)((char*)kfnd_array + (smget_count*sizeof(uint32_t))); uint32_t *kmis_array = (uint32_t*)((char*)flag_array + (smget_count*sizeof(uint32_t))); @@ -6718,10 +7039,17 @@ static void process_bin_bop_smget_complete_old(conn *c) { &c->coll_bkrange, (c->coll_efilter.ncompval==0 ? NULL : &c->coll_efilter), c->coll_roffset, c->coll_rcount, +#ifdef USE_EBLOCK_RESULT + &c->eblk_ret, kfnd_array, flag_array, +#else elem_array, kfnd_array, flag_array, &elem_count, +#endif kmis_array, &kmis_count, &trimmed, &duplicated, c->binary_header.request.vbucket); } +#ifdef USE_EBLOCK_RESULT + elem_count = EBLOCK_ELEM_COUNT(&c->eblk_ret); +#endif switch (ret) { case ENGINE_SUCCESS: @@ -6734,6 +7062,10 @@ static void process_bin_bop_smget_complete_old(conn *c) { uint32_t *vlenptr; uint32_t *flagptr; uint32_t *klenptr; +#ifdef USE_EBLOCK_RESULT + eitem *elem; + eblock_scan_t eblk_sc; +#endif if (((long)resultptr % 8) != 0) /* NOT aligned */ resultptr += (8 - ((long)resultptr % 8)); @@ -6743,9 +7075,18 @@ static void process_bin_bop_smget_complete_old(conn *c) { klenptr = (uint32_t *)((char*)flagptr + (sizeof(uint32_t) * elem_count)); eitem_info info[elem_count+1]; /* elem_count might be 0. */ +#ifdef USE_EBLOCK_RESULT + EBLOCK_SCAN_INIT(&c->eblk_ret, &eblk_sc); +#endif for (i = 0; i < elem_count; i++) { +#ifdef USE_EBLOCK_RESULT + EBLOCK_SCAN_NEXT(&eblk_sc, elem); + mc_engine.v1->get_elem_info(mc_engine.v0, c, ITEM_TYPE_BTREE, + elem, &info[i]); +#else mc_engine.v1->get_elem_info(mc_engine.v0, c, ITEM_TYPE_BTREE, elem_array[i], &info[i]); +#endif } bodylen = sizeof(rsp->message.body); @@ -6806,15 +7147,26 @@ static void process_bin_bop_smget_complete_old(conn *c) { } if (ret == ENGINE_SUCCESS) { +#ifdef USE_EBLOCK_RESULT + STATS_NOKEY2(c, cmd_bop_smget, bop_smget_oks); + /* Remember this command so we can garbage collect it later */ + /* c->coll_resps = (void *)kfnd_array; */ + c->coll_eitem = (void *)&c->eblk_ret; +#else /* Remember this command so we can garbage collect it later */ /* c->coll_eitem = (void *)elem_array; */ STATS_NOKEY2(c, cmd_bop_smget, bop_smget_oks); c->coll_ecount = elem_count; +#endif c->coll_op = OPERATION_BOP_SMGET; conn_set_state(c, conn_mwrite); } else { STATS_NOKEY(c, cmd_bop_smget); +#ifdef USE_EBLOCK_RESULT + mc_engine.v1->btree_elem_release(mc_engine.v0, c, &c->eblk_ret, EITEM_TYPE_BLOCK); +#else mc_engine.v1->btree_elem_release(mc_engine.v0, c, elem_array, elem_count); +#endif write_bin_packet(c, PROTOCOL_BINARY_RESPONSE_ENOMEM, 0); } } @@ -6869,7 +7221,11 @@ static void process_bin_bop_smget_complete_old(conn *c) { #endif static void process_bin_bop_smget_complete(conn *c) { +#ifdef USE_EBLOCK_RESULT + assert(c->coll_resps != NULL); +#else assert(c->coll_eitem != NULL); +#endif #ifdef JHPARK_OLD_SMGET_INTERFACE if (c->coll_smgmode == 0) { process_bin_bop_smget_complete_old(c); @@ -6899,8 +7255,14 @@ static void process_bin_bop_smget_complete(conn *c) { memcpy(vptr + c->coll_lenkeys - 2, "\r\n", 2); #endif +#ifdef USE_EBLOCK_RESULT + smres.eblk_ret = &c->eblk_ret; + smres.trim_elems = (void *)c->coll_resps; + smres.elem_kinfo = (smget_ehit_t *)&smres.trim_elems[c->coll_numkeys]; +#else smres.elem_array = (eitem **)c->coll_eitem; smres.elem_kinfo = (smget_ehit_t *)&smres.elem_array[c->coll_rcount+c->coll_numkeys]; +#endif smres.miss_kinfo = (smget_emis_t *)&smres.elem_kinfo[c->coll_rcount]; resultptr = (char *)&smres.miss_kinfo[c->coll_numkeys]; @@ -6963,6 +7325,10 @@ static void process_bin_bop_smget_complete(conn *c) { uint32_t *vlenptr; uint32_t *flagptr; uint32_t *klenptr; +#ifdef USE_EBLOCK_RESULT + eitem *elem; + eblock_scan_t eblk_sc; +#endif if (((long)resultptr % 8) != 0) /* NOT aligned */ resultptr += (8 - ((long)resultptr % 8)); @@ -6972,9 +7338,18 @@ static void process_bin_bop_smget_complete(conn *c) { klenptr = (uint32_t *)((char*)flagptr + (sizeof(uint32_t) * smres.elem_count)); eitem_info info[smres.elem_count+1]; /* elem_count might be 0. */ +#ifdef USE_EBLOCK_RESULT + EBLOCK_SCAN_INIT(smres.eblk_ret, &eblk_sc); +#endif for (i = 0; i < smres.elem_count; i++) { +#ifdef USE_EBLOCK_RESULT + EBLOCK_SCAN_NEXT(&eblk_sc, elem); + mc_engine.v1->get_elem_info(mc_engine.v0, c, ITEM_TYPE_BTREE, + elem, &info[i]); +#else mc_engine.v1->get_elem_info(mc_engine.v0, c, ITEM_TYPE_BTREE, smres.elem_array[i], &info[i]); +#endif } bodylen = sizeof(rsp->message.body); @@ -7051,16 +7426,27 @@ static void process_bin_bop_smget_complete(conn *c) { } if (ret == ENGINE_SUCCESS) { +#ifdef USE_EBLOCK_RESULT + STATS_NOKEY2(c, cmd_bop_smget, bop_smget_oks); + /* Remember this command so we can garbage collect it later */ + /* c->coll_resps = (void *)smres.trim_count */ + c->coll_eitem = (void *)smres.eblk_ret; +#else /* Remember this command so we can garbage collect it later */ /* c->coll_eitem = (void *)elem_array; */ STATS_NOKEY2(c, cmd_bop_smget, bop_smget_oks); c->coll_ecount = smres.elem_count+smres.trim_count; +#endif c->coll_op = OPERATION_BOP_SMGET; conn_set_state(c, conn_mwrite); } else { STATS_NOKEY(c, cmd_bop_smget); +#ifdef USE_EBLOCK_RESULT + mc_engine.v1->btree_elem_release(mc_engine.v0, c, smres.eblk_ret, EITEM_TYPE_BLOCK); +#else mc_engine.v1->btree_elem_release(mc_engine.v0, c, smres.elem_array, smres.elem_count+smres.trim_count); +#endif write_bin_packet(c, PROTOCOL_BINARY_RESPONSE_ENOMEM, 0); } } @@ -7108,10 +7494,17 @@ static void process_bin_bop_smget_complete(conn *c) { #endif c->coll_strkeys = NULL; } +#ifdef USE_EBLOCK_RESULT + if (c->coll_resps != NULL) { + free((void *)c->coll_resps); + c->coll_resps = NULL; + } +#else if (c->coll_eitem != NULL) { free((void *)c->coll_eitem); c->coll_eitem = NULL; } +#endif } } #endif @@ -10731,6 +11124,7 @@ static void process_lop_get(conn *c, char *key, size_t nkey, #else if (ret != ENGINE_SUCCESS && elem_array != NULL) { free((void *)elem_array); + } #endif } @@ -10758,7 +11152,10 @@ static void process_lop_prepare_nread(conn *c, int cmd, size_t vlen, c->ritem = (char *)info.value; c->rlbytes = vlen; c->coll_eitem = (void *)elem; +#ifdef USE_EBLOCK_RESULT +#else c->coll_ecount = 1; +#endif c->coll_op = OPERATION_LOP_INSERT; c->coll_key = key; c->coll_nkey = nkey; @@ -11263,7 +11660,10 @@ static void process_sop_prepare_nread(conn *c, int cmd, size_t vlen, char *key, c->rlbytes = vlen; } c->coll_eitem = (void *)elem; +#ifdef USE_EBLOCK_RESULT +#else c->coll_ecount = 1; +#endif c->coll_op = cmd; c->coll_key = key; c->coll_nkey = nkey; @@ -11487,18 +11887,32 @@ static void process_bop_get(conn *c, char *key, size_t nkey, const uint32_t offset, const uint32_t count, const bool delete, const bool drop_if_empty) { +#ifdef USE_EBLOCK_RESULT +#else eitem **elem_array = NULL; +#endif uint32_t elem_count; uint32_t access_count; uint32_t flags, i; bool dropped_trimmed; +#ifdef USE_EBLOCK_RESULT +#else int est_count; int need_size; +#endif assert(c->ewouldblock == false); ENGINE_ERROR_CODE ret = ENGINE_SUCCESS; +#ifdef USE_EBLOCK_RESULT + ret = mc_engine.v1->btree_elem_get(mc_engine.v0, c, key, nkey, + bkrange, efilter, offset, count, + delete, drop_if_empty, + &c->eblk_ret, &access_count, + &flags, &dropped_trimmed, 0); + elem_count = EBLOCK_ELEM_COUNT(&c->eblk_ret); +#else est_count = MAX_BTREE_SIZE; if (count > 0 && count < MAX_BTREE_SIZE) { est_count = count; @@ -11514,6 +11928,7 @@ static void process_bop_get(conn *c, char *key, size_t nkey, delete, drop_if_empty, elem_array, &elem_count, &access_count, &flags, &dropped_trimmed, 0); +#endif if (ret == ENGINE_EWOULDBLOCK) { c->ewouldblock = true; ret = ENGINE_SUCCESS; @@ -11557,6 +11972,11 @@ static void process_bop_get(conn *c, char *key, size_t nkey, char *respbuf; /* response string buffer */ char *respptr; int resplen; +#ifdef USE_EBLOCK_RESULT + int need_size; + eitem *elem; + eblock_scan_t eblk_sc; +#endif do { need_size = ((2*lenstr_size) + 30) /* response head and tail size */ @@ -11572,9 +11992,18 @@ static void process_bop_get(conn *c, char *key, size_t nkey, } respptr += strlen(respptr); +#ifdef USE_EBLOCK_RESULT + EBLOCK_SCAN_INIT(&c->eblk_ret, &eblk_sc); +#endif for (i = 0; i < elem_count; i++) { +#ifdef USE_EBLOCK_RESULT + EBLOCK_SCAN_NEXT(&eblk_sc, elem); + mc_engine.v1->get_elem_info(mc_engine.v0, c, ITEM_TYPE_BTREE, + elem, &info); +#else mc_engine.v1->get_elem_info(mc_engine.v0, c, ITEM_TYPE_BTREE, elem_array[i], &info); +#endif resplen = make_bop_elem_response(respptr, &info); if ((add_iov(c, respptr, resplen) != 0) || (add_iov(c, info.value, info.nbytes) != 0)) { @@ -11597,15 +12026,23 @@ static void process_bop_get(conn *c, char *key, size_t nkey, if (ret == ENGINE_SUCCESS) { STATS_ELEM_HITS(c, bop_get, key, nkey); +#ifdef USE_EBLOCK_RESULT + c->coll_eitem = (void *)&c->eblk_ret; +#else c->coll_eitem = (void *)elem_array; c->coll_ecount = elem_count; +#endif c->coll_resps = respbuf; c->coll_op = OPERATION_BOP_GET; conn_set_state(c, conn_mwrite); c->msgcurr = 0; } else { /* ENGINE_ENOMEM */ STATS_NOKEY(c, cmd_bop_get); +#ifdef USE_EBLOCK_RESULT + mc_engine.v1->btree_elem_release(mc_engine.v0, c, &c->eblk_ret, EITEM_TYPE_BLOCK); +#else mc_engine.v1->btree_elem_release(mc_engine.v0, c, elem_array, elem_count); +#endif free(respbuf); if (c->ewouldblock) c->ewouldblock = false; @@ -11628,6 +12065,11 @@ static void process_bop_get(conn *c, char *key, size_t nkey, else if (ret == ENGINE_EBKEYOOR) out_string(c, "OUT_OF_RANGE"); else out_string(c, "UNREADABLE"); break; +#ifdef USE_EBLOCK_RESULT + case ENGINE_ENOMEM: + out_string(c, "SERVER_ERROR out of memory getting elements"); + break; +#endif default: STATS_NOKEY(c, cmd_bop_get); if (ret == ENGINE_EBADTYPE) out_string(c, "TYPE_MISMATCH"); @@ -11636,9 +12078,12 @@ static void process_bop_get(conn *c, char *key, size_t nkey, else handle_unexpected_errorcode_ascii(c, ret); } +#ifdef USE_EBLOCK_RESULT +#else if (ret != ENGINE_SUCCESS && elem_array != NULL) { free((void *)elem_array); } +#endif } static void process_bop_count(conn *c, char *key, size_t nkey, @@ -11751,15 +12196,28 @@ static void process_bop_position(conn *c, char *key, size_t nkey, static void process_bop_pwg(conn *c, char *key, size_t nkey, const bkey_range *bkrange, ENGINE_BTREE_ORDER order, const uint32_t count) { +#ifdef USE_EBLOCK_RESULT +#else eitem **elem_array = NULL; +#endif uint32_t elem_count; uint32_t elem_index; uint32_t flags, i; int position; +#ifdef USE_EBLOCK_RESULT +#else int need_size; +#endif ENGINE_ERROR_CODE ret = ENGINE_SUCCESS; +#ifdef USE_EBLOCK_RESULT + ret = mc_engine.v1->btree_posi_find_with_get(mc_engine.v0, c, key, nkey, + bkrange, order, count, &position, + &c->eblk_ret, &elem_index, + &flags, 0); + elem_count = EBLOCK_ELEM_COUNT(&c->eblk_ret); +#else need_size = ((count*2) + 1) * sizeof(eitem*); if ((elem_array = (eitem **)malloc(need_size)) == NULL) { out_string(c, "SERVER_ERROR out of memory"); @@ -11770,6 +12228,7 @@ static void process_bop_pwg(conn *c, char *key, size_t nkey, const bkey_range *b bkrange, order, count, &position, elem_array, &elem_count, &elem_index, &flags, 0); +#endif if (settings.detail_enabled) { stats_prefix_record_bop_pwg(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); @@ -11782,6 +12241,11 @@ static void process_bop_pwg(conn *c, char *key, size_t nkey, const bkey_range *b char *respbuf; /* response string buffer */ char *respptr; int resplen; +#ifdef USE_EBLOCK_RESULT + int need_size; + eitem *elem; + eblock_scan_t eblk_sc; +#endif do { need_size = ((4*lenstr_size) + 30) /* response head and tail size */ @@ -11797,9 +12261,18 @@ static void process_bop_pwg(conn *c, char *key, size_t nkey, const bkey_range *b } respptr += strlen(respptr); +#ifdef USE_EBLOCK_RESULT + EBLOCK_SCAN_INIT(&c->eblk_ret, &eblk_sc); +#endif for (i = 0; i < elem_count; i++) { +#ifdef USE_EBLOCK_RESULT + EBLOCK_SCAN_NEXT(&eblk_sc, elem); + mc_engine.v1->get_elem_info(mc_engine.v0, c, ITEM_TYPE_BTREE, + elem, &info); +#else mc_engine.v1->get_elem_info(mc_engine.v0, c, ITEM_TYPE_BTREE, elem_array[i], &info); +#endif resplen = make_bop_elem_response(respptr, &info); if ((add_iov(c, respptr, resplen) != 0) || (add_iov(c, info.value, info.nbytes) != 0)) { @@ -11818,15 +12291,23 @@ static void process_bop_pwg(conn *c, char *key, size_t nkey, const bkey_range *b if (ret == ENGINE_SUCCESS) { STATS_ELEM_HITS(c, bop_pwg, key, nkey); +#ifdef USE_EBLOCK_RESULT + c->coll_eitem = (void *)&c->eblk_ret; +#else c->coll_eitem = (void *)elem_array; c->coll_ecount = elem_count; +#endif c->coll_resps = respbuf; c->coll_op = OPERATION_BOP_PWG; conn_set_state(c, conn_mwrite); c->msgcurr = 0; } else { /* ENGINE_ENOMEM */ STATS_NOKEY(c, cmd_bop_pwg); +#ifdef USE_EBLOCK_RESULT + mc_engine.v1->btree_elem_release(mc_engine.v0, c, &c->eblk_ret, EITEM_TYPE_BLOCK); +#else mc_engine.v1->btree_elem_release(mc_engine.v0, c, elem_array, elem_count); +#endif free(respbuf); out_string(c, "SERVER_ERROR out of memory writing get response"); } @@ -11845,6 +12326,11 @@ static void process_bop_pwg(conn *c, char *key, size_t nkey, const bkey_range *b if (ret == ENGINE_KEY_ENOENT) out_string(c, "NOT_FOUND"); else out_string(c, "UNREADABLE"); break; +#ifdef USE_EBLOCK_RESULT + case ENGINE_ENOMEM: + out_string(c, "SERVER_ERROR out of memory getting elements"); + break; +#endif default: STATS_NOKEY(c, cmd_bop_pwg); if (ret == ENGINE_EBADTYPE) out_string(c, "TYPE_MISMATCH"); @@ -11853,25 +12339,40 @@ static void process_bop_pwg(conn *c, char *key, size_t nkey, const bkey_range *b else handle_unexpected_errorcode_ascii(c, ret); } +#ifdef USE_EBLOCK_RESULT +#else if (ret != ENGINE_SUCCESS && elem_array != NULL) { free((void *)elem_array); } +#endif } static void process_bop_gbp(conn *c, char *key, size_t nkey, ENGINE_BTREE_ORDER order, uint32_t from_posi, uint32_t to_posi) { +#ifdef USE_EBLOCK_RESULT +#else eitem **elem_array = NULL; +#endif uint32_t elem_count; uint32_t flags, i; +#ifdef USE_EBLOCK_RESULT +#else int est_count; int need_size; +#endif ENGINE_ERROR_CODE ret = ENGINE_SUCCESS; if (from_posi > MAX_BTREE_SIZE) from_posi = MAX_BTREE_SIZE; if (to_posi > MAX_BTREE_SIZE) to_posi = MAX_BTREE_SIZE; +#ifdef USE_EBLOCK_RESULT + ret = mc_engine.v1->btree_elem_get_by_posi(mc_engine.v0, c, key, nkey, + order, from_posi, to_posi, + &c->eblk_ret, &flags, 0); + elem_count = EBLOCK_ELEM_COUNT(&c->eblk_ret); +#else est_count = (from_posi <= to_posi ? (to_posi - from_posi + 1) : (from_posi - to_posi + 1)); need_size = est_count * sizeof(eitem*); @@ -11883,6 +12384,7 @@ static void process_bop_gbp(conn *c, char *key, size_t nkey, ENGINE_BTREE_ORDER ret = mc_engine.v1->btree_elem_get_by_posi(mc_engine.v0, c, key, nkey, order, from_posi, to_posi, elem_array, &elem_count, &flags, 0); +#endif if (settings.detail_enabled) { stats_prefix_record_bop_gbp(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); @@ -11913,6 +12415,11 @@ static void process_bop_gbp(conn *c, char *key, size_t nkey, ENGINE_BTREE_ORDER char *respbuf; /* response string buffer */ char *respptr; int resplen; +#ifdef USE_EBLOCK_RESULT + int need_size; + eitem *elem; + eblock_scan_t eblk_sc; +#endif do { need_size = ((2*lenstr_size) + 30) /* response head and tail size */ @@ -11928,9 +12435,18 @@ static void process_bop_gbp(conn *c, char *key, size_t nkey, ENGINE_BTREE_ORDER } respptr += strlen(respptr); +#ifdef USE_EBLOCK_RESULT + EBLOCK_SCAN_INIT(&c->eblk_ret, &eblk_sc); +#endif for (i = 0; i < elem_count; i++) { +#ifdef USE_EBLOCK_RESULT + EBLOCK_SCAN_NEXT(&eblk_sc, elem); + mc_engine.v1->get_elem_info(mc_engine.v0, c, ITEM_TYPE_BTREE, + elem, &info); +#else mc_engine.v1->get_elem_info(mc_engine.v0, c, ITEM_TYPE_BTREE, elem_array[i], &info); +#endif resplen = make_bop_elem_response(respptr, &info); if ((add_iov(c, respptr, resplen) != 0) || (add_iov(c, info.value, info.nbytes) != 0)) { @@ -11949,15 +12465,23 @@ static void process_bop_gbp(conn *c, char *key, size_t nkey, ENGINE_BTREE_ORDER if (ret == ENGINE_SUCCESS) { STATS_ELEM_HITS(c, bop_gbp, key, nkey); +#ifdef USE_EBLOCK_RESULT + c->coll_eitem = (void *)&c->eblk_ret; +#else c->coll_eitem = (void *)elem_array; c->coll_ecount = elem_count; +#endif c->coll_resps = respbuf; c->coll_op = OPERATION_BOP_GBP; conn_set_state(c, conn_mwrite); c->msgcurr = 0; } else { /* ENGINE_ENOMEM */ STATS_NOKEY(c, cmd_bop_gbp); +#ifdef USE_EBLOCK_RESULT + mc_engine.v1->btree_elem_release(mc_engine.v0, c, &c->eblk_ret, EITEM_TYPE_BLOCK); +#else mc_engine.v1->btree_elem_release(mc_engine.v0, c, elem_array, elem_count); +#endif free(respbuf); out_string(c, "SERVER_ERROR out of memory writing get response"); } @@ -11976,6 +12500,11 @@ static void process_bop_gbp(conn *c, char *key, size_t nkey, ENGINE_BTREE_ORDER if (ret == ENGINE_KEY_ENOENT) out_string(c, "NOT_FOUND"); else out_string(c, "UNREADABLE"); break; +#ifdef USE_EBLOCK_RESULT + case ENGINE_ENOMEM: + out_string(c, "SERVER_ERROR out of memory getting elements"); + break; +#endif default: STATS_NOKEY(c, cmd_bop_gbp); if (ret == ENGINE_EBADTYPE) out_string(c, "TYPE_MISMATCH"); @@ -11983,9 +12512,12 @@ static void process_bop_gbp(conn *c, char *key, size_t nkey, ENGINE_BTREE_ORDER else handle_unexpected_errorcode_ascii(c, ret); } +#ifdef USE_EBLOCK_RESULT +#else if (ret != ENGINE_SUCCESS && elem_array != NULL) { free((void *)elem_array); } +#endif } static void process_bop_update_prepare_nread(conn *c, int cmd, char *key, size_t nkey, const int vlen) @@ -12012,7 +12544,10 @@ static void process_bop_update_prepare_nread(conn *c, int cmd, char *key, size_t c->ritem = ((elem_value *)elem)->value; c->rlbytes = vlen; c->coll_eitem = (void *)elem; +#ifdef USE_EBLOCK_RESULT +#else c->coll_ecount = 1; +#endif c->coll_op = cmd; c->coll_key = key; c->coll_nkey = nkey; @@ -12060,7 +12595,10 @@ static void process_bop_prepare_nread(conn *c, int cmd, char *key, size_t nkey, c->ritem = (char *)info.value; c->rlbytes = vlen; c->coll_eitem = (void *)elem; +#ifdef USE_EBLOCK_RESULT +#else c->coll_ecount = 1; +#endif c->coll_op = cmd; /* OPERATION_BOP_INSERT | OPERATION_BOP_UPSERT */ c->coll_key = key; c->coll_nkey = nkey; @@ -12090,6 +12628,55 @@ static void process_bop_prepare_nread_keys(conn *c, int cmd, uint32_t vlen, uint ENGINE_ERROR_CODE ret = ENGINE_SUCCESS; int need_size = 0; +#ifdef USE_EBLOCK_RESULT +#ifdef SUPPORT_BOP_MGET + if (cmd == OPERATION_BOP_MGET) { + int bmget_count = c->coll_numkeys * c->coll_rcount; + int respon_hdr_size = c->coll_numkeys * ((lenstr_size*2)+30); + int respon_bdy_size = bmget_count * ((MAX_BKEY_LENG*2+2)+(MAX_EFLAG_LENG*2+2)+lenstr_size+15); + + need_size = respon_hdr_size + respon_bdy_size; + } +#endif +#ifdef SUPPORT_BOP_SMGET + if (cmd == OPERATION_BOP_SMGET) { +#ifdef JHPARK_OLD_SMGET_INTERFACE + if (c->coll_smgmode == 0) { + int smget_count = c->coll_roffset + c->coll_rcount; + int kfnd_array_size; /* key index array where the find key indexes are to be saved */ + int kmis_array_size; /* key index array where the missed key indexes are to be saved */ + int respon_hdr_size; /* the size of response head and tail */ + int respon_bdy_size; /* the size of response body */ + + kfnd_array_size = smget_count * (2*sizeof(uint32_t)); + kmis_array_size = c->coll_numkeys * sizeof(uint32_t); + respon_hdr_size = (2*lenstr_size) + 30; /* result head and tail size */ + respon_bdy_size = smget_count * ((MAX_BKEY_LENG*2+2)+(MAX_EFLAG_LENG*2+2)+(lenstr_size*2)+5); /* result body size */ + + need_size = kfnd_array_size + kmis_array_size + respon_hdr_size + respon_bdy_size; + } else { +#endif + int trim_array_size; /* smget trim element array size */ + int ehit_array_size; /* smget hitted elem array size */ + int emis_array_size; /* element missed keys array size */ + int respon_hdr_size; /* the size of response head and tail */ + int respon_bdy_size; /* the size of response body */ + + trim_array_size = c->coll_numkeys * sizeof(eitem*); + ehit_array_size = c->coll_rcount * sizeof(smget_ehit_t); + emis_array_size = c->coll_numkeys * sizeof(smget_emis_t); + respon_hdr_size = (3*lenstr_size) + 50; /* result head and tail size */ + respon_bdy_size = (c->coll_rcount * ((MAX_BKEY_LENG*2+2)+(MAX_EFLAG_LENG*2+2)+(lenstr_size*2)+10)) + + (c->coll_numkeys * ((MAX_EFLAG_LENG*2+2) + 5)); /* result body size */ + need_size = trim_array_size + ehit_array_size + emis_array_size + + respon_hdr_size + respon_bdy_size; +#ifdef JHPARK_OLD_SMGET_INTERFACE + } +#endif + } +#endif + +#else /* USE_EBLOCK_RESULT */ #ifdef SUPPORT_BOP_MGET if (cmd == OPERATION_BOP_MGET) { int bmget_count = c->coll_numkeys * c->coll_rcount; @@ -12137,6 +12724,7 @@ static void process_bop_prepare_nread_keys(conn *c, int cmd, uint32_t vlen, uint #endif } #endif +#endif /* USE_EBLOCK_RESULT */ assert(need_size > 0); if ((elem = (eitem *)malloc(need_size)) == NULL) { @@ -12172,8 +12760,12 @@ static void process_bop_prepare_nread_keys(conn *c, int cmd, uint32_t vlen, uint c->ritem = (char *)c->coll_strkeys; c->rlbytes = vlen; #endif +#ifdef USE_EBLOCK_RESULT + c->coll_resps = (void *)elem; +#else c->coll_eitem = (void *)elem; c->coll_ecount = 0; +#endif c->coll_op = cmd; conn_set_state(c, conn_nread); } @@ -12587,7 +13179,10 @@ static void process_mop_prepare_nread(conn *c, int cmd, char *key, size_t nkey, } c->rlbytes = vlen; c->coll_eitem = (void *)elem; +#ifdef USE_EBLOCK_RESULT +#else c->coll_ecount = 1; +#endif c->coll_op = cmd; c->coll_key = key; c->coll_nkey = nkey; @@ -12647,7 +13242,10 @@ static void process_mop_prepare_nread_fields(conn *c, int cmd, char *key, size_t c->ritem = (char *)c->coll_strkeys; c->rlbytes = flen; #endif +#ifdef USE_EBLOCK_RESULT +#else c->coll_ecount = 1; +#endif c->coll_op = cmd; c->coll_key = key; c->coll_nkey = nkey;