diff --git a/data/zz.csv b/data/zz.csv new file mode 100644 index 00000000..d411e90b --- /dev/null +++ b/data/zz.csv @@ -0,0 +1,17 @@ +# time, object, mass +0, 10, 0.5 +1, 11, 0.25 +2, 12, 0.0125 +3, 13, 0.006125 +4, 14, 0.0030625 +5, 15, 0.00152125 +6, 10, 0.5 +7, 10, 0.5 +8, 10, 0.5 +9, 10, 0.5 +10, 10, 0.5 +11, 10, 0.5 +12, 10, 0.5 +13, 10, 0.5 +14, 10, 0.5 + diff --git a/doc/API.md b/doc/API.md index c161267a..2f421646 100644 --- a/doc/API.md +++ b/doc/API.md @@ -1,128 +1,135 @@ traceReader: typedef struct { - int time_field; - int obj_id_field; - int obj_size_field; - int op_field; - int ttl_field; - - // csv reader - gboolean has_header; - char delimiter; - - // binary reader - char binary_fmt[MAX_BIN_FMT_STR_LEN]; +int time_field; +int obj_id_field; +int obj_size_field; +int obj_mass_field; +int op_field; +int ttl_field; + +// csv reader +gboolean has_header; +char delimiter; + +// binary reader +char binary_fmt[MAX_BIN_FMT_STR_LEN]; } reader_init_param_t; typedef struct reader { - char *mapped_file; /* mmap the file, this should not change during runtime */ - uint64_t mmap_offset; +char _mapped_file; /_ mmap the file, this should not change during runtime \*/ +uint64_t mmap_offset; - FILE *file; - size_t file_size; +FILE \*file; +size_t file_size; - trace_type_e trace_type; /* possible types see trace_type_t */ - obj_id_type_e obj_id_type; /* possible types see obj_id_type_e in request.h */ +trace_type_e trace_type; /_ possible types see trace_type_t _/ +obj_id_type_e obj_id_type; /_ possible types see obj_id_type_e in request.h _/ - size_t item_size; /* the size of one record, used to - * locate the memory location of next element, - * when used in vscsiReaser and binaryReader, - * it is a const value, - * when it is used in plainReader or csvReader, - * it is the size of last record, it does not - * include LFCR or \0 */ +size_t item_size; /_ the size of one record, used to +_ locate the memory location of next element, +_ when used in vscsiReaser and binaryReader, +_ it is a const value, +_ when it is used in plainReader or csvReader, +_ it is the size of last record, it does not +_ include LFCR or \0 _/ - uint64_t n_total_req; /* number of requests in the trace */ - uint64_t n_uniq_obj; /* number of objects in the trace */ +uint64_t n_total_req; /_ number of requests in the trace _/ +uint64_t n_uniq_obj; /_ number of objects in the trace _/ - char trace_path[MAX_FILE_PATH_LEN]; - reader_init_param_t init_params; +char trace_path[MAX_FILE_PATH_LEN]; +reader_init_param_t init_params; - void *reader_params; - void *other_params; /* currently not used */ +void *reader_params; +void *other_params; /_ currently not used _/ - gint ver; +gint ver; - bool cloned; // true if this is a cloned reader, else false +bool cloned; // true if this is a cloned reader, else false } reader_t; -/** - * setup the reader struct for reading trace - * @param trace_path - * @param trace_type CSV_TRACE, PLAIN_TXT_TRACE, BIN_TRACE, VSCSI_TRACE, - * TWR_BIN_TRACE - * @param obj_id_type OBJ_ID_NUM, OBJ_ID_STR - * @param setup_params - * @return a pointer to reader_t struct, the returned reader needs to be - * explicitly closed by calling close_reader or close_trace - */ -reader_t *setup_reader(const char *trace_path, const trace_type_e trace_type, - const obj_id_type_e obj_id_type, - const reader_init_param_t *const reader_init_param); - -/* this is the same function as setup_reader */ +/\*\* + +- setup the reader struct for reading trace +- @param trace_path +- @param trace_type CSV_TRACE, PLAIN_TXT_TRACE, BIN_TRACE, VSCSI_TRACE, +- TWR_BIN_TRACE +- @param obj_id_type OBJ_ID_NUM, OBJ_ID_STR +- @param setup_params +- @return a pointer to reader_t struct, the returned reader needs to be +- explicitly closed by calling close_reader or close_trace + */ + reader_t *setup_reader(const char *trace_path, const trace_type_e trace_type, + const obj_id_type_e obj_id_type, + const reader_init_param_t *const reader_init_param); + +/_ this is the same function as setup_reader _/ static inline reader_t * open_trace(const char *path, const trace_type_e type, - const obj_id_type_e obj_id_type, - const reader_init_param_t *const reader_init_param) { - return setup_reader(path, type, obj_id_type, reader_init_param); +const obj_id_type_e obj_id_type, +const reader_init_param_t \*const reader_init_param) { +return setup_reader(path, type, obj_id_type, reader_init_param); } -/** - * read one request from reader, and store it in the pre-allocated request_t req - * @param reader - * @param req - */ -uint64_t get_num_of_req(reader_t *const reader); - -/** - * as the name suggests - * @param reader - * @return - */ -static inline trace_type_e get_trace_type(const reader_t *const reader) { +/\*\* + +- read one request from reader, and store it in the pre-allocated request_t req +- @param reader +- @param req + */ + uint64_t get_num_of_req(reader_t *const reader); + +/\*\* + +- as the name suggests +- @param reader +- @return + */ + static inline trace_type_e get_trace_type(const reader_t *const reader) { return reader->trace_type; -} + } + +/\*\* -/** - * as the name suggests - * @param reader - * @return - */ -static inline obj_id_type_e get_obj_id_type(const reader_t *const reader) { +- as the name suggests +- @param reader +- @return + */ + static inline obj_id_type_e get_obj_id_type(const reader_t *const reader) { return reader->obj_id_type; -} + } + +/\*\* -/** - * read one request from reader/trace, stored the info in pre-allocated req - * @param reader - * @param req - * return 0 on success and 1 if reach end of trace - */ -int read_one_req(reader_t *const reader, request_t *const req); +- read one request from reader/trace, stored the info in pre-allocated req +- @param reader +- @param req +- return 0 on success and 1 if reach end of trace + */ + int read_one_req(reader_t *const reader, request_t \*const req); -/** - * reset reader, so we can read from the beginning - * @param reader - */ -void reset_reader(reader_t *const reader); +/\*\* -/** - * close reader and release resources - * @param reader - * @return - */ -int close_reader(reader_t *const reader); +- reset reader, so we can read from the beginning +- @param reader + */ + void reset_reader(reader_t *const reader); -/** - * clone a reader, mostly used in multithreading - * @param reader - * @return - */ -reader_t *clone_reader(const reader_t *const reader); +/\*\* +- close reader and release resources +- @param reader +- @return + */ + int close_reader(reader_t *const reader); +/\*\* + +- clone a reader, mostly used in multithreading +- @param reader +- @return + */ + reader_t *clone_reader(const reader_t \*const reader); cache and cacheAlgo: @@ -130,33 +137,24 @@ static inline request_t *new_request(); static inline void copy_request(request_t *req_dest, request_t *req_src); static inline request_t *clone_request(request_t *req); static inline void free_request(request_t *req); -static inline void print_request(request_t *req); - +static inline void print_request(request_t \*req); simulator: sim_res_t * simulate_at_multi_sizes(reader_t *const reader, - const cache_t *const cache, - const gint num_of_sizes, - const guint64 *const cache_sizes, - reader_t *const warmup_reader, - const double warmup_perc, - const gint num_of_threads); - +const cache_t *const cache, +const gint num_of_sizes, +const guint64 *const cache_sizes, +reader_t \*const warmup_reader, +const double warmup_perc, +const gint num_of_threads); sim_res_t * simulate_at_multi_sizes_with_step_size(reader_t *const reader_in, - const cache_t *const cache_in, - const guint64 step_size, - reader_t *const warmup_reader, - const double warmup_perc, - const gint num_of_threads); - - - +const cache_t *const cache_in, +const guint64 step_size, +reader_t *const warmup_reader, +const double warmup_perc, +const gint num_of_threads); profiler: - - - - diff --git a/example/cacheCluster/main.cpp b/example/cacheCluster/main.cpp index 008d2882..55a8862c 100644 --- a/example/cacheCluster/main.cpp +++ b/example/cacheCluster/main.cpp @@ -27,6 +27,7 @@ void simulate(int argc, char *argv[]) { reader_init_param_t init_params = default_reader_init_params(); init_params.obj_id_field = 5; init_params.obj_size_field = 4; + init_params.obj_mass_field = 0; init_params.time_field = 2; init_params.has_header_set = true; init_params.has_header = true; diff --git a/libCacheSim/bin/cachesim/sim.c b/libCacheSim/bin/cachesim/sim.c index f5edc0e1..4f414943 100644 --- a/libCacheSim/bin/cachesim/sim.c +++ b/libCacheSim/bin/cachesim/sim.c @@ -41,11 +41,12 @@ void simulate(reader_t *reader, cache_t *cache, int report_interval, req_cnt++; req_byte += req->obj_size; if (cache->get(cache, req) == false) { - printf("%lu miss\n", (unsigned long)req->obj_id); + printf("%lu miss, %f, %f\n", (unsigned long)req->obj_id, req->obj_mass, cache->get_total_mass(cache)); miss_cnt++; miss_byte += req->obj_size; + } else { + printf("%lu hit, %f, %f\n", (unsigned long)req->obj_id, req->obj_mass, cache->get_total_mass(cache)); } - printf("%lu hit\n", (unsigned long)req->obj_id); if (req->clock_time - last_report_ts >= report_interval && req->clock_time != 0) { INFO( diff --git a/libCacheSim/bin/cli_reader_utils.c b/libCacheSim/bin/cli_reader_utils.c index a38cde0c..95989584 100644 --- a/libCacheSim/bin/cli_reader_utils.c +++ b/libCacheSim/bin/cli_reader_utils.c @@ -126,6 +126,13 @@ void parse_reader_params(const char *reader_params_str, params->obj_size_field = (int)strtol(value, &end, 0); if (strlen(end) > 2) ERROR("param parsing error, find string \"%s\" after number\n", end); + } else if (strcasecmp(key, "obj-mass-col") == 0 || + strcasecmp(key, "obj-mass-field") == 0 || + strcasecmp(key, "mass-col") == 0 || + strcasecmp(key, "mass-field") == 0) { + params->obj_mass_field = (int)strtol(value, &end, 0); + if (strlen(end) > 2) + ERROR("param parsing error, find string \"%s\" after number\n", end); } else if (strcasecmp(key, "cnt-col") == 0 || strcasecmp(key, "cnt-field") == 0) { params->cnt_field = (int)strtol(value, &end, 0); diff --git a/libCacheSim/cache/cache.c b/libCacheSim/cache/cache.c index 04a5023b..641429f3 100644 --- a/libCacheSim/cache/cache.c +++ b/libCacheSim/cache/cache.c @@ -38,12 +38,15 @@ cache_t *cache_struct_init(const char *const cache_name, cache->future_stack_dist_array_size = 0; cache->default_ttl = params.default_ttl; cache->n_req = 0; + cache->total_mass = 0.0; cache->to_evict_candidate = NULL; cache->to_evict_candidate_gen_vtime = -1; cache->can_insert = cache_can_insert_default; cache->get_occupied_byte = cache_get_occupied_byte_default; cache->get_n_obj = cache_get_n_obj_default; + cache->get_total_mass = cache_get_total_mass_default; + /* this option works only when eviction age tracking * is on in config.h */ @@ -268,6 +271,7 @@ cache_obj_t *cache_insert_base(cache_t *cache, const request_t *req) { cache->occupied_byte += (int64_t)cache_obj->obj_size + (int64_t)cache->obj_md_size; cache->n_obj += 1; + cache->total_mass += cache_obj->obj_mass; #ifdef SUPPORT_TTL if (cache->default_ttl != 0 && req->ttl == 0) { @@ -329,6 +333,7 @@ void cache_remove_obj_base(cache_t *cache, cache_obj_t *obj, DEBUG_ASSERT(cache->occupied_byte >= obj->obj_size + cache->obj_md_size); cache->occupied_byte -= (obj->obj_size + cache->obj_md_size); cache->n_obj -= 1; + cache->total_mass -= obj->obj_mass; if (remove_from_hashtable) { hashtable_delete(cache->hashtable, obj); } diff --git a/libCacheSim/cache/cacheObj.c b/libCacheSim/cache/cacheObj.c index 07380b02..2cfefc02 100644 --- a/libCacheSim/cache/cacheObj.c +++ b/libCacheSim/cache/cacheObj.c @@ -16,6 +16,7 @@ void copy_cache_obj_to_request(request_t *req_dest, const cache_obj_t *cache_obj) { req_dest->obj_id = cache_obj->obj_id; req_dest->obj_size = cache_obj->obj_size; + req_dest->obj_mass = cache_obj->obj_mass; req_dest->next_access_vtime = cache_obj->misc.next_access_vtime; req_dest->valid = true; } @@ -27,6 +28,7 @@ void copy_cache_obj_to_request(request_t *req_dest, */ void copy_request_to_cache_obj(cache_obj_t *cache_obj, const request_t *req) { cache_obj->obj_size = req->obj_size; + cache_obj->obj_mass = req->obj_mass; #ifdef SUPPORT_TTL if (req->ttl != 0) cache_obj->exp_time = req->clock_time + req->ttl; diff --git a/libCacheSim/include/libCacheSim/cache.h b/libCacheSim/include/libCacheSim/cache.h index bd979367..7fb1c456 100644 --- a/libCacheSim/include/libCacheSim/cache.h +++ b/libCacheSim/include/libCacheSim/cache.h @@ -64,6 +64,8 @@ typedef int64_t (*cache_get_occupied_byte_func_ptr)(const cache_t *); typedef int64_t (*cache_get_n_obj_func_ptr)(const cache_t *); +typedef double (*cache_get_total_mass_func_ptr)(const cache_t *); + typedef void (*cache_print_cache_func_ptr)(const cache_t *); // #define EVICTION_AGE_ARRAY_SZE 40 @@ -80,6 +82,7 @@ typedef struct { int64_t n_obj; int64_t occupied_byte; + double total_mass; int64_t cache_size; /* current trace time, used to determine obj expiration */ @@ -106,6 +109,8 @@ struct cache { cache_to_evict_func_ptr to_evict; cache_get_occupied_byte_func_ptr get_occupied_byte; cache_get_n_obj_func_ptr get_n_obj; + cache_get_total_mass_func_ptr get_total_mass; + cache_print_cache_func_ptr print_cache; admissioner_t *admissioner; @@ -124,6 +129,10 @@ struct cache { // use cache->get_occupied_byte to obtain the number of objects in the cache // do not use this variable directly int64_t occupied_byte; + + // use cache->get_total_mass to obtain the mass of objects in the cache + // do not use this variable directly + double total_mass; /************ end of private fields *************/ // because some algorithms choose different candidates @@ -286,6 +295,11 @@ static inline int64_t cache_get_n_obj_default(const cache_t *cache) { return cache->n_obj; } +static inline double cache_get_total_mass_default(const cache_t *cache) { + return cache->total_mass; +} + + static inline int64_t cache_get_reference_time(const cache_t *cache) { return cache->n_req; } diff --git a/libCacheSim/include/libCacheSim/cacheObj.h b/libCacheSim/include/libCacheSim/cacheObj.h index 918e1a56..92f34825 100644 --- a/libCacheSim/include/libCacheSim/cacheObj.h +++ b/libCacheSim/include/libCacheSim/cacheObj.h @@ -136,6 +136,8 @@ typedef struct cache_obj { struct cache_obj *hash_next; obj_id_t obj_id; uint32_t obj_size; + double obj_mass; + struct { struct cache_obj *prev; struct cache_obj *next; diff --git a/libCacheSim/include/libCacheSim/reader.h b/libCacheSim/include/libCacheSim/reader.h index 644fb667..c7e4ccc4 100644 --- a/libCacheSim/include/libCacheSim/reader.h +++ b/libCacheSim/include/libCacheSim/reader.h @@ -44,6 +44,7 @@ typedef struct { int time_field; int obj_id_field; int obj_size_field; + int obj_mass_field; int op_field; int ttl_field; int cnt_field; @@ -140,6 +141,7 @@ static inline void set_default_reader_init_params(reader_init_param_t *params) { params->time_field = 0; params->obj_id_field = 0; params->obj_size_field = 0; + params->obj_mass_field = 0; params->op_field = 0; params->ttl_field = 0; params->next_access_vtime_field = 0; diff --git a/libCacheSim/include/libCacheSim/request.h b/libCacheSim/include/libCacheSim/request.h index a32c4337..0779eeaa 100644 --- a/libCacheSim/include/libCacheSim/request.h +++ b/libCacheSim/include/libCacheSim/request.h @@ -23,6 +23,7 @@ typedef struct request { uint64_t hv; /* hash value, used when offloading hash to reader */ obj_id_t obj_id; int64_t obj_size; + double obj_mass; int32_t ttl; req_op_e op; @@ -72,6 +73,7 @@ static inline request_t *new_request(void) { request_t *req = my_malloc(request_t); memset(req, 0, sizeof(request_t)); req->obj_size = 1; + req->obj_mass = 1; req->op = OP_INVALID; req->valid = true; req->obj_id = 0; @@ -110,13 +112,13 @@ static inline void free_request(request_t *req) { my_free(request_t, req); } static inline void print_request(request_t *req) { #ifdef SUPPORT_TTL - INFO("req clcok_time %lu, id %llu, size %ld, ttl %ld, op %s, valid %d\n", + INFO("req clcok_time %lu, id %llu, size %ld, mass %f, ttl %ld, op %s, valid %d\n", (unsigned long)req->clock_time, (unsigned long long)req->obj_id, - (long)req->obj_size, (long)req->ttl, req_op_str[req->op], req->valid); + (long)req->obj_size, req->obj_mass, (long)req->ttl, req_op_str[req->op], req->valid); #else - printf("req clcok_time %lu, id %llu, size %ld, op %s, valid %d\n", + printf("req clcok_time %lu, id %llu, size %ld, mass: %f, op %s, valid %d\n", (unsigned long)req->clock_time, (unsigned long long)req->obj_id, - (long)req->obj_size, req_op_str[req->op], req->valid); + (long)req->obj_size, req->obj_mass, req_op_str[req->op], req->valid); #endif } diff --git a/libCacheSim/traceReader/generalReader/csv.c b/libCacheSim/traceReader/generalReader/csv.c index acdc133b..cf092483 100644 --- a/libCacheSim/traceReader/generalReader/csv.c +++ b/libCacheSim/traceReader/generalReader/csv.c @@ -216,6 +216,11 @@ static inline void csv_cb1(void *s, size_t len, void *data) { if (req->obj_size == 0 && end == s) { ERROR("csvReader obj_size is not a number: \"%s\"\n", (char *)s); } + } else if (csv_params->curr_field_idx == csv_params->obj_mass_field_idx) { + req->obj_mass = (double)strtof((char *)s, &end); + if (req->obj_mass == 0.0 && end == s) { + ERROR("csvReader obj_mass is not a number: \"%s\"\n", (char *)s); + } } else if (csv_params->curr_field_idx == csv_params->cnt_field_idx) { reader->n_req_left = (uint64_t)strtoull((char *)s, &end, 0) - 1; } @@ -254,6 +259,7 @@ void csv_setup_reader(reader_t *const reader) { csv_params->time_field_idx = init_params->time_field; csv_params->obj_id_field_idx = init_params->obj_id_field; csv_params->obj_size_field_idx = init_params->obj_size_field; + csv_params->obj_mass_field_idx = init_params->obj_mass_field; csv_params->cnt_field_idx = init_params->cnt_field; csv_params->csv_parser = (struct csv_parser *)malloc(sizeof(struct csv_parser)); diff --git a/libCacheSim/traceReader/generalReader/readerInternal.h b/libCacheSim/traceReader/generalReader/readerInternal.h index b01b2b7c..8684e080 100644 --- a/libCacheSim/traceReader/generalReader/readerInternal.h +++ b/libCacheSim/traceReader/generalReader/readerInternal.h @@ -25,6 +25,7 @@ typedef struct { int time_field_idx; int obj_id_field_idx; int obj_size_field_idx; + int obj_mass_field_idx; int op_field_idx; int cnt_field_idx; int ttl_field_idx;