metrics.json

[
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1,"eval_per_second":null,"eval_per_token_ms":0.0,"eval_time":0.0,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":0.0,"predicted_n":1,"predicted_per_second":null,"predicted_per_token_ms":0.0,"prompt_eval_count":512,"prompt_eval_per_second":39.2986842769802,"prompt_eval_per_token_ms":25.44614453125,"prompt_eval_time":13028.426,"prompt_ms":13028.426,"prompt_n":512,"prompt_per_second":39.2986842769802,"prompt_per_token_ms":25.44614453125,"sample_count":1,"sample_per_second":null,"sample_per_token_ms":0.0,"sample_time":0.0,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138809,"total_time":25946.707999999868,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1,"eval_per_second":null,"eval_per_token_ms":0.0,"eval_time":0.0,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":0.0,"predicted_n":1,"predicted_per_second":null,"predicted_per_token_ms":0.0,"prompt_eval_count":512,"prompt_eval_per_second":39.2986842769802,"prompt_eval_per_token_ms":25.44614453125,"prompt_eval_time":13028.426,"prompt_ms":13028.426,"prompt_n":512,"prompt_per_second":39.2986842769802,"prompt_per_token_ms":25.44614453125,"sample_count":1,"sample_per_second":null,"sample_per_token_ms":0.0,"sample_time":0.0,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138810,"total_time":26950.707999999868,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1,"eval_per_second":null,"eval_per_token_ms":0.0,"eval_time":0.0,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":0.0,"predicted_n":1,"predicted_per_second":null,"predicted_per_token_ms":0.0,"prompt_eval_count":512,"prompt_eval_per_second":39.2986842769802,"prompt_eval_per_token_ms":25.44614453125,"prompt_eval_time":13028.426,"prompt_ms":13028.426,"prompt_n":512,"prompt_per_second":39.2986842769802,"prompt_per_token_ms":25.44614453125,"sample_count":1,"sample_per_second":null,"sample_per_token_ms":0.0,"sample_time":0.0,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138811,"total_time":27963.707999999868,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1,"eval_per_second":null,"eval_per_token_ms":0.0,"eval_time":0.0,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":0.0,"predicted_n":1,"predicted_per_second":null,"predicted_per_token_ms":0.0,"prompt_eval_count":512,"prompt_eval_per_second":39.2986842769802,"prompt_eval_per_token_ms":25.44614453125,"prompt_eval_time":13028.426,"prompt_ms":13028.426,"prompt_n":512,"prompt_per_second":39.2986842769802,"prompt_per_token_ms":25.44614453125,"sample_count":1,"sample_per_second":null,"sample_per_token_ms":0.0,"sample_time":0.0,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138812,"total_time":28975.707999999868,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1,"eval_per_second":null,"eval_per_token_ms":0.0,"eval_time":0.0,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":0.0,"predicted_n":1,"predicted_per_second":null,"predicted_per_token_ms":0.0,"prompt_eval_count":512,"prompt_eval_per_second":39.2986842769802,"prompt_eval_per_token_ms":25.44614453125,"prompt_eval_time":13028.426,"prompt_ms":13028.426,"prompt_n":512,"prompt_per_second":39.2986842769802,"prompt_per_token_ms":25.44614453125,"sample_count":1,"sample_per_second":null,"sample_per_token_ms":0.0,"sample_time":0.0,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138813,"total_time":29985.707999999868,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1,"eval_per_second":null,"eval_per_token_ms":0.0,"eval_time":0.0,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":0.0,"predicted_n":1,"predicted_per_second":null,"predicted_per_token_ms":0.0,"prompt_eval_count":512,"prompt_eval_per_second":39.2986842769802,"prompt_eval_per_token_ms":25.44614453125,"prompt_eval_time":13028.426,"prompt_ms":13028.426,"prompt_n":512,"prompt_per_second":39.2986842769802,"prompt_per_token_ms":25.44614453125,"sample_count":1,"sample_per_second":null,"sample_per_token_ms":0.0,"sample_time":0.0,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138814,"total_time":30997.707999999868,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1,"eval_per_second":null,"eval_per_token_ms":0.0,"eval_time":0.0,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":0.0,"predicted_n":1,"predicted_per_second":null,"predicted_per_token_ms":0.0,"prompt_eval_count":512,"prompt_eval_per_second":39.2986842769802,"prompt_eval_per_token_ms":25.44614453125,"prompt_eval_time":13028.426,"prompt_ms":13028.426,"prompt_n":512,"prompt_per_second":39.2986842769802,"prompt_per_token_ms":25.44614453125,"sample_count":1,"sample_per_second":null,"sample_per_token_ms":0.0,"sample_time":0.0,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138815,"total_time":32013.707999999868,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1,"eval_per_second":null,"eval_per_token_ms":0.0,"eval_time":0.0,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":0.0,"predicted_n":1,"predicted_per_second":null,"predicted_per_token_ms":0.0,"prompt_eval_count":512,"prompt_eval_per_second":39.2986842769802,"prompt_eval_per_token_ms":25.44614453125,"prompt_eval_time":13028.426,"prompt_ms":13028.426,"prompt_n":512,"prompt_per_second":39.2986842769802,"prompt_per_token_ms":25.44614453125,"sample_count":1,"sample_per_second":null,"sample_per_token_ms":0.0,"sample_time":0.0,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138816,"total_time":33019.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1,"eval_per_second":null,"eval_per_token_ms":0.0,"eval_time":0.0,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":0.0,"predicted_n":1,"predicted_per_second":null,"predicted_per_token_ms":0.0,"prompt_eval_count":512,"prompt_eval_per_second":39.2986842769802,"prompt_eval_per_token_ms":25.44614453125,"prompt_eval_time":13028.426,"prompt_ms":13028.426,"prompt_n":512,"prompt_per_second":39.2986842769802,"prompt_per_token_ms":25.44614453125,"sample_count":1,"sample_per_second":null,"sample_per_token_ms":0.0,"sample_time":0.0,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138817,"total_time":34031.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1,"eval_per_second":null,"eval_per_token_ms":0.0,"eval_time":0.0,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":0.0,"predicted_n":1,"predicted_per_second":null,"predicted_per_token_ms":0.0,"prompt_eval_count":512,"prompt_eval_per_second":39.2986842769802,"prompt_eval_per_token_ms":25.44614453125,"prompt_eval_time":13028.426,"prompt_ms":13028.426,"prompt_n":512,"prompt_per_second":39.2986842769802,"prompt_per_token_ms":25.44614453125,"sample_count":1,"sample_per_second":null,"sample_per_token_ms":0.0,"sample_time":0.0,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138818,"total_time":35041.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":6,"eval_per_second":6.7076804058593815,"eval_per_token_ms":149.08283333333335,"eval_time":894.4970000000001,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":894.4970000000001,"predicted_n":6,"predicted_per_second":6.7076804058593815,"predicted_per_token_ms":149.08283333333335,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":7,"sample_per_second":348.83141476055215,"sample_per_token_ms":2.8667142857142855,"sample_time":20.067,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138819,"total_time":36053.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":12,"eval_per_second":6.722869858685275,"eval_per_token_ms":148.746,"eval_time":1784.952,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":1784.952,"predicted_n":12,"predicted_per_second":6.722869858685275,"predicted_per_token_ms":148.746,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":13,"sample_per_second":311.50408549589054,"sample_per_token_ms":3.2102307692307694,"sample_time":41.733000000000004,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138820,"total_time":37058.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":19,"eval_per_second":6.819612631645444,"eval_per_token_ms":146.6358947368421,"eval_time":2786.082,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":2786.082,"predicted_n":19,"predicted_per_second":6.819612631645444,"predicted_per_token_ms":146.6358947368421,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":20,"sample_per_second":322.21685194135654,"sample_per_token_ms":3.1035,"sample_time":62.07,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138821,"total_time":38064.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":26,"eval_per_second":6.825762949653697,"eval_per_token_ms":146.50376923076922,"eval_time":3809.098,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":3809.098,"predicted_n":26,"predicted_per_second":6.825762949653697,"predicted_per_token_ms":146.50376923076922,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":27,"sample_per_second":332.2300015996259,"sample_per_token_ms":3.009962962962963,"sample_time":81.269,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138822,"total_time":39079.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":32,"eval_per_second":6.734764961964363,"eval_per_token_ms":148.48328125,"eval_time":4751.465,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":4751.465,"predicted_n":32,"predicted_per_second":6.734764961964363,"predicted_per_token_ms":148.48328125,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":33,"sample_per_second":334.06219630709427,"sample_per_token_ms":2.993454545454546,"sample_time":98.784,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138823,"total_time":40090.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":39,"eval_per_second":6.743122145087437,"eval_per_token_ms":148.29925641025642,"eval_time":5783.671,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":5783.671,"predicted_n":39,"predicted_per_second":6.743122145087437,"predicted_per_token_ms":148.29925641025642,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":40,"sample_per_second":334.6188273283196,"sample_per_token_ms":2.988475,"sample_time":119.539,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138824,"total_time":41102.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":46,"eval_per_second":6.7696576878440196,"eval_per_token_ms":147.71795652173913,"eval_time":6795.026,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":6795.026,"predicted_n":46,"predicted_per_second":6.7696576878440196,"predicted_per_token_ms":147.71795652173913,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":47,"sample_per_second":333.30733063378034,"sample_per_token_ms":3.0002340425531915,"sample_time":141.011,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138825,"total_time":42114.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":52,"eval_per_second":6.710887976953778,"eval_per_token_ms":149.01157692307692,"eval_time":7748.602,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":7748.602,"predicted_n":52,"predicted_per_second":6.710887976953778,"predicted_per_token_ms":149.01157692307692,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":53,"sample_per_second":331.7725417532614,"sample_per_token_ms":3.0141132075471697,"sample_time":159.748,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138826,"total_time":43122.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":59,"eval_per_second":6.735761114747914,"eval_per_token_ms":148.4613220338983,"eval_time":8759.218,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":8759.218,"predicted_n":59,"predicted_per_second":6.735761114747914,"predicted_per_token_ms":148.4613220338983,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":60,"sample_per_second":335.1150282334411,"sample_per_token_ms":2.9840500000000003,"sample_time":179.043,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138827,"total_time":44134.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":65,"eval_per_second":6.745340163353458,"eval_per_token_ms":148.25049230769233,"eval_time":9636.282000000001,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":9636.282000000001,"predicted_n":65,"predicted_per_second":6.745340163353458,"predicted_per_token_ms":148.25049230769233,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":66,"sample_per_second":335.7480058603288,"sample_per_token_ms":2.9784242424242424,"sample_time":196.576,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138828,"total_time":45148.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":72,"eval_per_second":6.7427088139659475,"eval_per_token_ms":148.30834722222224,"eval_time":10678.201000000001,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":10678.201000000001,"predicted_n":72,"predicted_per_second":6.7427088139659475,"predicted_per_token_ms":148.30834722222224,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":73,"sample_per_second":338.6528112822416,"sample_per_token_ms":2.9528767123287674,"sample_time":215.56,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138829,"total_time":46152.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":79,"eval_per_second":6.752660227592001,"eval_per_token_ms":148.0897848101266,"eval_time":11699.093,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":11699.093,"predicted_n":79,"predicted_per_second":6.752660227592001,"predicted_per_token_ms":148.0897848101266,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":80,"sample_per_second":340.3024437969245,"sample_per_token_ms":2.9385625,"sample_time":235.085,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138830,"total_time":47162.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":85,"eval_per_second":6.726356086802281,"eval_per_token_ms":148.66890588235293,"eval_time":12636.857,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":12636.857,"predicted_n":85,"predicted_per_second":6.726356086802281,"predicted_per_token_ms":148.66890588235293,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":86,"sample_per_second":335.4592085503091,"sample_per_token_ms":2.9809883720930235,"sample_time":256.365,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138831,"total_time":48177.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":91,"eval_per_second":6.7053286583683684,"eval_per_token_ms":149.13512087912088,"eval_time":13571.296,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":13571.296,"predicted_n":91,"predicted_per_second":6.7053286583683684,"predicted_per_token_ms":149.13512087912088,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":92,"sample_per_second":329.9288501262336,"sample_per_token_ms":3.0309565217391308,"sample_time":278.848,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138832,"total_time":49188.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":98,"eval_per_second":6.705652050420755,"eval_per_token_ms":149.12792857142858,"eval_time":14614.537,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":14614.537,"predicted_n":98,"predicted_per_second":6.705652050420755,"predicted_per_token_ms":149.12792857142858,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":99,"sample_per_second":330.72649586925945,"sample_per_token_ms":3.0236464646464647,"sample_time":299.341,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138833,"total_time":50202.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":104,"eval_per_second":6.708980551052439,"eval_per_token_ms":149.05394230769232,"eval_time":15501.61,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":15501.61,"predicted_n":104,"predicted_per_second":6.708980551052439,"predicted_per_token_ms":149.05394230769232,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":105,"sample_per_second":332.5236646007974,"sample_per_token_ms":3.007304761904762,"sample_time":315.767,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138834,"total_time":51219.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":111,"eval_per_second":6.682641993951065,"eval_per_token_ms":149.6414144144144,"eval_time":16610.197,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":16610.197,"predicted_n":111,"predicted_per_second":6.682641993951065,"predicted_per_token_ms":149.6414144144144,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":112,"sample_per_second":331.20318428204314,"sample_per_token_ms":3.019294642857143,"sample_time":338.161,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138835,"total_time":52226.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":117,"eval_per_second":6.687599042198636,"eval_per_token_ms":149.53049572649573,"eval_time":17495.068,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":17495.068,"predicted_n":117,"predicted_per_second":6.687599042198636,"predicted_per_token_ms":149.53049572649573,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":118,"sample_per_second":331.822323206191,"sample_per_token_ms":3.0136610169491527,"sample_time":355.612,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138836,"total_time":53236.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":124,"eval_per_second":6.681678530328166,"eval_per_token_ms":149.66299193548386,"eval_time":18558.211,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":18558.211,"predicted_n":124,"predicted_per_second":6.681678530328166,"predicted_per_token_ms":149.66299193548386,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":125,"sample_per_second":332.73529709269206,"sample_per_token_ms":3.005392,"sample_time":375.67400000000004,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138837,"total_time":54246.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":130,"eval_per_second":6.6725350757204405,"eval_per_token_ms":149.86807692307693,"eval_time":19482.850000000002,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":19482.850000000002,"predicted_n":130,"predicted_per_second":6.6725350757204405,"predicted_per_token_ms":149.86807692307693,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":131,"sample_per_second":331.75391456956834,"sample_per_token_ms":3.0142824427480917,"sample_time":394.871,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138838,"total_time":55259.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":136,"eval_per_second":6.672522459293565,"eval_per_token_ms":149.86836029411765,"eval_time":20382.097,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":20382.097,"predicted_n":136,"predicted_per_second":6.672522459293565,"predicted_per_token_ms":149.86836029411765,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":137,"sample_per_second":331.57220892437493,"sample_per_token_ms":3.015934306569343,"sample_time":413.183,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138839,"total_time":56269.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":143,"eval_per_second":6.6739915668749354,"eval_per_token_ms":149.83537062937063,"eval_time":21426.458,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":21426.458,"predicted_n":143,"predicted_per_second":6.6739915668749354,"predicted_per_token_ms":149.83537062937063,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":144,"sample_per_second":332.5942350332594,"sample_per_token_ms":3.006666666666667,"sample_time":432.96000000000004,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138840,"total_time":57271.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":149,"eval_per_second":6.664554584334638,"eval_per_token_ms":150.04753691275167,"eval_time":22357.083,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":22357.083,"predicted_n":149,"predicted_per_second":6.664554584334638,"predicted_per_token_ms":150.04753691275167,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":150,"sample_per_second":332.9907228784606,"sample_per_token_ms":3.003086666666667,"sample_time":450.463,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138841,"total_time":58278.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":156,"eval_per_second":6.6628867597548815,"eval_per_token_ms":150.08509615384617,"eval_time":23413.275,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":23413.275,"predicted_n":156,"predicted_per_second":6.6628867597548815,"predicted_per_token_ms":150.08509615384617,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":157,"sample_per_second":333.5436570406392,"sample_per_token_ms":2.9981082802547774,"sample_time":470.70300000000003,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138842,"total_time":59288.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":162,"eval_per_second":6.659038368269238,"eval_per_token_ms":150.17183333333332,"eval_time":24327.837,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":24327.837,"predicted_n":162,"predicted_per_second":6.659038368269238,"predicted_per_token_ms":150.17183333333332,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":163,"sample_per_second":334.4392397765207,"sample_per_token_ms":2.9900797546012274,"sample_time":487.38300000000004,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138843,"total_time":60303.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":168,"eval_per_second":6.640010450111685,"eval_per_token_ms":150.60217261904762,"eval_time":25301.165,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":25301.165,"predicted_n":168,"predicted_per_second":6.640010450111685,"predicted_per_token_ms":150.60217261904762,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":169,"sample_per_second":335.336755487915,"sample_per_token_ms":2.982076923076923,"sample_time":503.971,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138844,"total_time":61315.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":175,"eval_per_second":6.638758338233053,"eval_per_token_ms":150.63057714285713,"eval_time":26360.351,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":26360.351,"predicted_n":175,"predicted_per_second":6.638758338233053,"predicted_per_token_ms":150.63057714285713,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":176,"sample_per_second":336.41008389226465,"sample_per_token_ms":2.9725625000000004,"sample_time":523.171,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138845,"total_time":62324.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":181,"eval_per_second":6.635330311142002,"eval_per_token_ms":150.70839779005524,"eval_time":27278.22,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":27278.22,"predicted_n":181,"predicted_per_second":6.635330311142002,"predicted_per_token_ms":150.70839779005524,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":182,"sample_per_second":337.37753844157527,"sample_per_token_ms":2.9640384615384616,"sample_time":539.455,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138846,"total_time":63338.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":188,"eval_per_second":6.630412184158288,"eval_per_token_ms":150.82018617021276,"eval_time":28354.195,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":28354.195,"predicted_n":188,"predicted_per_second":6.630412184158288,"predicted_per_token_ms":150.82018617021276,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":189,"sample_per_second":337.0936817229232,"sample_per_token_ms":2.966534391534392,"sample_time":560.6750000000001,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138847,"total_time":64347.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":194,"eval_per_second":6.62380925505261,"eval_per_token_ms":150.97053092783506,"eval_time":29288.283,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":29288.283,"predicted_n":194,"predicted_per_second":6.62380925505261,"predicted_per_token_ms":150.97053092783506,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":195,"sample_per_second":337.1934781593732,"sample_per_token_ms":2.9656564102564102,"sample_time":578.303,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138848,"total_time":65358.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":200,"eval_per_second":6.6147967380452855,"eval_per_token_ms":151.176225,"eval_time":30235.245,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":30235.245,"predicted_n":200,"predicted_per_second":6.6147967380452855,"predicted_per_token_ms":151.176225,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":201,"sample_per_second":336.96111368162707,"sample_per_token_ms":2.9677014925373135,"sample_time":596.508,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138849,"total_time":66370.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":205,"eval_per_second":6.563161238494779,"eval_per_token_ms":152.3656,"eval_time":31234.948,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":31234.948,"predicted_n":205,"predicted_per_second":6.563161238494779,"predicted_per_token_ms":152.3656,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":206,"sample_per_second":328.14774932060675,"sample_per_token_ms":3.047407766990291,"sample_time":627.766,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138850,"total_time":67373.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":211,"eval_per_second":6.55907544764602,"eval_per_token_ms":152.46051184834124,"eval_time":32169.168,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":32169.168,"predicted_n":211,"predicted_per_second":6.55907544764602,"predicted_per_token_ms":152.46051184834124,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":212,"sample_per_second":328.45301727476954,"sample_per_token_ms":3.0445754716981135,"sample_time":645.45,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138851,"total_time":68386.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":217,"eval_per_second":6.554860679910023,"eval_per_token_ms":152.55854377880183,"eval_time":33105.204,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":33105.204,"predicted_n":217,"predicted_per_second":6.554860679910023,"predicted_per_token_ms":152.55854377880183,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":218,"sample_per_second":328.1911698509742,"sample_per_token_ms":3.0470045871559632,"sample_time":664.247,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138852,"total_time":69388.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":223,"eval_per_second":6.5452657813769095,"eval_per_token_ms":152.78218385650226,"eval_time":34070.427,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":34070.427,"predicted_n":223,"predicted_per_second":6.5452657813769095,"predicted_per_token_ms":152.78218385650226,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":224,"sample_per_second":327.4217147835289,"sample_per_token_ms":3.0541651785714286,"sample_time":684.133,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138853,"total_time":70393.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":230,"eval_per_second":6.540119152440368,"eval_per_token_ms":152.90241304347828,"eval_time":35167.555,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":35167.555,"predicted_n":230,"predicted_per_second":6.540119152440368,"predicted_per_token_ms":152.90241304347828,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":231,"sample_per_second":328.3143403724307,"sample_per_token_ms":3.045861471861472,"sample_time":703.594,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138854,"total_time":71409.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":236,"eval_per_second":6.537985627235918,"eval_per_token_ms":152.9523093220339,"eval_time":36096.745,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":36096.745,"predicted_n":236,"predicted_per_second":6.537985627235918,"predicted_per_token_ms":152.9523093220339,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":237,"sample_per_second":329.10495948676294,"sample_per_token_ms":3.0385443037974684,"sample_time":720.135,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138855,"total_time":72422.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":242,"eval_per_second":6.522334413561281,"eval_per_token_ms":153.3193388429752,"eval_time":37103.28,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":37103.28,"predicted_n":242,"predicted_per_second":6.522334413561281,"predicted_per_token_ms":153.3193388429752,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":243,"sample_per_second":328.88815501865724,"sample_per_token_ms":3.0405473251028807,"sample_time":738.8530000000001,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138856,"total_time":73435.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":248,"eval_per_second":6.518921102785906,"eval_per_token_ms":153.3996169354839,"eval_time":38043.105,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":38043.105,"predicted_n":248,"predicted_per_second":6.518921102785906,"predicted_per_token_ms":153.3996169354839,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":249,"sample_per_second":329.14869457512714,"sample_per_token_ms":3.038140562248996,"sample_time":756.4970000000001,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138857,"total_time":74446.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":254,"eval_per_second":6.515526461079234,"eval_per_token_ms":153.47953937007873,"eval_time":38983.803,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":38983.803,"predicted_n":254,"predicted_per_second":6.515526461079234,"predicted_per_token_ms":153.47953937007873,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":255,"sample_per_second":329.37906234176893,"sample_per_token_ms":3.0360156862745096,"sample_time":774.184,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138858,"total_time":75456.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":261,"eval_per_second":6.511471116461329,"eval_per_token_ms":153.5751264367816,"eval_time":40083.108,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":40083.108,"predicted_n":261,"predicted_per_second":6.511471116461329,"predicted_per_token_ms":153.5751264367816,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":262,"sample_per_second":328.7810130219869,"sample_per_token_ms":3.0415381679389313,"sample_time":796.883,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138859,"total_time":76473.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":267,"eval_per_second":6.51110840207816,"eval_per_token_ms":153.58368164794007,"eval_time":41006.843,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":41006.843,"predicted_n":267,"predicted_per_second":6.51110840207816,"predicted_per_token_ms":153.58368164794007,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":268,"sample_per_second":328.66216105181707,"sample_per_token_ms":3.042638059701493,"sample_time":815.427,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138860,"total_time":77487.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":273,"eval_per_second":6.509901870740723,"eval_per_token_ms":153.61214652014652,"eval_time":41936.116,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":41936.116,"predicted_n":273,"predicted_per_second":6.509901870740723,"predicted_per_token_ms":153.61214652014652,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":274,"sample_per_second":329.4207604810505,"sample_per_token_ms":3.035631386861314,"sample_time":831.763,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138861,"total_time":78496.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":280,"eval_per_second":6.504513028597672,"eval_per_token_ms":153.73941071428573,"eval_time":43047.035,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":43047.035,"predicted_n":280,"predicted_per_second":6.504513028597672,"predicted_per_token_ms":153.73941071428573,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":281,"sample_per_second":329.4027728445512,"sample_per_token_ms":3.035797153024911,"sample_time":853.059,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138862,"total_time":79498.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":286,"eval_per_second":6.4959055126480045,"eval_per_token_ms":153.94312587412588,"eval_time":44027.734000000004,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":44027.734000000004,"predicted_n":286,"predicted_per_second":6.4959055126480045,"predicted_per_token_ms":153.94312587412588,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":286,"sample_per_second":329.89709748228535,"sample_per_token_ms":3.0312482517482517,"sample_time":866.937,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138863,"total_time":80501.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":292,"eval_per_second":6.490903521099537,"eval_per_token_ms":154.0617568493151,"eval_time":44986.033,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":44986.033,"predicted_n":292,"predicted_per_second":6.490903521099537,"predicted_per_token_ms":154.0617568493151,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":293,"sample_per_second":328.82000751909226,"sample_per_token_ms":3.0411774744027307,"sample_time":891.065,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138864,"total_time":81509.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":298,"eval_per_second":6.483880138382538,"eval_per_token_ms":154.2286375838926,"eval_time":45960.134,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":45960.134,"predicted_n":298,"predicted_per_second":6.483880138382538,"predicted_per_token_ms":154.2286375838926,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":299,"sample_per_second":329.33033594998585,"sample_per_token_ms":3.036464882943144,"sample_time":907.903,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138865,"total_time":82522.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":303,"eval_per_second":6.472734566934004,"eval_per_token_ms":154.4942079207921,"eval_time":46811.745,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":46811.745,"predicted_n":303,"predicted_per_second":6.472734566934004,"predicted_per_token_ms":154.4942079207921,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":304,"sample_per_second":329.2733781390639,"sample_per_token_ms":3.0369901315789476,"sample_time":923.245,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138866,"total_time":83530.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":310,"eval_per_second":6.467751862237886,"eval_per_token_ms":154.61322903225806,"eval_time":47930.101,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":47930.101,"predicted_n":310,"predicted_per_second":6.467751862237886,"predicted_per_token_ms":154.61322903225806,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":311,"sample_per_second":329.67825098797465,"sample_per_token_ms":3.033260450160772,"sample_time":943.344,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138867,"total_time":84533.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":315,"eval_per_second":6.46019368111844,"eval_per_token_ms":154.79412063492063,"eval_time":48760.148,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":48760.148,"predicted_n":315,"predicted_per_second":6.46019368111844,"predicted_per_token_ms":154.79412063492063,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":316,"sample_per_second":329.52055801595253,"sample_per_token_ms":3.034712025316456,"sample_time":958.969,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138868,"total_time":85540.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":321,"eval_per_second":6.45458885073331,"eval_per_token_ms":154.92853582554517,"eval_time":49732.06,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":49732.06,"predicted_n":321,"predicted_per_second":6.45458885073331,"predicted_per_token_ms":154.92853582554517,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":322,"sample_per_second":330.18393911490193,"sample_per_token_ms":3.0286149068322983,"sample_time":975.214,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138869,"total_time":86543.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":327,"eval_per_second":6.446087086163426,"eval_per_token_ms":155.13287155963303,"eval_time":50728.449,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":50728.449,"predicted_n":327,"predicted_per_second":6.446087086163426,"predicted_per_token_ms":155.13287155963303,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":328,"sample_per_second":329.9115072731406,"sample_per_token_ms":3.0311158536585365,"sample_time":994.206,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138870,"total_time":87550.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":333,"eval_per_second":6.430727957052933,"eval_per_token_ms":155.5033903903904,"eval_time":51782.629,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":51782.629,"predicted_n":333,"predicted_per_second":6.430727957052933,"predicted_per_token_ms":155.5033903903904,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":334,"sample_per_second":322.42899080691234,"sample_per_token_ms":3.1014580838323353,"sample_time":1035.887,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138871,"total_time":88566.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":339,"eval_per_second":6.426418334696624,"eval_per_token_ms":155.6076725663717,"eval_time":52751.001000000004,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":52751.001000000004,"predicted_n":339,"predicted_per_second":6.426418334696624,"predicted_per_token_ms":155.6076725663717,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":340,"sample_per_second":322.3637989246323,"sample_per_token_ms":3.1020852941176473,"sample_time":1054.709,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138872,"total_time":89579.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":345,"eval_per_second":6.4238469883590215,"eval_per_token_ms":155.66995942028984,"eval_time":53706.136,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":53706.136,"predicted_n":345,"predicted_per_second":6.4238469883590215,"predicted_per_token_ms":155.66995942028984,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":346,"sample_per_second":322.9359168206679,"sample_per_token_ms":3.0965895953757228,"sample_time":1071.42,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138873,"total_time":90584.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":351,"eval_per_second":6.418421036607181,"eval_per_token_ms":155.80155840455842,"eval_time":54686.347,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":54686.347,"predicted_n":351,"predicted_per_second":6.418421036607181,"predicted_per_token_ms":155.80155840455842,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":352,"sample_per_second":322.9671876347608,"sample_per_token_ms":3.096289772727273,"sample_time":1089.894,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138874,"total_time":91594.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":357,"eval_per_second":6.415893299920368,"eval_per_token_ms":155.8629411764706,"eval_time":55643.07,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":55643.07,"predicted_n":357,"predicted_per_second":6.415893299920368,"predicted_per_token_ms":155.8629411764706,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":358,"sample_per_second":323.35479698647777,"sample_per_token_ms":3.092578212290503,"sample_time":1107.143,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138875,"total_time":92602.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":363,"eval_per_second":6.413457816134927,"eval_per_token_ms":155.92212947658402,"eval_time":56599.733,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":56599.733,"predicted_n":363,"predicted_per_second":6.413457816134927,"predicted_per_token_ms":155.92212947658402,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":364,"sample_per_second":323.4796456653727,"sample_per_token_ms":3.091384615384616,"sample_time":1125.2640000000001,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138876,"total_time":93614.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":369,"eval_per_second":6.407325118008083,"eval_per_token_ms":156.07136856368564,"eval_time":57590.335,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":57590.335,"predicted_n":369,"predicted_per_second":6.407325118008083,"predicted_per_token_ms":156.07136856368564,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":370,"sample_per_second":324.1014069504861,"sample_per_token_ms":3.085454054054054,"sample_time":1141.618,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138877,"total_time":94629.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":375,"eval_per_second":6.403043409219358,"eval_per_token_ms":156.17573333333334,"eval_time":58565.9,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":58565.9,"predicted_n":375,"predicted_per_second":6.403043409219358,"predicted_per_token_ms":156.17573333333334,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":376,"sample_per_second":324.1918698542085,"sample_per_token_ms":3.084593085106383,"sample_time":1159.807,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138878,"total_time":95634.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":381,"eval_per_second":6.3983590106813635,"eval_per_token_ms":156.29007349081365,"eval_time":59546.518000000004,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":59546.518000000004,"predicted_n":381,"predicted_per_second":6.3983590106813635,"predicted_per_token_ms":156.29007349081365,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":382,"sample_per_second":324.27155620636233,"sample_per_token_ms":3.083835078534032,"sample_time":1178.025,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138879,"total_time":96636.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":387,"eval_per_second":6.385617859348829,"eval_per_token_ms":156.6019173126615,"eval_time":60604.942,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":60604.942,"predicted_n":387,"predicted_per_second":6.385617859348829,"predicted_per_token_ms":156.6019173126615,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":388,"sample_per_second":323.9477542405429,"sample_per_token_ms":3.086917525773196,"sample_time":1197.724,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138880,"total_time":97647.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":393,"eval_per_second":6.381567046562056,"eval_per_token_ms":156.7013231552163,"eval_time":61583.62,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":61583.62,"predicted_n":393,"predicted_per_second":6.381567046562056,"predicted_per_token_ms":156.7013231552163,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":394,"sample_per_second":322.97252766169527,"sample_per_token_ms":3.0962385786802034,"sample_time":1219.9180000000001,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138881,"total_time":98662.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":398,"eval_per_second":6.369382512690675,"eval_per_token_ms":157.0010904522613,"eval_time":62486.434,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":62486.434,"predicted_n":398,"predicted_per_second":6.369382512690675,"predicted_per_token_ms":157.0010904522613,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":399,"sample_per_second":322.5780372056172,"sample_per_token_ms":3.1000250626566417,"sample_time":1236.91,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138883,"total_time":99678.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":404,"eval_per_second":6.365235980095087,"eval_per_token_ms":157.10336633663366,"eval_time":63469.76,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":63469.76,"predicted_n":404,"predicted_per_second":6.365235980095087,"predicted_per_token_ms":157.10336633663366,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":405,"sample_per_second":322.5775619826812,"sample_per_token_ms":3.1000296296296295,"sample_time":1255.512,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138884,"total_time":100691.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":410,"eval_per_second":6.362890480501279,"eval_per_token_ms":157.1612780487805,"eval_time":64436.124,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":64436.124,"predicted_n":410,"predicted_per_second":6.362890480501279,"predicted_per_token_ms":157.1612780487805,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":411,"sample_per_second":323.14902831210065,"sample_per_token_ms":3.094547445255474,"sample_time":1271.859,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138885,"total_time":101698.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":416,"eval_per_second":6.36106918012898,"eval_per_token_ms":157.20627644230768,"eval_time":65397.811,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":65397.811,"predicted_n":416,"predicted_per_second":6.36106918012898,"predicted_per_token_ms":157.20627644230768,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":417,"sample_per_second":323.23977725136115,"sample_per_token_ms":3.093678657074341,"sample_time":1290.064,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138886,"total_time":102701.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":422,"eval_per_second":6.358606160983093,"eval_per_token_ms":157.26717061611373,"eval_time":66366.746,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":66366.746,"predicted_n":422,"predicted_per_second":6.358606160983093,"predicted_per_token_ms":157.26717061611373,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":423,"sample_per_second":323.6278142422251,"sample_per_token_ms":3.08996926713948,"sample_time":1307.057,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138887,"total_time":103703.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":428,"eval_per_second":6.355614290539333,"eval_per_token_ms":157.34120327102804,"eval_time":67342.035,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":67342.035,"predicted_n":428,"predicted_per_second":6.355614290539333,"predicted_per_token_ms":157.34120327102804,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":429,"sample_per_second":324.1407813681763,"sample_per_token_ms":3.085079254079254,"sample_time":1323.499,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138888,"total_time":104706.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":434,"eval_per_second":6.352122908016503,"eval_per_token_ms":157.42768433179725,"eval_time":68323.615,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":68323.615,"predicted_n":434,"predicted_per_second":6.352122908016503,"predicted_per_token_ms":157.42768433179725,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":435,"sample_per_second":324.2203804185797,"sample_per_token_ms":3.08432183908046,"sample_time":1341.68,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138889,"total_time":105708.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":440,"eval_per_second":6.340904156729281,"eval_per_token_ms":157.7062159090909,"eval_time":69390.735,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":69390.735,"predicted_n":440,"predicted_per_second":6.340904156729281,"predicted_per_token_ms":157.7062159090909,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":441,"sample_per_second":323.95742273872577,"sample_per_token_ms":3.0868253968253967,"sample_time":1361.29,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138890,"total_time":106721.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":446,"eval_per_second":6.337715546274134,"eval_per_token_ms":157.78556053811658,"eval_time":70372.36,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":70372.36,"predicted_n":446,"predicted_per_second":6.337715546274134,"predicted_per_token_ms":157.78556053811658,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":447,"sample_per_second":324.0419558215551,"sample_per_token_ms":3.086020134228188,"sample_time":1379.451,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138891,"total_time":107727.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":452,"eval_per_second":6.334230196800047,"eval_per_token_ms":157.87238053097346,"eval_time":71358.316,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":71358.316,"predicted_n":452,"predicted_per_second":6.334230196800047,"predicted_per_token_ms":157.87238053097346,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":453,"sample_per_second":324.41606897959326,"sample_per_token_ms":3.0824613686534215,"sample_time":1396.355,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138892,"total_time":108740.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":457,"eval_per_second":6.324102445533184,"eval_per_token_ms":158.1252056892779,"eval_time":72263.219,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":72263.219,"predicted_n":457,"predicted_per_second":6.324102445533184,"predicted_per_token_ms":158.1252056892779,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":458,"sample_per_second":323.4408456353847,"sample_per_token_ms":3.091755458515284,"sample_time":1416.0240000000001,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138893,"total_time":109749.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":463,"eval_per_second":6.317927183883298,"eval_per_token_ms":158.27976025917926,"eval_time":73283.529,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":73283.529,"predicted_n":463,"predicted_per_second":6.317927183883298,"predicted_per_token_ms":158.27976025917926,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":464,"sample_per_second":323.184369020828,"sample_per_token_ms":3.094209051724138,"sample_time":1435.713,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138894,"total_time":110760.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":469,"eval_per_second":6.314456089582017,"eval_per_token_ms":158.36676759061834,"eval_time":74274.014,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":74274.014,"predicted_n":469,"predicted_per_second":6.314456089582017,"predicted_per_token_ms":158.36676759061834,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":470,"sample_per_second":323.61278559689003,"sample_per_token_ms":3.0901127659574468,"sample_time":1452.353,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138895,"total_time":111771.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":475,"eval_per_second":6.308420204374487,"eval_per_token_ms":158.51829263157893,"eval_time":75296.189,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":75296.189,"predicted_n":475,"predicted_per_second":6.308420204374487,"predicted_per_token_ms":158.51829263157893,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":476,"sample_per_second":323.5317706839434,"sample_per_token_ms":3.0908865546218487,"sample_time":1471.262,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138896,"total_time":112782.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":481,"eval_per_second":6.303446526677123,"eval_per_token_ms":158.64337006237005,"eval_time":76307.461,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":76307.461,"predicted_n":481,"predicted_per_second":6.303446526677123,"predicted_per_token_ms":158.64337006237005,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":482,"sample_per_second":323.33954744538283,"sample_per_token_ms":3.0927240663900415,"sample_time":1490.693,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138897,"total_time":113783.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":486,"eval_per_second":6.299488329893264,"eval_per_token_ms":158.74305144032922,"eval_time":77149.123,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":77149.123,"predicted_n":486,"predicted_per_second":6.299488329893264,"predicted_per_token_ms":158.74305144032922,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":487,"sample_per_second":323.1751058282341,"sample_per_token_ms":3.0942977412731008,"sample_time":1506.923,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138898,"total_time":114788.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":492,"eval_per_second":6.291852971161317,"eval_per_token_ms":158.93569105691057,"eval_time":78196.36,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":78196.36,"predicted_n":492,"predicted_per_second":6.291852971161317,"predicted_per_token_ms":158.93569105691057,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":493,"sample_per_second":323.2384161973262,"sample_per_token_ms":3.09369168356998,"sample_time":1525.19,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138899,"total_time":115800.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":498,"eval_per_second":6.287527734246959,"eval_per_token_ms":159.04502409638556,"eval_time":79204.422,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":79204.422,"predicted_n":498,"predicted_per_second":6.287527734246959,"predicted_per_token_ms":159.04502409638556,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":499,"sample_per_second":323.5054243885488,"sample_per_token_ms":3.091138276553106,"sample_time":1542.478,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138900,"total_time":116810.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":504,"eval_per_second":6.282863489831485,"eval_per_token_ms":159.16309523809522,"eval_time":80218.2,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":80218.2,"predicted_n":504,"predicted_per_second":6.282863489831485,"predicted_per_token_ms":159.16309523809522,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":505,"sample_per_second":323.7986908273922,"sample_per_token_ms":3.0883386138613864,"sample_time":1559.611,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138901,"total_time":117824.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":509,"eval_per_second":6.2779463428982965,"eval_per_token_ms":159.28775834970529,"eval_time":81077.469,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":81077.469,"predicted_n":509,"predicted_per_second":6.2779463428982965,"predicted_per_token_ms":159.28775834970529,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":510,"sample_per_second":323.5576624164301,"sample_per_token_ms":3.0906392156862745,"sample_time":1576.226,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138902,"total_time":118839.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":515,"eval_per_second":6.2746660625767445,"eval_per_token_ms":159.37103106796118,"eval_time":82076.081,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":82076.081,"predicted_n":515,"predicted_per_second":6.2746660625767445,"predicted_per_token_ms":159.37103106796118,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":516,"sample_per_second":323.92100094100306,"sample_per_token_ms":3.087172480620155,"sample_time":1592.981,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138903,"total_time":119852.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":521,"eval_per_second":6.271913249303579,"eval_per_token_ms":159.44098080614205,"eval_time":83068.751,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":83068.751,"predicted_n":521,"predicted_per_second":6.271913249303579,"predicted_per_token_ms":159.44098080614205,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":522,"sample_per_second":324.0350181292006,"sample_per_token_ms":3.0860862068965518,"sample_time":1610.9370000000001,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138904,"total_time":120864.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":527,"eval_per_second":6.2649657043057205,"eval_per_token_ms":159.61779316888047,"eval_time":84118.577,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":84118.577,"predicted_n":527,"predicted_per_second":6.2649657043057205,"predicted_per_token_ms":159.61779316888047,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":528,"sample_per_second":324.08046770703857,"sample_per_token_ms":3.0856534090909093,"sample_time":1629.2250000000001,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138905,"total_time":121876.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":533,"eval_per_second":6.261809988042763,"eval_per_token_ms":159.69823452157598,"eval_time":85119.159,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":85119.159,"predicted_n":533,"predicted_per_second":6.261809988042763,"predicted_per_token_ms":159.69823452157598,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":534,"sample_per_second":323.83203669126135,"sample_per_token_ms":3.088020599250936,"sample_time":1649.003,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138906,"total_time":122892.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":538,"eval_per_second":6.25458696352719,"eval_per_token_ms":159.88265985130113,"eval_time":86016.871,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":86016.871,"predicted_n":538,"predicted_per_second":6.25458696352719,"predicted_per_token_ms":159.88265985130113,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":539,"sample_per_second":323.7377231928347,"sample_per_token_ms":3.0889202226345085,"sample_time":1664.928,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138907,"total_time":123908.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":544,"eval_per_second":6.247352196432372,"eval_per_token_ms":160.0678125,"eval_time":87076.89,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":87076.89,"predicted_n":544,"predicted_per_second":6.247352196432372,"predicted_per_token_ms":160.0678125,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":545,"sample_per_second":323.5408308528536,"sample_per_token_ms":3.0908,"sample_time":1684.486,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138908,"total_time":124917.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":549,"eval_per_second":6.240772480771515,"eval_per_token_ms":160.23657377049182,"eval_time":87969.879,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":87969.879,"predicted_n":549,"predicted_per_second":6.240772480771515,"predicted_per_token_ms":160.23657377049182,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":550,"sample_per_second":323.4388488870175,"sample_per_token_ms":3.0917745454545456,"sample_time":1700.476,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138909,"total_time":125927.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":555,"eval_per_second":6.23627471781194,"eval_per_token_ms":160.35214054054052,"eval_time":88995.438,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":88995.438,"predicted_n":555,"predicted_per_second":6.23627471781194,"predicted_per_token_ms":160.35214054054052,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":556,"sample_per_second":323.44800224319334,"sample_per_token_ms":3.0916870503597123,"sample_time":1718.978,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138910,"total_time":126941.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":561,"eval_per_second":6.230607234981121,"eval_per_token_ms":160.498,"eval_time":90039.378,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":90039.378,"predicted_n":561,"predicted_per_second":6.230607234981121,"predicted_per_token_ms":160.498,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":562,"sample_per_second":323.4113568027161,"sample_per_token_ms":3.092037366548043,"sample_time":1737.7250000000001,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138911,"total_time":127949.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":566,"eval_per_second":6.225977051048589,"eval_per_token_ms":160.61736042402828,"eval_time":90909.426,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":90909.426,"predicted_n":566,"predicted_per_second":6.225977051048589,"predicted_per_token_ms":160.61736042402828,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":567,"sample_per_second":323.6576626665167,"sample_per_token_ms":3.08968430335097,"sample_time":1751.851,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138912,"total_time":128962.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":572,"eval_per_second":6.222294946200917,"eval_per_token_ms":160.71240734265734,"eval_time":91927.497,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":91927.497,"predicted_n":572,"predicted_per_second":6.222294946200917,"predicted_per_token_ms":160.71240734265734,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":573,"sample_per_second":323.9834242616655,"sample_per_token_ms":3.0865776614310643,"sample_time":1768.609,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138913,"total_time":129971.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":578,"eval_per_second":6.215978340090091,"eval_per_token_ms":160.8757214532872,"eval_time":92986.167,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":92986.167,"predicted_n":578,"predicted_per_second":6.215978340090091,"predicted_per_token_ms":160.8757214532872,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":579,"sample_per_second":324.0411683391071,"sample_per_token_ms":3.086027633851468,"sample_time":1786.81,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138914,"total_time":130984.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":583,"eval_per_second":6.211760315594928,"eval_per_token_ms":160.98496226415097,"eval_time":93854.23300000001,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":93854.23300000001,"predicted_n":583,"predicted_per_second":6.211760315594928,"predicted_per_token_ms":160.98496226415097,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":584,"sample_per_second":324.1125891656487,"sample_per_token_ms":3.085347602739726,"sample_time":1801.843,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138915,"total_time":131988.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":589,"eval_per_second":6.2084401456460006,"eval_per_token_ms":161.0710543293718,"eval_time":94870.851,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":94870.851,"predicted_n":589,"predicted_per_second":6.2084401456460006,"predicted_per_token_ms":161.0710543293718,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":590,"sample_per_second":324.44072467960103,"sample_per_token_ms":3.082227118644068,"sample_time":1818.5140000000001,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138916,"total_time":133003.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":595,"eval_per_second":6.201069522952243,"eval_per_token_ms":161.26250420168068,"eval_time":95951.19,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":95951.19,"predicted_n":595,"predicted_per_second":6.201069522952243,"predicted_per_token_ms":161.26250420168068,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":596,"sample_per_second":324.05834137588863,"sample_per_token_ms":3.0858640939597315,"sample_time":1839.175,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138917,"total_time":134018.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":600,"eval_per_second":6.197029150422317,"eval_per_token_ms":161.367645,"eval_time":96820.587,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":96820.587,"predicted_n":600,"predicted_per_second":6.197029150422317,"predicted_per_token_ms":161.367645,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":601,"sample_per_second":324.3103817635038,"sample_per_token_ms":3.0834658901830285,"sample_time":1853.163,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138918,"total_time":135033.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":606,"eval_per_second":6.19230775794081,"eval_per_token_ms":161.49068151815183,"eval_time":97863.353,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":97863.353,"predicted_n":606,"predicted_per_second":6.19230775794081,"predicted_per_token_ms":161.49068151815183,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":607,"sample_per_second":324.53836155712116,"sample_per_token_ms":3.0812998352553542,"sample_time":1870.349,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138919,"total_time":136043.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":611,"eval_per_second":6.186719341198433,"eval_per_token_ms":161.63655482815057,"eval_time":98759.935,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":98759.935,"predicted_n":611,"predicted_per_second":6.186719341198433,"predicted_per_token_ms":161.63655482815057,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":612,"sample_per_second":324.264351744166,"sample_per_token_ms":3.083903594771242,"sample_time":1887.349,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138920,"total_time":137053.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":616,"eval_per_second":6.17447865849588,"eval_per_token_ms":161.95699350649352,"eval_time":99765.508,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":99765.508,"predicted_n":616,"predicted_per_second":6.17447865849588,"predicted_per_token_ms":161.95699350649352,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":617,"sample_per_second":323.8986265543328,"sample_per_token_ms":3.0873857374392224,"sample_time":1904.9170000000001,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138921,"total_time":138068.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":622,"eval_per_second":6.170033525998086,"eval_per_token_ms":162.0736736334405,"eval_time":100809.825,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":100809.825,"predicted_n":622,"predicted_per_second":6.170033525998086,"predicted_per_token_ms":162.0736736334405,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":623,"sample_per_second":324.2586032625724,"sample_per_token_ms":3.0839582664526484,"sample_time":1921.306,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138922,"total_time":139082.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":627,"eval_per_second":6.164840883981943,"eval_per_token_ms":162.21018819776714,"eval_time":101705.788,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":101705.788,"predicted_n":627,"predicted_per_second":6.164840883981943,"predicted_per_token_ms":162.21018819776714,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":628,"sample_per_second":324.01263855667844,"sample_per_token_ms":3.086299363057325,"sample_time":1938.1960000000001,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138923,"total_time":140088.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":633,"eval_per_second":6.160471883971279,"eval_per_token_ms":162.32522748815165,"eval_time":102751.869,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":102751.869,"predicted_n":633,"predicted_per_second":6.160471883971279,"predicted_per_token_ms":162.32522748815165,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":634,"sample_per_second":324.285229830763,"sample_per_token_ms":3.083705047318612,"sample_time":1955.069,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138924,"total_time":141096.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":639,"eval_per_second":6.156751945100041,"eval_per_token_ms":162.42330516431926,"eval_time":103788.492,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":103788.492,"predicted_n":639,"predicted_per_second":6.156751945100041,"predicted_per_token_ms":162.42330516431926,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":640,"sample_per_second":324.1955770605289,"sample_per_token_ms":3.0845578125,"sample_time":1974.117,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138925,"total_time":142107.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":644,"eval_per_second":6.152276897016607,"eval_per_token_ms":162.541448757764,"eval_time":104676.693,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":104676.693,"predicted_n":644,"predicted_per_second":6.152276897016607,"predicted_per_token_ms":162.541448757764,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":645,"sample_per_second":323.59829260510173,"sample_per_token_ms":3.0902511627906977,"sample_time":1993.212,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138926,"total_time":143118.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":650,"eval_per_second":6.148060835875407,"eval_per_token_ms":162.6529123076923,"eval_time":105724.393,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":105724.393,"predicted_n":650,"predicted_per_second":6.148060835875407,"predicted_per_token_ms":162.6529123076923,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":651,"sample_per_second":323.3596822631005,"sample_per_token_ms":3.092531490015361,"sample_time":2013.238,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138927,"total_time":144129.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":655,"eval_per_second":6.143603860723205,"eval_per_token_ms":162.77091145038167,"eval_time":106614.947,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":106614.947,"predicted_n":655,"predicted_per_second":6.143603860723205,"predicted_per_token_ms":162.77091145038167,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":656,"sample_per_second":322.88533485325695,"sample_per_token_ms":3.097074695121951,"sample_time":2031.681,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138928,"total_time":145135.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":660,"eval_per_second":6.135489855139504,"eval_per_token_ms":162.98617121212123,"eval_time":107570.873,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":107570.873,"predicted_n":660,"predicted_per_second":6.135489855139504,"predicted_per_token_ms":162.98617121212123,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":661,"sample_per_second":322.69103822058105,"sample_per_token_ms":3.0989394856278363,"sample_time":2048.399,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138929,"total_time":146141.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":666,"eval_per_second":6.1323934013237125,"eval_per_token_ms":163.06846846846847,"eval_time":108603.6,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":108603.6,"predicted_n":666,"predicted_per_second":6.1323934013237125,"predicted_per_token_ms":163.06846846846847,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":667,"sample_per_second":322.7678089679124,"sample_per_token_ms":3.0982023988006,"sample_time":2066.501,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138930,"total_time":147144.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":672,"eval_per_second":6.126706862239902,"eval_per_token_ms":163.21982142857144,"eval_time":109683.72,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":109683.72,"predicted_n":672,"predicted_per_second":6.126706862239902,"predicted_per_token_ms":163.21982142857144,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":673,"sample_per_second":322.5483331248511,"sample_per_token_ms":3.1003105497771175,"sample_time":2086.509,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138931,"total_time":148146.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":677,"eval_per_second":6.1183204919245355,"eval_per_token_ms":163.44354652880355,"eval_time":110651.281,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":110651.281,"predicted_n":677,"predicted_per_second":6.1183204919245355,"predicted_per_token_ms":163.44354652880355,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":678,"sample_per_second":322.56238228019254,"sample_per_token_ms":3.1001755162241884,"sample_time":2101.919,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138932,"total_time":149159.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":682,"eval_per_second":6.115419577637022,"eval_per_token_ms":163.52107771260998,"eval_time":111521.375,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":111521.375,"predicted_n":682,"predicted_per_second":6.115419577637022,"predicted_per_token_ms":163.52107771260998,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":683,"sample_per_second":322.81422159707495,"sample_per_token_ms":3.097756954612006,"sample_time":2115.768,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138933,"total_time":150171.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":688,"eval_per_second":6.111591658693615,"eval_per_token_ms":163.62349709302325,"eval_time":112572.966,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":112572.966,"predicted_n":688,"predicted_per_second":6.111591658693615,"predicted_per_token_ms":163.62349709302325,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":689,"sample_per_second":323.02512797936765,"sample_per_token_ms":3.0957343976777945,"sample_time":2132.9610000000002,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138934,"total_time":151185.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":693,"eval_per_second":6.107677669875668,"eval_per_token_ms":163.7283520923521,"eval_time":113463.748,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":113463.748,"predicted_n":693,"predicted_per_second":6.107677669875668,"predicted_per_token_ms":163.7283520923521,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":694,"sample_per_second":322.9738163172515,"sample_per_token_ms":3.0962262247838614,"sample_time":2148.781,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138935,"total_time":152195.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":699,"eval_per_second":6.101346401305873,"eval_per_token_ms":163.8982503576538,"eval_time":114564.87700000001,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":114564.87700000001,"predicted_n":699,"predicted_per_second":6.101346401305873,"predicted_per_token_ms":163.8982503576538,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":700,"sample_per_second":322.4576060378805,"sample_per_token_ms":3.1011828571428572,"sample_time":2170.828,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138936,"total_time":153201.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":704,"eval_per_second":6.097655195198762,"eval_per_token_ms":163.99746590909092,"eval_time":115454.216,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":115454.216,"predicted_n":704,"predicted_per_second":6.097655195198762,"predicted_per_token_ms":163.99746590909092,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":705,"sample_per_second":322.64973693749107,"sample_per_token_ms":3.099336170212766,"sample_time":2185.032,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138937,"total_time":154215.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":710,"eval_per_second":6.092021808442758,"eval_per_token_ms":164.14911690140846,"eval_time":116545.873,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":116545.873,"predicted_n":710,"predicted_per_second":6.092021808442758,"predicted_per_token_ms":164.14911690140846,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":711,"sample_per_second":322.5168470825607,"sample_per_token_ms":3.1006132208157524,"sample_time":2204.536,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138938,"total_time":155217.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":715,"eval_per_second":6.089160157141552,"eval_per_token_ms":164.22626013986013,"eval_time":117421.776,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":117421.776,"predicted_n":715,"predicted_per_second":6.089160157141552,"predicted_per_token_ms":164.22626013986013,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":716,"sample_per_second":322.7503377383946,"sample_per_token_ms":3.0983701117318434,"sample_time":2218.433,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138939,"total_time":156230.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":721,"eval_per_second":6.085412486392013,"eval_per_token_ms":164.32739805825244,"eval_time":118480.054,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":118480.054,"predicted_n":721,"predicted_per_second":6.085412486392013,"predicted_per_token_ms":164.32739805825244,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":722,"sample_per_second":322.942660680747,"sample_per_token_ms":3.096524930747923,"sample_time":2235.6910000000003,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138940,"total_time":157245.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":726,"eval_per_second":6.080888684859903,"eval_per_token_ms":164.44964738292012,"eval_time":119390.444,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":119390.444,"predicted_n":726,"predicted_per_second":6.080888684859903,"predicted_per_token_ms":164.44964738292012,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":727,"sample_per_second":322.69891702776096,"sample_per_token_ms":3.0988638239339754,"sample_time":2252.8740000000003,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138941,"total_time":158256.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":732,"eval_per_second":6.076578990956996,"eval_per_token_ms":164.5662800546448,"eval_time":120462.517,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":120462.517,"predicted_n":732,"predicted_per_second":6.076578990956996,"predicted_per_token_ms":164.5662800546448,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":733,"sample_per_second":322.9616953330052,"sample_per_token_ms":3.096342428376535,"sample_time":2269.619,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138942,"total_time":159270.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":737,"eval_per_second":6.0727649240196175,"eval_per_token_ms":164.66963772048848,"eval_time":121361.523,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":121361.523,"predicted_n":737,"predicted_per_second":6.0727649240196175,"predicted_per_token_ms":164.66963772048848,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":738,"sample_per_second":322.7176146533037,"sample_per_token_ms":3.098684281842819,"sample_time":2286.829,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138943,"total_time":160282.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":742,"eval_per_second":6.067262059685014,"eval_per_token_ms":164.81898921832885,"eval_time":122295.69,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":122295.69,"predicted_n":742,"predicted_per_second":6.067262059685014,"predicted_per_token_ms":164.81898921832885,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":743,"sample_per_second":322.6719846020755,"sample_per_token_ms":3.0991224764468375,"sample_time":2302.648,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138944,"total_time":161286.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":748,"eval_per_second":6.062497852696491,"eval_per_token_ms":164.94851203208557,"eval_time":123381.48700000001,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":123381.48700000001,"predicted_n":748,"predicted_per_second":6.062497852696491,"predicted_per_token_ms":164.94851203208557,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":749,"sample_per_second":322.79418229208864,"sample_per_token_ms":3.0979492656875833,"sample_time":2320.364,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138945,"total_time":162300.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":753,"eval_per_second":6.057355530884054,"eval_per_token_ms":165.0885431606906,"eval_time":124311.67300000001,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":124311.67300000001,"predicted_n":753,"predicted_per_second":6.057355530884054,"predicted_per_token_ms":165.0885431606906,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":754,"sample_per_second":322.97002204248986,"sample_per_token_ms":3.0962625994694957,"sample_time":2334.582,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138946,"total_time":163313.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":758,"eval_per_second":6.052402048260608,"eval_per_token_ms":165.22365699208444,"eval_time":125239.532,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":125239.532,"predicted_n":758,"predicted_per_second":6.052402048260608,"predicted_per_token_ms":165.22365699208444,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":759,"sample_per_second":323.1554275422944,"sample_per_token_ms":3.0944861660079055,"sample_time":2348.715,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138947,"total_time":164319.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":764,"eval_per_second":6.046438802663294,"eval_per_token_ms":165.38660732984295,"eval_time":126355.368,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":126355.368,"predicted_n":764,"predicted_per_second":6.046438802663294,"predicted_per_token_ms":165.38660732984295,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":765,"sample_per_second":323.2542789992893,"sample_per_token_ms":3.0935398692810456,"sample_time":2366.558,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138948,"total_time":165334.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":769,"eval_per_second":6.040134201885258,"eval_per_token_ms":165.5592353706112,"eval_time":127315.052,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":127315.052,"predicted_n":769,"predicted_per_second":6.040134201885258,"predicted_per_token_ms":165.5592353706112,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":770,"sample_per_second":323.2048489122268,"sample_per_token_ms":3.0940129870129867,"sample_time":2382.39,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138949,"total_time":166338.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":774,"eval_per_second":6.036015736126978,"eval_per_token_ms":165.67219896640827,"eval_time":128230.282,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":128230.282,"predicted_n":774,"predicted_per_second":6.036015736126978,"predicted_per_token_ms":165.67219896640827,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":775,"sample_per_second":323.1764197140118,"sample_per_token_ms":3.0942851612903226,"sample_time":2398.071,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138950,"total_time":167354.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":780,"eval_per_second":6.032311986350084,"eval_per_token_ms":165.77391923076925,"eval_time":129303.657,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":129303.657,"predicted_n":780,"predicted_per_second":6.032311986350084,"predicted_per_token_ms":165.77391923076925,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":781,"sample_per_second":323.0309485156501,"sample_per_token_ms":3.0956786171574904,"sample_time":2417.725,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138951,"total_time":168367.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":785,"eval_per_second":6.027554631989149,"eval_per_token_ms":165.9047592356688,"eval_time":130235.236,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":130235.236,"predicted_n":785,"predicted_per_second":6.027554631989149,"predicted_per_token_ms":165.9047592356688,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":786,"sample_per_second":323.12595170048115,"sample_per_token_ms":3.0947684478371498,"sample_time":2432.488,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138952,"total_time":169374.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":790,"eval_per_second":6.02249987657687,"eval_per_token_ms":166.04400506329114,"eval_time":131174.764,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":131174.764,"predicted_n":790,"predicted_per_second":6.02249987657687,"predicted_per_token_ms":166.04400506329114,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":791,"sample_per_second":322.9757870238047,"sample_per_token_ms":3.0962073324905184,"sample_time":2449.1,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138953,"total_time":170388.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":796,"eval_per_second":6.01791346858856,"eval_per_token_ms":166.17055150753768,"eval_time":132271.759,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":132271.759,"predicted_n":796,"predicted_per_second":6.01791346858856,"predicted_per_token_ms":166.17055150753768,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":797,"sample_per_second":323.2022735272601,"sample_per_token_ms":3.0940376411543284,"sample_time":2465.948,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138954,"total_time":171398.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":801,"eval_per_second":6.0145665593598405,"eval_per_token_ms":166.26301997503123,"eval_time":133176.679,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":133176.679,"predicted_n":801,"predicted_per_second":6.0145665593598405,"predicted_per_token_ms":166.26301997503123,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":802,"sample_per_second":323.1452590319503,"sample_per_token_ms":3.0945835411471325,"sample_time":2481.856,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138955,"total_time":172412.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":806,"eval_per_second":6.010569325153411,"eval_per_token_ms":166.3735905707196,"eval_time":134097.114,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":134097.114,"predicted_n":806,"predicted_per_second":6.010569325153411,"predicted_per_token_ms":166.3735905707196,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":807,"sample_per_second":322.99573541566184,"sample_per_token_ms":3.096016109045849,"sample_time":2498.485,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138956,"total_time":173415.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":812,"eval_per_second":6.0069028338710755,"eval_per_token_ms":166.47514162561578,"eval_time":135177.815,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":135177.815,"predicted_n":812,"predicted_per_second":6.0069028338710755,"predicted_per_token_ms":166.47514162561578,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":813,"sample_per_second":323.22766478945476,"sample_per_token_ms":3.0937945879458795,"sample_time":2515.255,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138957,"total_time":174422.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":817,"eval_per_second":6.003124799130448,"eval_per_token_ms":166.57991187270503,"eval_time":136095.788,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":136095.788,"predicted_n":817,"predicted_per_second":6.003124799130448,"predicted_per_token_ms":166.57991187270503,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":818,"sample_per_second":323.4089997054527,"sample_per_token_ms":3.092059902200489,"sample_time":2529.305,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138958,"total_time":175435.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":822,"eval_per_second":5.996922921943642,"eval_per_token_ms":166.75218491484185,"eval_time":137070.296,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":137070.296,"predicted_n":822,"predicted_per_second":5.996922921943642,"predicted_per_token_ms":166.75218491484185,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":823,"sample_per_second":323.3387733870581,"sample_per_token_ms":3.092731470230863,"sample_time":2545.318,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138959,"total_time":176444.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":828,"eval_per_second":5.992632839571509,"eval_per_token_ms":166.8715615942029,"eval_time":138169.653,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":138169.653,"predicted_n":828,"predicted_per_second":5.992632839571509,"predicted_per_token_ms":166.8715615942029,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":829,"sample_per_second":323.5998354286,"sample_per_token_ms":3.0902364294330518,"sample_time":2561.806,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138960,"total_time":177457.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":833,"eval_per_second":5.9858888240078265,"eval_per_token_ms":167.05956782713085,"eval_time":139160.62,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":139160.62,"predicted_n":833,"predicted_per_second":5.9858888240078265,"predicted_per_token_ms":167.05956782713085,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":834,"sample_per_second":323.287390861433,"sample_per_token_ms":3.093223021582734,"sample_time":2579.748,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138961,"total_time":178469.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":838,"eval_per_second":5.981340217061266,"eval_per_token_ms":167.1866109785203,"eval_time":140102.38,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":140102.38,"predicted_n":838,"predicted_per_second":5.981340217061266,"predicted_per_token_ms":167.1866109785203,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":839,"sample_per_second":323.0515161281255,"sample_per_token_ms":3.0954815256257446,"sample_time":2597.109,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138962,"total_time":179483.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":843,"eval_per_second":5.976848750808479,"eval_per_token_ms":167.31224792408068,"eval_time":141044.225,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":141044.225,"predicted_n":843,"predicted_per_second":5.976848750808479,"predicted_per_token_ms":167.31224792408068,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":844,"sample_per_second":323.1048693511679,"sample_per_token_ms":3.0949703791469196,"sample_time":2612.155,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138963,"total_time":180493.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":849,"eval_per_second":5.973207567928754,"eval_per_token_ms":167.41423910482922,"eval_time":142134.689,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":142134.689,"predicted_n":849,"predicted_per_second":5.973207567928754,"predicted_per_token_ms":167.41423910482922,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":850,"sample_per_second":323.358915461806,"sample_per_token_ms":3.0925388235294116,"sample_time":2628.658,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138964,"total_time":181492.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":853,"eval_per_second":5.966398766968503,"eval_per_token_ms":167.60529073856975,"eval_time":142967.313,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":142967.313,"predicted_n":853,"predicted_per_second":5.966398766968503,"predicted_per_token_ms":167.60529073856975,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":854,"sample_per_second":322.32946827338213,"sample_per_token_ms":3.102415690866511,"sample_time":2649.463,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138965,"total_time":182498.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":859,"eval_per_second":5.961988871458457,"eval_per_token_ms":167.72926309662398,"eval_time":144079.437,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":144079.437,"predicted_n":859,"predicted_per_second":5.961988871458457,"predicted_per_token_ms":167.72926309662398,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":860,"sample_per_second":322.60387844886054,"sample_per_token_ms":3.0997767441860464,"sample_time":2665.808,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138966,"total_time":183502.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":864,"eval_per_second":5.957347625010286,"eval_per_token_ms":167.8599375,"eval_time":145030.986,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":145030.986,"predicted_n":864,"predicted_per_second":5.957347625010286,"predicted_per_token_ms":167.8599375,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":865,"sample_per_second":322.8021465409791,"sample_per_token_ms":3.097872832369942,"sample_time":2679.66,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138967,"total_time":184510.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":869,"eval_per_second":5.952971470579462,"eval_per_token_ms":167.98333486766398,"eval_time":145977.518,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":145977.518,"predicted_n":869,"predicted_per_second":5.952971470579462,"predicted_per_token_ms":167.98333486766398,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":870,"sample_per_second":322.5995442076325,"sample_per_token_ms":3.099818390804598,"sample_time":2696.842,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138968,"total_time":185512.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":874,"eval_per_second":5.948718385718209,"eval_per_token_ms":168.10343592677344,"eval_time":146922.403,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":146922.403,"predicted_n":874,"predicted_per_second":5.948718385718209,"predicted_per_token_ms":168.10343592677344,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":875,"sample_per_second":322.75079581124794,"sample_per_token_ms":3.0983657142857144,"sample_time":2711.07,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138969,"total_time":186527.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":879,"eval_per_second":5.94508506662604,"eval_per_token_ms":168.2061717861206,"eval_time":147853.225,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":147853.225,"predicted_n":879,"predicted_per_second":5.94508506662604,"predicted_per_token_ms":168.2061717861206,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":880,"sample_per_second":322.93945362314213,"sample_per_token_ms":3.096555681818182,"sample_time":2724.969,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138970,"total_time":187532.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":884,"eval_per_second":5.940023422238058,"eval_per_token_ms":168.34950452488687,"eval_time":148820.962,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":148820.962,"predicted_n":884,"predicted_per_second":5.940023422238058,"predicted_per_token_ms":168.34950452488687,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":885,"sample_per_second":322.89712789389705,"sample_per_token_ms":3.096961581920904,"sample_time":2740.811,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138971,"total_time":188536.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":889,"eval_per_second":5.933698059308961,"eval_per_token_ms":168.5289662542182,"eval_time":149822.251,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":149822.251,"predicted_n":889,"predicted_per_second":5.933698059308961,"predicted_per_token_ms":168.5289662542182,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":890,"sample_per_second":322.94631543654356,"sample_per_token_ms":3.0964898876404496,"sample_time":2755.876,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138972,"total_time":189547.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":894,"eval_per_second":5.928548536207877,"eval_per_token_ms":168.67535011185683,"eval_time":150795.763,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":150795.763,"predicted_n":894,"predicted_per_second":5.928548536207877,"predicted_per_token_ms":168.67535011185683,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":895,"sample_per_second":322.7454785882716,"sample_per_token_ms":3.098416759776536,"sample_time":2773.083,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138973,"total_time":190550.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":899,"eval_per_second":5.922918480066325,"eval_per_token_ms":168.83568520578422,"eval_time":151783.28100000002,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":151783.28100000002,"predicted_n":899,"predicted_per_second":5.922918480066325,"predicted_per_token_ms":168.83568520578422,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":900,"sample_per_second":322.485633265038,"sample_per_token_ms":3.1009133333333336,"sample_time":2790.822,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138974,"total_time":191561.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":905,"eval_per_second":5.918604886599007,"eval_per_token_ms":168.95873591160222,"eval_time":152907.65600000002,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":152907.65600000002,"predicted_n":905,"predicted_per_second":5.918604886599007,"predicted_per_token_ms":168.95873591160222,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":906,"sample_per_second":322.6472745920613,"sample_per_token_ms":3.0993598233995585,"sample_time":2808.02,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138975,"total_time":192571.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":910,"eval_per_second":5.914387857899514,"eval_per_token_ms":169.0792054945055,"eval_time":153862.077,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":153862.077,"predicted_n":910,"predicted_per_second":5.914387857899514,"predicted_per_token_ms":169.0792054945055,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":911,"sample_per_second":322.8086835890231,"sample_per_token_ms":3.097810098792536,"sample_time":2822.105,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138976,"total_time":193580.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":915,"eval_per_second":5.911343644921829,"eval_per_token_ms":169.16627759562843,"eval_time":154787.144,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":154787.144,"predicted_n":915,"predicted_per_second":5.911343644921829,"predicted_per_token_ms":169.16627759562843,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":916,"sample_per_second":322.7028121999982,"sample_per_token_ms":3.098826419213974,"sample_time":2838.525,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138977,"total_time":194591.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":920,"eval_per_second":5.9073339004587915,"eval_per_token_ms":169.28110326086954,"eval_time":155738.615,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":155738.615,"predicted_n":920,"predicted_per_second":5.9073339004587915,"predicted_per_token_ms":169.28110326086954,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":921,"sample_per_second":322.53240820345945,"sample_per_token_ms":3.1004636264929424,"sample_time":2855.527,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138978,"total_time":195610.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":925,"eval_per_second":5.903333934324637,"eval_per_token_ms":169.39580432432433,"eval_time":156691.119,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":156691.119,"predicted_n":925,"predicted_per_second":5.903333934324637,"predicted_per_token_ms":169.39580432432433,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":926,"sample_per_second":322.5708689244073,"sample_per_token_ms":3.1000939524838014,"sample_time":2870.687,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138979,"total_time":196623.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":931,"eval_per_second":5.898085666380986,"eval_per_token_ms":169.54653705692803,"eval_time":157847.826,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":157847.826,"predicted_n":931,"predicted_per_second":5.898085666380986,"predicted_per_token_ms":169.54653705692803,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":932,"sample_per_second":322.42987582644275,"sample_per_token_ms":3.1014495708154506,"sample_time":2890.551,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138980,"total_time":197625.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":936,"eval_per_second":5.892923242622315,"eval_per_token_ms":169.69506623931625,"eval_time":158834.582,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":158834.582,"predicted_n":936,"predicted_per_second":5.892923242622315,"predicted_per_token_ms":169.69506623931625,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":937,"sample_per_second":322.56776332821073,"sample_per_token_ms":3.100123799359659,"sample_time":2904.8160000000003,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138981,"total_time":198636.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":941,"eval_per_second":5.889447042821331,"eval_per_token_ms":169.7952274176408,"eval_time":159777.309,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":159777.309,"predicted_n":941,"predicted_per_second":5.889447042821331,"predicted_per_token_ms":169.7952274176408,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":942,"sample_per_second":322.6562210996617,"sample_per_token_ms":3.0992738853503186,"sample_time":2919.516,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138982,"total_time":199647.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":946,"eval_per_second":5.88493401278235,"eval_per_token_ms":169.9254397463002,"eval_time":160749.46600000001,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":160749.46600000001,"predicted_n":946,"predicted_per_second":5.88493401278235,"predicted_per_token_ms":169.9254397463002,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":947,"sample_per_second":322.3916490009433,"sample_per_token_ms":3.1018173178458293,"sample_time":2937.4210000000003,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138983,"total_time":200651.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":951,"eval_per_second":5.882129800816055,"eval_per_token_ms":170.00644900105152,"eval_time":161676.133,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":161676.133,"predicted_n":951,"predicted_per_second":5.882129800816055,"predicted_per_token_ms":170.00644900105152,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":952,"sample_per_second":322.5787869885542,"sample_per_token_ms":3.1000178571428574,"sample_time":2951.217,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138984,"total_time":201660.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":956,"eval_per_second":5.879757157910885,"eval_per_token_ms":170.07505125523014,"eval_time":162591.749,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":162591.749,"predicted_n":956,"predicted_per_second":5.879757157910885,"predicted_per_token_ms":170.07505125523014,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":957,"sample_per_second":322.76233292749146,"sample_per_token_ms":3.0982549634273773,"sample_time":2965.03,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138985,"total_time":202664.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":961,"eval_per_second":5.873113886296967,"eval_per_token_ms":170.26742872008325,"eval_time":163626.999,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":163626.999,"predicted_n":961,"predicted_per_second":5.873113886296967,"predicted_per_token_ms":170.26742872008325,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":962,"sample_per_second":322.2643458720517,"sample_per_token_ms":3.1030426195426193,"sample_time":2985.127,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138987,"total_time":203673.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":966,"eval_per_second":5.8702114479332295,"eval_per_token_ms":170.3516149068323,"eval_time":164559.66,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":164559.66,"predicted_n":966,"predicted_per_second":5.8702114479332295,"predicted_per_token_ms":170.3516149068323,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":967,"sample_per_second":322.42812720273093,"sample_per_token_ms":3.10146639089969,"sample_time":2999.118,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138988,"total_time":204688.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":972,"eval_per_second":5.866513248895497,"eval_per_token_ms":170.45900308641976,"eval_time":165686.151,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":165686.151,"predicted_n":972,"predicted_per_second":5.866513248895497,"predicted_per_token_ms":170.45900308641976,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":973,"sample_per_second":322.6111275971771,"sample_per_token_ms":3.099707091469681,"sample_time":3016.015,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138989,"total_time":205699.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":977,"eval_per_second":5.861880757621629,"eval_per_token_ms":170.59371238485159,"eval_time":166670.057,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":166670.057,"predicted_n":977,"predicted_per_second":5.861880757621629,"predicted_per_token_ms":170.59371238485159,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":978,"sample_per_second":322.461309588575,"sample_per_token_ms":3.1011472392638035,"sample_time":3032.922,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138990,"total_time":206713.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":982,"eval_per_second":5.85838525327264,"eval_per_token_ms":170.6955,"eval_time":167622.981,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":167622.981,"predicted_n":982,"predicted_per_second":5.85838525327264,"predicted_per_token_ms":170.6955,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":983,"sample_per_second":322.6204526269772,"sample_per_token_ms":3.099617497456765,"sample_time":3046.924,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138991,"total_time":207724.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":987,"eval_per_second":5.855229156554358,"eval_per_token_ms":170.78750861195542,"eval_time":168567.271,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":168567.271,"predicted_n":987,"predicted_per_second":5.855229156554358,"predicted_per_token_ms":170.78750861195542,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":988,"sample_per_second":322.81410253268575,"sample_per_token_ms":3.097758097165992,"sample_time":3060.585,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138992,"total_time":208739.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":992,"eval_per_second":5.852277323163396,"eval_per_token_ms":170.87365221774195,"eval_time":169506.663,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":169506.663,"predicted_n":992,"predicted_per_second":5.852277323163396,"predicted_per_token_ms":170.87365221774195,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":993,"sample_per_second":322.98092200807616,"sample_per_token_ms":3.0961581067472306,"sample_time":3074.485,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138993,"total_time":209753.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":997,"eval_per_second":5.848277143014197,"eval_per_token_ms":170.99052858575726,"eval_time":170477.557,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":170477.557,"predicted_n":997,"predicted_per_second":5.848277143014197,"predicted_per_token_ms":170.99052858575726,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":998,"sample_per_second":323.0848321059563,"sample_per_token_ms":3.0951623246492987,"sample_time":3088.972,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138994,"total_time":210757.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1002,"eval_per_second":5.8442917254155535,"eval_per_token_ms":171.10713273453095,"eval_time":171449.347,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":171449.347,"predicted_n":1002,"predicted_per_second":5.8442917254155535,"predicted_per_token_ms":171.10713273453095,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1003,"sample_per_second":323.27058295709753,"sample_per_token_ms":3.093383848454636,"sample_time":3102.664,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138995,"total_time":211764.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1007,"eval_per_second":5.839500206049693,"eval_per_token_ms":171.24753227408144,"eval_time":172446.265,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":172446.265,"predicted_n":1007,"predicted_per_second":5.839500206049693,"predicted_per_token_ms":171.24753227408144,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1008,"sample_per_second":323.23377433793195,"sample_per_token_ms":3.093736111111111,"sample_time":3118.486,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138996,"total_time":212771.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1013,"eval_per_second":5.835266195656037,"eval_per_token_ms":171.3717877591313,"eval_time":173599.621,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":173599.621,"predicted_n":1013,"predicted_per_second":5.835266195656037,"predicted_per_token_ms":171.3717877591313,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1014,"sample_per_second":323.3985552313065,"sample_per_token_ms":3.0921597633136098,"sample_time":3135.4500000000003,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138997,"total_time":213779.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1018,"eval_per_second":5.832077942272867,"eval_per_token_ms":171.4654724950884,"eval_time":174551.851,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":174551.851,"predicted_n":1018,"predicted_per_second":5.832077942272867,"predicted_per_token_ms":171.4654724950884,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1019,"sample_per_second":323.56766533386843,"sample_per_token_ms":3.0905436702649656,"sample_time":3149.264,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138998,"total_time":214795.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1023,"eval_per_second":5.8274038834241,"eval_per_token_ms":171.60300195503422,"eval_time":175549.871,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":175549.871,"predicted_n":1023,"predicted_per_second":5.8274038834241,"predicted_per_token_ms":171.60300195503422,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1024,"sample_per_second":323.613652071712,"sample_per_token_ms":3.0901044921875003,"sample_time":3164.2670000000003,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694138999,"total_time":215806.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1028,"eval_per_second":5.822516093394834,"eval_per_token_ms":171.7470564202335,"eval_time":176555.97400000002,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":176555.97400000002,"predicted_n":1028,"predicted_per_second":5.822516093394834,"predicted_per_token_ms":171.7470564202335,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1029,"sample_per_second":322.80519914006453,"sample_per_token_ms":3.097843537414966,"sample_time":3187.681,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139000,"total_time":216814.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1032,"eval_per_second":5.819091646898624,"eval_per_token_ms":171.8481269379845,"eval_time":177347.267,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":177347.267,"predicted_n":1032,"predicted_per_second":5.819091646898624,"predicted_per_token_ms":171.8481269379845,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1033,"sample_per_second":322.9131076150913,"sample_per_token_ms":3.096808325266215,"sample_time":3199.003,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139001,"total_time":217819.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1038,"eval_per_second":5.814979161114636,"eval_per_token_ms":171.96966184971097,"eval_time":178504.509,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":178504.509,"predicted_n":1038,"predicted_per_second":5.814979161114636,"predicted_per_token_ms":171.96966184971097,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1039,"sample_per_second":322.97773660683754,"sample_per_token_ms":3.0961886429258905,"sample_time":3216.94,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139002,"total_time":218831.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1043,"eval_per_second":5.812478725310821,"eval_per_token_ms":172.04364046021092,"eval_time":179441.517,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":179441.517,"predicted_n":1043,"predicted_per_second":5.812478725310821,"predicted_per_token_ms":172.04364046021092,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1044,"sample_per_second":323.02030170219945,"sample_per_token_ms":3.0957806513409962,"sample_time":3231.995,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139003,"total_time":219837.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1048,"eval_per_second":5.809461578491133,"eval_per_token_ms":172.13299141221373,"eval_time":180395.375,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":180395.375,"predicted_n":1048,"predicted_per_second":5.809461578491133,"predicted_per_token_ms":172.13299141221373,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1049,"sample_per_second":322.919164621358,"sample_per_token_ms":3.0967502383222114,"sample_time":3248.491,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139004,"total_time":220849.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1053,"eval_per_second":5.806804760130684,"eval_per_token_ms":172.21174833808166,"eval_time":181338.971,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":181338.971,"predicted_n":1053,"predicted_per_second":5.806804760130684,"predicted_per_token_ms":172.21174833808166,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1054,"sample_per_second":323.0562222038319,"sample_per_token_ms":3.0954364326375714,"sample_time":3262.59,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139005,"total_time":221861.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1058,"eval_per_second":5.8037442170118485,"eval_per_token_ms":172.30256238185257,"eval_time":182296.111,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":182296.111,"predicted_n":1058,"predicted_per_second":5.8037442170118485,"predicted_per_token_ms":172.30256238185257,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1059,"sample_per_second":323.23639293824624,"sample_per_token_ms":3.0937110481586405,"sample_time":3276.2400000000002,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139006,"total_time":222869.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1063,"eval_per_second":5.799228884773042,"eval_per_token_ms":172.43671872060207,"eval_time":183300.23200000002,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":183300.23200000002,"predicted_n":1063,"predicted_per_second":5.799228884773042,"predicted_per_token_ms":172.43671872060207,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1064,"sample_per_second":323.2407681950527,"sample_per_token_ms":3.093669172932331,"sample_time":3291.664,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139007,"total_time":223885.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1068,"eval_per_second":5.7959269741855275,"eval_per_token_ms":172.53495505617977,"eval_time":184267.332,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":184267.332,"predicted_n":1068,"predicted_per_second":5.7959269741855275,"predicted_per_token_ms":172.53495505617977,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1069,"sample_per_second":323.3413104971894,"sample_per_token_ms":3.092707202993452,"sample_time":3306.1040000000003,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139008,"total_time":224888.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1073,"eval_per_second":5.792838048163631,"eval_per_token_ms":172.6269561975769,"eval_time":185228.72400000002,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":185228.72400000002,"predicted_n":1073,"predicted_per_second":5.792838048163631,"predicted_per_token_ms":172.6269561975769,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1074,"sample_per_second":323.5086897024533,"sample_per_token_ms":3.0911070763500934,"sample_time":3319.849,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139009,"total_time":225890.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1079,"eval_per_second":5.789344407395919,"eval_per_token_ms":172.7311297497683,"eval_time":186376.889,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":186376.889,"predicted_n":1079,"predicted_per_second":5.789344407395919,"predicted_per_token_ms":172.7311297497683,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1080,"sample_per_second":323.53317398325953,"sample_per_token_ms":3.090873148148148,"sample_time":3338.143,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139010,"total_time":226906.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1084,"eval_per_second":5.785466717602968,"eval_per_token_ms":172.84690221402215,"eval_time":187366.04200000002,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":187366.04200000002,"predicted_n":1084,"predicted_per_second":5.785466717602968,"predicted_per_token_ms":172.84690221402215,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1085,"sample_per_second":323.5311833369215,"sample_per_token_ms":3.0908921658986173,"sample_time":3353.618,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139011,"total_time":227909.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1088,"eval_per_second":5.781811933132665,"eval_per_token_ms":172.95616176470588,"eval_time":188176.304,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":188176.304,"predicted_n":1088,"predicted_per_second":5.781811933132665,"predicted_per_token_ms":172.95616176470588,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1089,"sample_per_second":323.41411261582317,"sample_per_token_ms":3.092011019283747,"sample_time":3367.2000000000003,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139012,"total_time":228911.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1093,"eval_per_second":5.77792624528425,"eval_per_token_ms":173.07247575480332,"eval_time":189168.21600000001,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":189168.21600000001,"predicted_n":1093,"predicted_per_second":5.77792624528425,"predicted_per_token_ms":173.07247575480332,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1094,"sample_per_second":323.2990451152883,"sample_per_token_ms":3.093111517367459,"sample_time":3383.864,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139013,"total_time":229913.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1098,"eval_per_second":5.772871633711317,"eval_per_token_ms":173.224014571949,"eval_time":190199.968,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":190199.968,"predicted_n":1098,"predicted_per_second":5.772871633711317,"predicted_per_token_ms":173.224014571949,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1099,"sample_per_second":322.9755837511836,"sample_per_token_ms":3.096209281164695,"sample_time":3402.734,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139014,"total_time":230924.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1103,"eval_per_second":5.769535944245548,"eval_per_token_ms":173.32416500453309,"eval_time":191176.554,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":191176.554,"predicted_n":1103,"predicted_per_second":5.769535944245548,"predicted_per_token_ms":173.32416500453309,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1104,"sample_per_second":323.10537039757764,"sample_per_token_ms":3.094965579710145,"sample_time":3416.842,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139015,"total_time":231939.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1108,"eval_per_second":5.767087159669444,"eval_per_token_ms":173.39776083032493,"eval_time":192124.719,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":192124.719,"predicted_n":1108,"predicted_per_second":5.767087159669444,"predicted_per_token_ms":173.39776083032493,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1109,"sample_per_second":323.1814341737119,"sample_per_token_ms":3.0942371505861135,"sample_time":3431.509,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139016,"total_time":232945.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1113,"eval_per_second":5.763775498531107,"eval_per_token_ms":173.49738903863434,"eval_time":193102.594,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":193102.594,"predicted_n":1113,"predicted_per_second":5.763775498531107,"predicted_per_token_ms":173.49738903863434,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1114,"sample_per_second":323.18210789582633,"sample_per_token_ms":3.094230700179533,"sample_time":3446.973,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139017,"total_time":233951.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1119,"eval_per_second":5.7606007086465505,"eval_per_token_ms":173.5930071492404,"eval_time":194250.575,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":194250.575,"predicted_n":1119,"predicted_per_second":5.7606007086465505,"predicted_per_token_ms":173.5930071492404,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1120,"sample_per_second":323.35671991562697,"sample_per_token_ms":3.0925598214285714,"sample_time":3463.667,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139018,"total_time":234955.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1124,"eval_per_second":5.757924812751414,"eval_per_token_ms":173.6736814946619,"eval_time":195209.218,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":195209.218,"predicted_n":1124,"predicted_per_second":5.757924812751414,"predicted_per_token_ms":173.6736814946619,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1125,"sample_per_second":323.48677916722136,"sample_per_token_ms":3.0913164444444448,"sample_time":3477.731,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139019,"total_time":235972.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1129,"eval_per_second":5.754142943420053,"eval_per_token_ms":173.78782728077945,"eval_time":196206.457,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":196206.457,"predicted_n":1129,"predicted_per_second":5.754142943420053,"predicted_per_token_ms":173.78782728077945,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1130,"sample_per_second":323.2789757377698,"sample_per_token_ms":3.0933035398230087,"sample_time":3495.433,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139020,"total_time":236984.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1134,"eval_per_second":5.751338914488743,"eval_per_token_ms":173.87255643738976,"eval_time":197171.479,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":197171.479,"predicted_n":1134,"predicted_per_second":5.751338914488743,"predicted_per_token_ms":173.87255643738976,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1135,"sample_per_second":323.40098160033625,"sample_per_token_ms":3.092136563876652,"sample_time":3509.5750000000003,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139021,"total_time":237992.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1139,"eval_per_second":5.748637605693547,"eval_per_token_ms":173.95425987708518,"eval_time":198133.902,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":198133.902,"predicted_n":1139,"predicted_per_second":5.748637605693547,"predicted_per_token_ms":173.95425987708518,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1140,"sample_per_second":323.5072269811838,"sample_per_token_ms":3.091121052631579,"sample_time":3523.878,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139022,"total_time":238997.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1144,"eval_per_second":5.744863885794998,"eval_per_token_ms":174.06852797202797,"eval_time":199134.396,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":199134.396,"predicted_n":1144,"predicted_per_second":5.744863885794998,"predicted_per_token_ms":174.06852797202797,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1145,"sample_per_second":323.5060940637056,"sample_per_token_ms":3.0911318777292576,"sample_time":3539.346,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139023,"total_time":240003.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1149,"eval_per_second":5.741527208560266,"eval_per_token_ms":174.16968755439513,"eval_time":200120.971,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":200120.971,"predicted_n":1149,"predicted_per_second":5.741527208560266,"predicted_per_token_ms":174.16968755439513,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1150,"sample_per_second":323.47421432332567,"sample_per_token_ms":3.0914365217391304,"sample_time":3555.152,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139024,"total_time":241012.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1153,"eval_per_second":5.737008789893582,"eval_per_token_ms":174.3068620988725,"eval_time":200975.812,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":200975.812,"predicted_n":1153,"predicted_per_second":5.737008789893582,"predicted_per_token_ms":174.3068620988725,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1154,"sample_per_second":323.4441816343685,"sample_per_token_ms":3.0917235701906414,"sample_time":3567.849,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139025,"total_time":242015.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1158,"eval_per_second":5.733935584729966,"eval_per_token_ms":174.40028497409327,"eval_time":201955.53,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":201955.53,"predicted_n":1158,"predicted_per_second":5.733935584729966,"predicted_per_token_ms":174.40028497409327,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1159,"sample_per_second":323.34163959598754,"sample_per_token_ms":3.0927040552200173,"sample_time":3584.444,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139026,"total_time":243032.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1163,"eval_per_second":5.730525238657159,"eval_per_token_ms":174.5040739466896,"eval_time":202948.238,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":202948.238,"predicted_n":1163,"predicted_per_second":5.730525238657159,"predicted_per_token_ms":174.5040739466896,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1164,"sample_per_second":323.45786461120866,"sample_per_token_ms":3.091592783505155,"sample_time":3598.614,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139027,"total_time":244046.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1168,"eval_per_second":5.727621488956994,"eval_per_token_ms":174.59254280821918,"eval_time":203924.09,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":203924.09,"predicted_n":1168,"predicted_per_second":5.727621488956994,"predicted_per_token_ms":174.59254280821918,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1169,"sample_per_second":323.57667092474003,"sample_per_token_ms":3.090457656116339,"sample_time":3612.745,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139028,"total_time":245055.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1173,"eval_per_second":5.724362935502505,"eval_per_token_ms":174.69192838874682,"eval_time":204913.632,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":204913.632,"predicted_n":1173,"predicted_per_second":5.724362935502505,"predicted_per_token_ms":174.69192838874682,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1174,"sample_per_second":323.5811558903209,"sample_per_token_ms":3.090414821124361,"sample_time":3628.147,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139029,"total_time":246060.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1178,"eval_per_second":5.721808831248611,"eval_per_token_ms":174.76990747028862,"eval_time":205878.951,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":205878.951,"predicted_n":1178,"predicted_per_second":5.721808831248611,"predicted_per_token_ms":174.76990747028862,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1179,"sample_per_second":323.7233178811965,"sample_per_token_ms":3.0890576759966075,"sample_time":3641.9990000000003,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139030,"total_time":247063.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1183,"eval_per_second":5.718887315687283,"eval_per_token_ms":174.85918934911243,"eval_time":206858.421,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":206858.421,"predicted_n":1183,"predicted_per_second":5.718887315687283,"predicted_per_token_ms":174.85918934911243,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1184,"sample_per_second":323.7520234501466,"sample_per_token_ms":3.0887837837837835,"sample_time":3657.12,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139031,"total_time":248069.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1188,"eval_per_second":5.715528229985004,"eval_per_token_ms":174.96195622895624,"eval_time":207854.804,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":207854.804,"predicted_n":1188,"predicted_per_second":5.715528229985004,"predicted_per_token_ms":174.96195622895624,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1189,"sample_per_second":323.7194906631839,"sample_per_token_ms":3.089094196804037,"sample_time":3672.933,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139032,"total_time":249084.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1193,"eval_per_second":5.7128043938888124,"eval_per_token_ms":175.0453772003353,"eval_time":208829.135,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":208829.135,"predicted_n":1193,"predicted_per_second":5.7128043938888124,"predicted_per_token_ms":175.0453772003353,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1194,"sample_per_second":323.8183883282184,"sample_per_token_ms":3.0881507537688444,"sample_time":3687.252,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139033,"total_time":250089.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1198,"eval_per_second":5.709840318310157,"eval_per_token_ms":175.13624624373958,"eval_time":209813.223,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":209813.223,"predicted_n":1198,"predicted_per_second":5.709840318310157,"predicted_per_token_ms":175.13624624373958,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1199,"sample_per_second":323.6057289279771,"sample_per_token_ms":3.0901801501251045,"sample_time":3705.126,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139034,"total_time":251095.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1203,"eval_per_second":5.706776840191819,"eval_per_token_ms":175.23026184538654,"eval_time":210802.005,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":210802.005,"predicted_n":1203,"predicted_per_second":5.706776840191819,"predicted_per_token_ms":175.23026184538654,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1204,"sample_per_second":323.6686171016389,"sample_per_token_ms":3.0895797342192695,"sample_time":3719.8540000000003,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139035,"total_time":252106.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1208,"eval_per_second":5.703612566214315,"eval_per_token_ms":175.32747682119205,"eval_time":211795.592,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":211795.592,"predicted_n":1208,"predicted_per_second":5.703612566214315,"predicted_per_token_ms":175.32747682119205,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1209,"sample_per_second":323.8033204169108,"sample_per_token_ms":3.0882944582299423,"sample_time":3733.748,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139036,"total_time":253118.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1213,"eval_per_second":5.698654287454432,"eval_per_token_ms":175.48002555647156,"eval_time":212857.271,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":212857.271,"predicted_n":1213,"predicted_per_second":5.698654287454432,"predicted_per_token_ms":175.48002555647156,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1214,"sample_per_second":323.7220246439391,"sample_per_token_ms":3.0890700164744644,"sample_time":3750.131,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139037,"total_time":254130.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1217,"eval_per_second":5.692031641185401,"eval_per_token_ms":175.6841955628595,"eval_time":213807.666,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":213807.666,"predicted_n":1217,"predicted_per_second":5.692031641185401,"predicted_per_token_ms":175.6841955628595,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1218,"sample_per_second":323.3480379702982,"sample_per_token_ms":3.0926428571428572,"sample_time":3766.839,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139038,"total_time":255166.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1222,"eval_per_second":5.688886518571233,"eval_per_token_ms":175.7813232405892,"eval_time":214804.777,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":214804.777,"predicted_n":1222,"predicted_per_second":5.688886518571233,"predicted_per_token_ms":175.7813232405892,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1223,"sample_per_second":323.45281516351025,"sample_per_token_ms":3.091641046606705,"sample_time":3781.077,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139039,"total_time":256172.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1227,"eval_per_second":5.686084061379938,"eval_per_token_ms":175.86795925020374,"eval_time":215789.986,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":215789.986,"predicted_n":1227,"predicted_per_second":5.686084061379938,"predicted_per_token_ms":175.86795925020374,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1228,"sample_per_second":323.5400977628572,"sample_per_token_ms":3.090807003257329,"sample_time":3795.511,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139040,"total_time":257181.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1232,"eval_per_second":5.681009831591582,"eval_per_token_ms":176.02504301948053,"eval_time":216862.853,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":216862.853,"predicted_n":1232,"predicted_per_second":5.681009831591582,"predicted_per_token_ms":176.02504301948053,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1233,"sample_per_second":323.535516354025,"sample_per_token_ms":3.0908507704785078,"sample_time":3811.0190000000002,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139041,"total_time":258185.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1237,"eval_per_second":5.677661251902091,"eval_per_token_ms":176.1288593371059,"eval_time":217871.399,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":217871.399,"predicted_n":1237,"predicted_per_second":5.677661251902091,"predicted_per_token_ms":176.1288593371059,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1238,"sample_per_second":323.38696384831525,"sample_per_token_ms":3.092270597738288,"sample_time":3828.231,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139042,"total_time":259189.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1241,"eval_per_second":5.675181304436352,"eval_per_token_ms":176.20582433521355,"eval_time":218671.428,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":218671.428,"predicted_n":1241,"predicted_per_second":5.675181304436352,"predicted_per_token_ms":176.20582433521355,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1242,"sample_per_second":323.4802937471936,"sample_per_token_ms":3.091378421900161,"sample_time":3839.492,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139043,"total_time":260201.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1246,"eval_per_second":5.670816467907204,"eval_per_token_ms":176.34145024077048,"eval_time":219721.44700000001,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":219721.44700000001,"predicted_n":1246,"predicted_per_second":5.670816467907204,"predicted_per_token_ms":176.34145024077048,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1247,"sample_per_second":323.2282387949056,"sample_per_token_ms":3.09378909382518,"sample_time":3857.955,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139044,"total_time":261204.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1251,"eval_per_second":5.666179289190716,"eval_per_token_ms":176.48576738609114,"eval_time":220783.695,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":220783.695,"predicted_n":1251,"predicted_per_second":5.666179289190716,"predicted_per_token_ms":176.48576738609114,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1252,"sample_per_second":323.3420134651804,"sample_per_token_ms":3.092700479233227,"sample_time":3872.061,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139045,"total_time":262208.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1255,"eval_per_second":5.662582449738479,"eval_per_token_ms":176.5978701195219,"eval_time":221630.327,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":221630.327,"predicted_n":1255,"predicted_per_second":5.662582449738479,"predicted_per_token_ms":176.5978701195219,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1256,"sample_per_second":323.0308939442766,"sample_per_token_ms":3.095679140127389,"sample_time":3888.1730000000002,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139046,"total_time":263216.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1260,"eval_per_second":5.6580792276854766,"eval_per_token_ms":176.73842301587302,"eval_time":222690.413,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":222690.413,"predicted_n":1260,"predicted_per_second":5.6580792276854766,"predicted_per_token_ms":176.73842301587302,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1261,"sample_per_second":323.1111322931256,"sample_per_token_ms":3.094910388580492,"sample_time":3902.6820000000002,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139047,"total_time":264217.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1265,"eval_per_second":5.654942827276329,"eval_per_token_ms":176.83644743083005,"eval_time":223698.106,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":223698.106,"predicted_n":1265,"predicted_per_second":5.654942827276329,"predicted_per_token_ms":176.83644743083005,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1266,"sample_per_second":323.0758865957703,"sample_per_token_ms":3.0952480252764616,"sample_time":3918.5840000000003,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139048,"total_time":265221.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1269,"eval_per_second":5.6507287247002145,"eval_per_token_ms":176.96832545311267,"eval_time":224572.805,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":224572.805,"predicted_n":1269,"predicted_per_second":5.6507287247002145,"predicted_per_token_ms":176.96832545311267,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1270,"sample_per_second":322.97416995002544,"sample_per_token_ms":3.096222834645669,"sample_time":3932.203,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139049,"total_time":266233.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1274,"eval_per_second":5.645596842815626,"eval_per_token_ms":177.1291907378336,"eval_time":225662.589,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":225662.589,"predicted_n":1274,"predicted_per_second":5.645596842815626,"predicted_per_token_ms":177.1291907378336,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1275,"sample_per_second":322.9044891575004,"sample_per_token_ms":3.096890980392157,"sample_time":3948.536,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139050,"total_time":267243.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1279,"eval_per_second":5.642620339356628,"eval_per_token_ms":177.22262705238467,"eval_time":226667.74,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":226667.74,"predicted_n":1279,"predicted_per_second":5.642620339356628,"predicted_per_token_ms":177.22262705238467,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1280,"sample_per_second":323.0487727932362,"sample_per_token_ms":3.0955078125,"sample_time":3962.25,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139051,"total_time":268252.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1284,"eval_per_second":5.640103642439111,"eval_per_token_ms":177.30170638629284,"eval_time":227655.391,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":227655.391,"predicted_n":1284,"predicted_per_second":5.640103642439111,"predicted_per_token_ms":177.30170638629284,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1285,"sample_per_second":323.1576805778713,"sample_per_token_ms":3.0944645914396887,"sample_time":3976.387,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139052,"total_time":269266.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1288,"eval_per_second":5.63530249951611,"eval_per_token_ms":177.45276319875776,"eval_time":228559.159,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":228559.159,"predicted_n":1288,"predicted_per_second":5.63530249951611,"predicted_per_token_ms":177.45276319875776,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1289,"sample_per_second":322.84823778461487,"sample_per_token_ms":3.097430566330489,"sample_time":3992.588,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139053,"total_time":270272.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1293,"eval_per_second":5.6320133675510595,"eval_per_token_ms":177.55639675174015,"eval_time":229580.421,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":229580.421,"predicted_n":1293,"predicted_per_second":5.6320133675510595,"predicted_per_token_ms":177.55639675174015,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1294,"sample_per_second":322.9826625203081,"sample_per_token_ms":3.0961414219474497,"sample_time":4006.407,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139054,"total_time":271277.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1298,"eval_per_second":5.629118282233878,"eval_per_token_ms":177.64771494607086,"eval_time":230586.734,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":230586.734,"predicted_n":1298,"predicted_per_second":5.629118282233878,"predicted_per_token_ms":177.64771494607086,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1299,"sample_per_second":323.09976793527056,"sample_per_token_ms":3.095019245573518,"sample_time":4020.4300000000003,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139055,"total_time":272284.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1303,"eval_per_second":5.625692571651172,"eval_per_token_ms":177.75589178818112,"eval_time":231615.927,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":231615.927,"predicted_n":1303,"predicted_per_second":5.625692571651172,"predicted_per_token_ms":177.75589178818112,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1304,"sample_per_second":322.76045332992754,"sample_per_token_ms":3.0982730061349693,"sample_time":4040.148,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139056,"total_time":273298.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1308,"eval_per_second":5.6230222937486705,"eval_per_token_ms":177.84030504587156,"eval_time":232615.119,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":232615.119,"predicted_n":1308,"predicted_per_second":5.6230222937486705,"predicted_per_token_ms":177.84030504587156,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1309,"sample_per_second":322.9108849830404,"sample_per_token_ms":3.096829640947288,"sample_time":4053.75,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139057,"total_time":274310.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1313,"eval_per_second":5.620106427694319,"eval_per_token_ms":177.93257349581114,"eval_time":233625.469,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":233625.469,"predicted_n":1313,"predicted_per_second":5.620106427694319,"predicted_per_token_ms":177.93257349581114,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1314,"sample_per_second":323.0531621594113,"sample_per_token_ms":3.0954657534246577,"sample_time":4067.442,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139058,"total_time":275320.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1317,"eval_per_second":5.617259609790855,"eval_per_token_ms":178.02274943052393,"eval_time":234455.961,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":234455.961,"predicted_n":1317,"predicted_per_second":5.617259609790855,"predicted_per_token_ms":178.02274943052393,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1318,"sample_per_second":323.0008197554947,"sample_per_token_ms":3.0959673748103187,"sample_time":4080.485,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139059,"total_time":276333.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1322,"eval_per_second":5.614555101294728,"eval_per_token_ms":178.10850226928895,"eval_time":235459.44,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":235459.44,"predicted_n":1322,"predicted_per_second":5.614555101294728,"predicted_per_token_ms":178.10850226928895,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1323,"sample_per_second":323.1530751446922,"sample_per_token_ms":3.0945086923658356,"sample_time":4094.0350000000003,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139060,"total_time":277345.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1327,"eval_per_second":5.610988200730291,"eval_per_token_ms":178.22172569706106,"eval_time":236500.23,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":236500.23,"predicted_n":1327,"predicted_per_second":5.610988200730291,"predicted_per_token_ms":178.22172569706106,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1328,"sample_per_second":323.2954321374161,"sample_per_token_ms":3.0931460843373495,"sample_time":4107.698,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139061,"total_time":278356.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1332,"eval_per_second":5.607225285278951,"eval_per_token_ms":178.34132732732735,"eval_time":237550.64800000002,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":237550.64800000002,"predicted_n":1332,"predicted_per_second":5.607225285278951,"predicted_per_token_ms":178.34132732732735,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1333,"sample_per_second":323.18669353342943,"sample_per_token_ms":3.094186796699175,"sample_time":4124.551,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139062,"total_time":279365.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1336,"eval_per_second":5.60442907626466,"eval_per_token_ms":178.43030688622756,"eval_time":238382.89,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":238382.89,"predicted_n":1336,"predicted_per_second":5.60442907626466,"predicted_per_token_ms":178.43030688622756,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1337,"sample_per_second":323.1020113039789,"sample_per_token_ms":3.094997756170531,"sample_time":4138.012,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139063,"total_time":280377.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1341,"eval_per_second":5.6012330331352285,"eval_per_token_ms":178.53211856823265,"eval_time":239411.571,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":239411.571,"predicted_n":1341,"predicted_per_second":5.6012330331352285,"predicted_per_token_ms":178.53211856823265,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1342,"sample_per_second":323.2514373488372,"sample_per_token_ms":3.0935670640834574,"sample_time":4151.567,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139064,"total_time":281387.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1345,"eval_per_second":5.596064591083953,"eval_per_token_ms":178.69700817843867,"eval_time":240347.476,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":240347.476,"predicted_n":1345,"predicted_per_second":5.596064591083953,"predicted_per_token_ms":178.69700817843867,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1346,"sample_per_second":323.3565272373822,"sample_per_token_ms":3.092561664190193,"sample_time":4162.588,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139065,"total_time":282394.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1350,"eval_per_second":5.592930648347684,"eval_per_token_ms":178.79713925925927,"eval_time":241376.138,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":241376.138,"predicted_n":1350,"predicted_per_second":5.592930648347684,"predicted_per_token_ms":178.79713925925927,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1351,"sample_per_second":323.26111362616183,"sample_per_token_ms":3.0934744633604736,"sample_time":4179.284,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139066,"total_time":283401.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1355,"eval_per_second":5.590160620909329,"eval_per_token_ms":178.88573653136532,"eval_time":242390.173,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":242390.173,"predicted_n":1355,"predicted_per_second":5.590160620909329,"predicted_per_token_ms":178.88573653136532,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1356,"sample_per_second":323.39675344002563,"sample_per_token_ms":3.0921769911504424,"sample_time":4192.992,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139067,"total_time":284407.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1360,"eval_per_second":5.586555094179131,"eval_per_token_ms":179.00118823529414,"eval_time":243441.616,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":243441.616,"predicted_n":1360,"predicted_per_second":5.586555094179131,"predicted_per_token_ms":179.00118823529414,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1361,"sample_per_second":323.3478020189637,"sample_per_token_ms":3.092645113886848,"sample_time":4209.09,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139068,"total_time":285421.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1365,"eval_per_second":5.583659543472878,"eval_per_token_ms":179.09401391941392,"eval_time":244463.329,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":244463.329,"predicted_n":1365,"predicted_per_second":5.583659543472878,"predicted_per_token_ms":179.09401391941392,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1366,"sample_per_second":323.34705388462663,"sample_per_token_ms":3.092652269399707,"sample_time":4224.563,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139069,"total_time":286437.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1370,"eval_per_second":5.581060822457875,"eval_per_token_ms":179.17740583941605,"eval_time":245473.046,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":245473.046,"predicted_n":1370,"predicted_per_second":5.581060822457875,"predicted_per_token_ms":179.17740583941605,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1371,"sample_per_second":323.476234642252,"sample_per_token_ms":3.091417213712619,"sample_time":4238.3330000000005,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139070,"total_time":287443.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1374,"eval_per_second":5.577936110782586,"eval_per_token_ms":179.27777947598253,"eval_time":246327.669,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":246327.669,"predicted_n":1374,"predicted_per_second":5.577936110782586,"predicted_per_token_ms":179.27777947598253,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1375,"sample_per_second":323.4036267776321,"sample_per_token_ms":3.092111272727273,"sample_time":4251.653,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139071,"total_time":288451.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1379,"eval_per_second":5.574863376835352,"eval_per_token_ms":179.3765931834663,"eval_time":247360.32200000001,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":247360.32200000001,"predicted_n":1379,"predicted_per_second":5.574863376835352,"predicted_per_token_ms":179.3765931834663,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1380,"sample_per_second":323.4312587874281,"sample_per_token_ms":3.091847101449275,"sample_time":4266.749,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139072,"total_time":289460.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1384,"eval_per_second":5.571469669643731,"eval_per_token_ms":179.4858554913295,"eval_time":248408.424,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":248408.424,"predicted_n":1384,"predicted_per_second":5.571469669643731,"predicted_per_token_ms":179.4858554913295,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1385,"sample_per_second":323.5029695703997,"sample_per_token_ms":3.0911617328519854,"sample_time":4281.259,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139073,"total_time":290468.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1388,"eval_per_second":5.567382030734604,"eval_per_token_ms":179.61763616714697,"eval_time":249309.279,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":249309.279,"predicted_n":1388,"predicted_per_second":5.567382030734604,"predicted_per_token_ms":179.61763616714697,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1389,"sample_per_second":323.50203883142393,"sample_per_token_ms":3.0911706263498924,"sample_time":4293.636,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139074,"total_time":291478.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1393,"eval_per_second":5.564334950546903,"eval_per_token_ms":179.71599641062454,"eval_time":250344.383,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":250344.383,"predicted_n":1393,"predicted_per_second":5.564334950546903,"predicted_per_token_ms":179.71599641062454,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1394,"sample_per_second":323.58530662725457,"sample_per_token_ms":3.090375179340029,"sample_time":4307.983,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139075,"total_time":292493.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1397,"eval_per_second":5.561631437739174,"eval_per_token_ms":179.8033564781675,"eval_time":251185.28900000002,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":251185.28900000002,"predicted_n":1397,"predicted_per_second":5.561631437739174,"predicted_per_token_ms":179.8033564781675,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1398,"sample_per_second":323.6447491880522,"sample_per_token_ms":3.0898075822603723,"sample_time":4319.551,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139076,"total_time":293495.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1402,"eval_per_second":5.558084400264853,"eval_per_token_ms":179.91810271041368,"eval_time":252245.18,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":252245.18,"predicted_n":1402,"predicted_per_second":5.558084400264853,"predicted_per_token_ms":179.91810271041368,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1403,"sample_per_second":323.6564738330777,"sample_per_token_ms":3.089695652173913,"sample_time":4334.843,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139077,"total_time":294507.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1407,"eval_per_second":5.554704935159518,"eval_per_token_ms":180.02756432125088,"eval_time":253298.783,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":253298.783,"predicted_n":1407,"predicted_per_second":5.554704935159518,"predicted_per_token_ms":180.02756432125088,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1408,"sample_per_second":323.7806542438836,"sample_per_token_ms":3.088510653409091,"sample_time":4348.6230000000005,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139078,"total_time":295522.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1412,"eval_per_second":5.5516630350651655,"eval_per_token_ms":180.12620609065155,"eval_time":254338.203,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":254338.203,"predicted_n":1412,"predicted_per_second":5.5516630350651655,"predicted_per_token_ms":180.12620609065155,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1413,"sample_per_second":323.8832617451234,"sample_per_token_ms":3.0875322009907995,"sample_time":4362.683,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139079,"total_time":296525.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1416,"eval_per_second":5.549273911603782,"eval_per_token_ms":180.20375564971752,"eval_time":255168.518,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":255168.518,"predicted_n":1416,"predicted_per_second":5.549273911603782,"predicted_per_token_ms":180.20375564971752,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1417,"sample_per_second":323.8181380662732,"sample_per_token_ms":3.0881531404375444,"sample_time":4375.9130000000005,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139080,"total_time":297539.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1421,"eval_per_second":5.546342269629141,"eval_per_token_ms":180.29900633356792,"eval_time":256204.888,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":256204.888,"predicted_n":1421,"predicted_per_second":5.546342269629141,"predicted_per_token_ms":180.29900633356792,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1422,"sample_per_second":323.81546617358055,"sample_per_token_ms":3.0881786216596345,"sample_time":4391.39,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139081,"total_time":298552.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1426,"eval_per_second":5.542921956284669,"eval_per_token_ms":180.4102615708275,"eval_time":257265.033,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":257265.033,"predicted_n":1426,"predicted_per_second":5.542921956284669,"predicted_per_token_ms":180.4102615708275,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1427,"sample_per_second":323.9209369600372,"sample_per_token_ms":3.087173090399439,"sample_time":4405.396,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139082,"total_time":299561.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1430,"eval_per_second":5.53881724553588,"eval_per_token_ms":180.54396013986016,"eval_time":258177.863,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":258177.863,"predicted_n":1430,"predicted_per_second":5.53881724553588,"predicted_per_token_ms":180.54396013986016,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1431,"sample_per_second":323.8122866284081,"sample_per_token_ms":3.0882089447938506,"sample_time":4419.227,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139083,"total_time":300576.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1434,"eval_per_second":5.534741137841192,"eval_per_token_ms":180.67692329149233,"eval_time":259090.708,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":259090.708,"predicted_n":1434,"predicted_per_second":5.534741137841192,"predicted_per_token_ms":180.67692329149233,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1435,"sample_per_second":323.7895516834687,"sample_per_token_ms":3.088425783972126,"sample_time":4431.8910000000005,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139084,"total_time":301577.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1439,"eval_per_second":5.531485733812029,"eval_per_token_ms":180.78325573314802,"eval_time":260147.105,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":260147.105,"predicted_n":1439,"predicted_per_second":5.531485733812029,"predicted_per_token_ms":180.78325573314802,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1440,"sample_per_second":323.870395164615,"sample_per_token_ms":3.087654861111111,"sample_time":4446.223,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139085,"total_time":302588.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1444,"eval_per_second":5.527479862307568,"eval_per_token_ms":180.9142728531856,"eval_time":261240.21,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":261240.21,"predicted_n":1444,"predicted_per_second":5.527479862307568,"predicted_per_token_ms":180.9142728531856,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1445,"sample_per_second":323.74640686297573,"sample_per_token_ms":3.0888373702422145,"sample_time":4463.37,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139086,"total_time":303593.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1448,"eval_per_second":5.524845288402208,"eval_per_token_ms":181.0005435082873,"eval_time":262088.787,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":262088.787,"predicted_n":1448,"predicted_per_second":5.524845288402208,"predicted_per_token_ms":181.0005435082873,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1449,"sample_per_second":323.7776110351633,"sample_per_token_ms":3.0885396825396825,"sample_time":4475.294,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139087,"total_time":304604.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1453,"eval_per_second":5.521262402411165,"eval_per_token_ms":181.11799931176873,"eval_time":263164.453,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":263164.453,"predicted_n":1453,"predicted_per_second":5.521262402411165,"predicted_per_token_ms":181.11799931176873,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1454,"sample_per_second":323.8564126687996,"sample_per_token_ms":3.087788170563962,"sample_time":4489.644,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139088,"total_time":305617.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1457,"eval_per_second":5.517897619049143,"eval_per_token_ms":181.22844406314343,"eval_time":264049.843,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":264049.843,"predicted_n":1457,"predicted_per_second":5.517897619049143,"predicted_per_token_ms":181.22844406314343,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1458,"sample_per_second":323.74560789566243,"sample_per_token_ms":3.0888449931412896,"sample_time":4503.536,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139089,"total_time":306630.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1462,"eval_per_second":5.514412913913173,"eval_per_token_ms":181.34296716826265,"eval_time":265123.418,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":265123.418,"predicted_n":1462,"predicted_per_second":5.514412913913173,"predicted_per_token_ms":181.34296716826265,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1463,"sample_per_second":323.87125995660614,"sample_per_token_ms":3.0876466165413534,"sample_time":4517.227,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139090,"total_time":307640.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1467,"eval_per_second":5.511511635496106,"eval_per_token_ms":181.43842672119973,"eval_time":266170.172,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":266170.172,"predicted_n":1467,"predicted_per_second":5.511511635496106,"predicted_per_token_ms":181.43842672119973,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1468,"sample_per_second":323.8252675915099,"sample_per_token_ms":3.0880851498637605,"sample_time":4533.309,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139091,"total_time":308652.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1471,"eval_per_second":5.507023574006785,"eval_per_token_ms":181.58629367777024,"eval_time":267113.438,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":267113.438,"predicted_n":1471,"predicted_per_second":5.507023574006785,"predicted_per_token_ms":181.58629367777024,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1472,"sample_per_second":323.7773555572468,"sample_per_token_ms":3.088542119565217,"sample_time":4546.334,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139092,"total_time":309655.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1475,"eval_per_second":5.503880420402879,"eval_per_token_ms":181.68999389830506,"eval_time":267992.741,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":267992.741,"predicted_n":1475,"predicted_per_second":5.503880420402879,"predicted_per_token_ms":181.68999389830506,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1476,"sample_per_second":323.58841132559434,"sample_per_token_ms":3.090345528455285,"sample_time":4561.35,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139093,"total_time":310661.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1480,"eval_per_second":5.50058932682223,"eval_per_token_ms":181.79870202702705,"eval_time":269062.079,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":269062.079,"predicted_n":1480,"predicted_per_second":5.50058932682223,"predicted_per_token_ms":181.79870202702705,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1481,"sample_per_second":323.6876171892331,"sample_per_token_ms":3.089398379473329,"sample_time":4575.399,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139094,"total_time":311666.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1484,"eval_per_second":5.496898776761114,"eval_per_token_ms":181.92075943396227,"eval_time":269970.407,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":269970.407,"predicted_n":1484,"predicted_per_second":5.496898776761114,"predicted_per_token_ms":181.92075943396227,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1485,"sample_per_second":323.6087981496114,"sample_per_token_ms":3.0901508417508414,"sample_time":4588.874,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139096,"total_time":312676.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1489,"eval_per_second":5.49329643333507,"eval_per_token_ms":182.0400577568838,"eval_time":271057.646,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":271057.646,"predicted_n":1489,"predicted_per_second":5.49329643333507,"predicted_per_token_ms":182.0400577568838,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1490,"sample_per_second":323.61187338479124,"sample_per_token_ms":3.0901214765100673,"sample_time":4604.281,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139097,"total_time":313688.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1493,"eval_per_second":5.490769380433646,"eval_per_token_ms":182.12383924983254,"eval_time":271910.892,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":271910.892,"predicted_n":1493,"predicted_per_second":5.490769380433646,"predicted_per_token_ms":182.12383924983254,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1494,"sample_per_second":323.44173177717596,"sample_per_token_ms":3.091746987951807,"sample_time":4619.07,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139098,"total_time":314690.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1498,"eval_per_second":5.485755421241949,"eval_per_token_ms":182.2902997329773,"eval_time":273070.869,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":273070.869,"predicted_n":1498,"predicted_per_second":5.485755421241949,"predicted_per_token_ms":182.2902997329773,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1499,"sample_per_second":323.5627038186011,"sample_per_token_ms":3.0905910607071383,"sample_time":4632.796,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139099,"total_time":315695.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1502,"eval_per_second":5.482944185081067,"eval_per_token_ms":182.38376431424766,"eval_time":273940.414,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":273940.414,"predicted_n":1502,"predicted_per_second":5.482944185081067,"predicted_per_token_ms":182.38376431424766,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1503,"sample_per_second":323.65783746174736,"sample_per_token_ms":3.0896826347305386,"sample_time":4643.793,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139100,"total_time":316703.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1507,"eval_per_second":5.479699652680491,"eval_per_token_ms":182.49175381552757,"eval_time":275015.07300000003,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":275015.07300000003,"predicted_n":1507,"predicted_per_second":5.479699652680491,"predicted_per_token_ms":182.49175381552757,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1508,"sample_per_second":323.71046031284,"sample_per_token_ms":3.0891803713527852,"sample_time":4658.484,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139101,"total_time":317714.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1511,"eval_per_second":5.47638510437314,"eval_per_token_ms":182.60220582395766,"eval_time":275911.933,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":275911.933,"predicted_n":1511,"predicted_per_second":5.47638510437314,"predicted_per_token_ms":182.60220582395766,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1512,"sample_per_second":323.6026390736917,"sample_per_token_ms":3.090209656084656,"sample_time":4672.397,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139102,"total_time":318719.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1515,"eval_per_second":5.473180832283451,"eval_per_token_ms":182.70911023102312,"eval_time":276804.302,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":276804.302,"predicted_n":1515,"predicted_per_second":5.473180832283451,"predicted_per_token_ms":182.70911023102312,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1516,"sample_per_second":323.5848764625086,"sample_per_token_ms":3.0903792875989446,"sample_time":4685.015,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139103,"total_time":319731.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1520,"eval_per_second":5.4694222503545635,"eval_per_token_ms":182.8346677631579,"eval_time":277908.695,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":277908.695,"predicted_n":1520,"predicted_per_second":5.4694222503545635,"predicted_per_token_ms":182.8346677631579,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1521,"sample_per_second":323.65510239821833,"sample_per_token_ms":3.089708744247206,"sample_time":4699.447,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139104,"total_time":320744.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1524,"eval_per_second":5.465195955658885,"eval_per_token_ms":182.97605577427822,"eval_time":278855.509,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":278855.509,"predicted_n":1524,"predicted_per_second":5.465195955658885,"predicted_per_token_ms":182.97605577427822,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1525,"sample_per_second":323.39613014647836,"sample_per_token_ms":3.092182950819672,"sample_time":4715.579,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139105,"total_time":321754.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1529,"eval_per_second":5.4618928256387385,"eval_per_token_ms":183.08671223021582,"eval_time":279939.583,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":279939.583,"predicted_n":1529,"predicted_per_second":5.4618928256387385,"predicted_per_token_ms":183.08671223021582,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1530,"sample_per_second":323.44780989894053,"sample_per_token_ms":3.0916888888888887,"sample_time":4730.284,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139106,"total_time":322762.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1533,"eval_per_second":5.4592639057601,"eval_per_token_ms":183.1748780169602,"eval_time":280807.088,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":280807.088,"predicted_n":1533,"predicted_per_second":5.4592639057601,"predicted_per_token_ms":183.1748780169602,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1534,"sample_per_second":323.54568954462314,"sample_per_token_ms":3.0907535853976533,"sample_time":4741.216,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139107,"total_time":323771.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1537,"eval_per_second":5.455140644688096,"eval_per_token_ms":183.31333051398826,"eval_time":281752.589,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":281752.589,"predicted_n":1537,"predicted_per_second":5.455140644688096,"predicted_per_token_ms":183.31333051398826,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1538,"sample_per_second":323.41710144677637,"sample_per_token_ms":3.09198244473342,"sample_time":4755.469,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139108,"total_time":324780.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1542,"eval_per_second":5.4517208961577,"eval_per_token_ms":183.42831906614785,"eval_time":282846.468,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":282846.468,"predicted_n":1542,"predicted_per_second":5.4517208961577,"predicted_per_token_ms":183.42831906614785,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1543,"sample_per_second":323.35318298562913,"sample_per_token_ms":3.092593648736228,"sample_time":4771.872,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139109,"total_time":325781.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1546,"eval_per_second":5.449334921362114,"eval_per_token_ms":183.50863260025875,"eval_time":283704.346,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":283704.346,"predicted_n":1546,"predicted_per_second":5.449334921362114,"predicted_per_token_ms":183.50863260025875,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1547,"sample_per_second":323.23983685477396,"sample_per_token_ms":3.093678086619263,"sample_time":4785.92,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139110,"total_time":326787.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1551,"eval_per_second":5.4457833452292626,"eval_per_token_ms":183.62831141199226,"eval_time":284807.511,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":284807.511,"predicted_n":1551,"predicted_per_second":5.4457833452292626,"predicted_per_token_ms":183.62831141199226,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1552,"sample_per_second":323.1044004029645,"sample_per_token_ms":3.0949748711340206,"sample_time":4803.401,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139111,"total_time":327790.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1555,"eval_per_second":5.442173728079568,"eval_per_token_ms":183.75010610932475,"eval_time":285731.415,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":285731.415,"predicted_n":1555,"predicted_per_second":5.442173728079568,"predicted_per_token_ms":183.75010610932475,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1556,"sample_per_second":323.13163564623727,"sample_per_token_ms":3.094714010282776,"sample_time":4815.375,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139112,"total_time":328804.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1560,"eval_per_second":5.439107593623185,"eval_per_token_ms":183.85368974358974,"eval_time":286811.756,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":286811.756,"predicted_n":1560,"predicted_per_second":5.439107593623185,"predicted_per_token_ms":183.85368974358974,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1561,"sample_per_second":323.24823683721166,"sample_per_token_ms":3.0935976937860343,"sample_time":4829.106,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139113,"total_time":329812.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1564,"eval_per_second":5.436330467479963,"eval_per_token_ms":183.94761061381075,"eval_time":287694.063,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":287694.063,"predicted_n":1564,"predicted_per_second":5.436330467479963,"predicted_per_token_ms":183.94761061381075,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1565,"sample_per_second":323.17697212294024,"sample_per_token_ms":3.094279872204473,"sample_time":4842.548,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139114,"total_time":330820.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1569,"eval_per_second":5.432647606939851,"eval_per_token_ms":184.0723110261313,"eval_time":288809.456,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":288809.456,"predicted_n":1569,"predicted_per_second":5.432647606939851,"predicted_per_token_ms":184.0723110261313,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1570,"sample_per_second":323.26689821087217,"sample_per_token_ms":3.0934191082802545,"sample_time":4856.668,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139115,"total_time":331832.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1573,"eval_per_second":5.430008500466009,"eval_per_token_ms":184.1617743165925,"eval_time":289686.471,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":289686.471,"predicted_n":1573,"predicted_per_second":5.430008500466009,"predicted_per_token_ms":184.1617743165925,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1574,"sample_per_second":323.3466335486307,"sample_per_token_ms":3.092656289707751,"sample_time":4867.841,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139116,"total_time":332846.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1577,"eval_per_second":5.426682868211324,"eval_per_token_ms":184.27463411540901,"eval_time":290601.098,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":290601.098,"predicted_n":1577,"predicted_per_second":5.426682868211324,"predicted_per_token_ms":184.27463411540901,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1578,"sample_per_second":322.7467094868511,"sample_per_token_ms":3.0984049429657796,"sample_time":4889.283,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139117,"total_time":333854.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1582,"eval_per_second":5.422723509478064,"eval_per_token_ms":184.40918078381796,"eval_time":291735.324,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":291735.324,"predicted_n":1582,"predicted_per_second":5.422723509478064,"predicted_per_token_ms":184.40918078381796,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1583,"sample_per_second":322.69782722232395,"sample_per_token_ms":3.098874289324068,"sample_time":4905.518,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139118,"total_time":334864.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1586,"eval_per_second":5.420360694313891,"eval_per_token_ms":184.48956746532158,"eval_time":292600.454,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":292600.454,"predicted_n":1586,"predicted_per_second":5.420360694313891,"predicted_per_token_ms":184.48956746532158,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1587,"sample_per_second":322.7159054814527,"sample_per_token_ms":3.098700693131695,"sample_time":4917.638,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139119,"total_time":335875.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1591,"eval_per_second":5.416367211803544,"eval_per_token_ms":184.62559145191702,"eval_time":293739.316,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":293739.316,"predicted_n":1591,"predicted_per_second":5.416367211803544,"predicted_per_token_ms":184.62559145191702,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1592,"sample_per_second":322.5928685496877,"sample_per_token_ms":3.0998825376884422,"sample_time":4935.013,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139120,"total_time":336888.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1595,"eval_per_second":5.413790960755137,"eval_per_token_ms":184.7134489028213,"eval_time":294617.951,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":294617.951,"predicted_n":1595,"predicted_per_second":5.413790960755137,"predicted_per_token_ms":184.7134489028213,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1596,"sample_per_second":322.65296752789,"sample_per_token_ms":3.0993051378446115,"sample_time":4946.491,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139121,"total_time":337896.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1600,"eval_per_second":5.410752069310003,"eval_per_token_ms":184.81719125,"eval_time":295707.506,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":295707.506,"predicted_n":1600,"predicted_per_second":5.410752069310003,"predicted_per_token_ms":184.81719125,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1601,"sample_per_second":322.7218778422102,"sample_per_token_ms":3.098643347907558,"sample_time":4960.928,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139122,"total_time":338914.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1604,"eval_per_second":5.4079469436439975,"eval_per_token_ms":184.91305673316708,"eval_time":296600.543,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":296600.543,"predicted_n":1604,"predicted_per_second":5.4079469436439975,"predicted_per_token_ms":184.91305673316708,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1605,"sample_per_second":322.60962848033984,"sample_per_token_ms":3.0997214953271026,"sample_time":4975.053,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139123,"total_time":339928.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1609,"eval_per_second":5.404652750338603,"eval_per_token_ms":185.02576320696085,"eval_time":297706.453,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":297706.453,"predicted_n":1609,"predicted_per_second":5.404652750338603,"predicted_per_token_ms":185.02576320696085,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1610,"sample_per_second":322.62156270270305,"sample_per_token_ms":3.0996068322981367,"sample_time":4990.367,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139124,"total_time":340938.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1613,"eval_per_second":5.402234614755664,"eval_per_token_ms":185.1085840049597,"eval_time":298580.146,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":298580.146,"predicted_n":1613,"predicted_per_second":5.402234614755664,"predicted_per_token_ms":185.1085840049597,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1614,"sample_per_second":322.704156865411,"sample_per_token_ms":3.0988135068153655,"sample_time":5001.485,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139125,"total_time":341949.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1617,"eval_per_second":5.398978645460164,"eval_per_token_ms":185.22021768707484,"eval_time":299501.092,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":299501.092,"predicted_n":1617,"predicted_per_second":5.398978645460164,"predicted_per_token_ms":185.22021768707484,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1618,"sample_per_second":322.6532707728981,"sample_per_token_ms":3.099302224969098,"sample_time":5014.671,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139126,"total_time":342956.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1622,"eval_per_second":5.39546776184062,"eval_per_token_ms":185.34074229346487,"eval_time":300622.684,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":300622.684,"predicted_n":1622,"predicted_per_second":5.39546776184062,"predicted_per_token_ms":185.34074229346487,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1623,"sample_per_second":322.70629492014956,"sample_per_token_ms":3.0987929759704254,"sample_time":5029.341,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139127,"total_time":343967.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1626,"eval_per_second":5.393152545064342,"eval_per_token_ms":185.42030688806886,"eval_time":301493.419,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":301493.419,"predicted_n":1626,"predicted_per_second":5.393152545064342,"predicted_per_token_ms":185.42030688806886,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1627,"sample_per_second":322.7265333973364,"sample_per_token_ms":3.0985986478180703,"sample_time":5041.42,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139128,"total_time":344977.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1631,"eval_per_second":5.38958597431858,"eval_per_token_ms":185.54300919681177,"eval_time":302620.648,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":302620.648,"predicted_n":1631,"predicted_per_second":5.38958597431858,"predicted_per_token_ms":185.54300919681177,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1632,"sample_per_second":322.82382226174883,"sample_per_token_ms":3.097664828431373,"sample_time":5055.389,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139129,"total_time":345985.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1635,"eval_per_second":5.387309140569986,"eval_per_token_ms":185.62142507645262,"eval_time":303491.03,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":303491.03,"predicted_n":1635,"predicted_per_second":5.387309140569986,"predicted_per_token_ms":185.62142507645262,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1636,"sample_per_second":322.9231413315763,"sample_per_token_ms":3.096712102689487,"sample_time":5066.2210000000005,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139130,"total_time":346992.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1640,"eval_per_second":5.383960879825088,"eval_per_token_ms":185.73686219512197,"eval_time":304608.454,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":304608.454,"predicted_n":1640,"predicted_per_second":5.383960879825088,"predicted_per_token_ms":185.73686219512197,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1641,"sample_per_second":323.0219580130507,"sample_per_token_ms":3.09576477757465,"sample_time":5080.1500000000005,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139131,"total_time":348004.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1644,"eval_per_second":5.381232922686779,"eval_per_token_ms":185.83101946472019,"eval_time":305506.196,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":305506.196,"predicted_n":1644,"predicted_per_second":5.381232922686779,"predicted_per_token_ms":185.83101946472019,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1645,"sample_per_second":322.9507450051594,"sample_per_token_ms":3.096447416413374,"sample_time":5093.656,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139132,"total_time":349010.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1649,"eval_per_second":5.378176738400641,"eval_per_token_ms":185.93661916312917,"eval_time":306609.485,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":306609.485,"predicted_n":1649,"predicted_per_second":5.378176738400641,"predicted_per_token_ms":185.93661916312917,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1650,"sample_per_second":322.90474936133353,"sample_per_token_ms":3.096888484848485,"sample_time":5109.866,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139133,"total_time":350011.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1653,"eval_per_second":5.374806296842124,"eval_per_token_ms":186.05321657592256,"eval_time":307545.967,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":307545.967,"predicted_n":1653,"predicted_per_second":5.374806296842124,"predicted_per_token_ms":186.05321657592256,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1654,"sample_per_second":322.9802603213087,"sample_per_token_ms":3.096164449818622,"sample_time":5121.0560000000005,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139134,"total_time":351014.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1657,"eval_per_second":5.372219774949706,"eval_per_token_ms":186.1427942063971,"eval_time":308438.61,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":308438.61,"predicted_n":1657,"predicted_per_second":5.372219774949706,"predicted_per_token_ms":186.1427942063971,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1658,"sample_per_second":322.9494753531894,"sample_per_token_ms":3.0964595898673104,"sample_time":5133.93,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139135,"total_time":352024.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1662,"eval_per_second":5.369108593358126,"eval_per_token_ms":186.25065643802648,"eval_time":309548.591,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":309548.591,"predicted_n":1662,"predicted_per_second":5.369108593358126,"predicted_per_token_ms":186.25065643802648,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1663,"sample_per_second":323.0188726155749,"sample_per_token_ms":3.0957943475646426,"sample_time":5148.3060000000005,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139136,"total_time":353030.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1666,"eval_per_second":5.366606205909915,"eval_per_token_ms":186.3375030012005,"eval_time":310438.28,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":310438.28,"predicted_n":1666,"predicted_per_second":5.366606205909915,"predicted_per_token_ms":186.3375030012005,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1667,"sample_per_second":323.1036558006022,"sample_per_token_ms":3.0949820035992803,"sample_time":5159.335,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139137,"total_time":354044.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1670,"eval_per_second":5.362393851734806,"eval_per_token_ms":186.48387784431137,"eval_time":311428.076,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":311428.076,"predicted_n":1670,"predicted_per_second":5.362393851734806,"predicted_per_token_ms":186.48387784431137,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1671,"sample_per_second":323.0039717310636,"sample_per_token_ms":3.095937163375224,"sample_time":5173.311,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139138,"total_time":355047.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1674,"eval_per_second":5.359941749152056,"eval_per_token_ms":186.5691917562724,"eval_time":312316.827,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":312316.827,"predicted_n":1674,"predicted_per_second":5.359941749152056,"predicted_per_token_ms":186.5691917562724,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1675,"sample_per_second":323.0075065016108,"sample_per_token_ms":3.0959032835820897,"sample_time":5185.638,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139139,"total_time":356053.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1679,"eval_per_second":5.356677343969863,"eval_per_token_ms":186.68288862418106,"eval_time":313440.57,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":313440.57,"predicted_n":1679,"predicted_per_second":5.356677343969863,"predicted_per_token_ms":186.68288862418106,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1680,"sample_per_second":323.0294625943419,"sample_per_token_ms":3.0956928571428572,"sample_time":5200.764,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139140,"total_time":357070.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1683,"eval_per_second":5.353821582499252,"eval_per_token_ms":186.78246642899583,"eval_time":314354.891,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":314354.891,"predicted_n":1683,"predicted_per_second":5.353821582499252,"predicted_per_token_ms":186.78246642899583,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1684,"sample_per_second":323.0241815441937,"sample_per_token_ms":3.0957434679334916,"sample_time":5213.232,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139141,"total_time":358074.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1688,"eval_per_second":5.350974625979474,"eval_per_token_ms":186.88184300947867,"eval_time":315456.551,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":315456.551,"predicted_n":1688,"predicted_per_second":5.350974625979474,"predicted_per_token_ms":186.88184300947867,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1689,"sample_per_second":323.11222305418147,"sample_per_token_ms":3.094899940793369,"sample_time":5227.286,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139142,"total_time":359086.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1692,"eval_per_second":5.348555380954445,"eval_per_token_ms":186.9663729314421,"eval_time":316347.103,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":316347.103,"predicted_n":1692,"predicted_per_second":5.348555380954445,"predicted_per_token_ms":186.9663729314421,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1693,"sample_per_second":323.10985506051486,"sample_per_token_ms":3.0949226225634967,"sample_time":5239.704,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139143,"total_time":360093.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1696,"eval_per_second":5.345400591979162,"eval_per_token_ms":187.07671816037737,"eval_time":317282.114,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":317282.114,"predicted_n":1696,"predicted_per_second":5.345400591979162,"predicted_per_token_ms":187.07671816037737,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1697,"sample_per_second":323.01690556656916,"sample_per_token_ms":3.09581319976429,"sample_time":5253.595,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139144,"total_time":361106.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1701,"eval_per_second":5.342019350018887,"eval_per_token_ms":187.19512874779542,"eval_time":318418.914,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":318418.914,"predicted_n":1701,"predicted_per_second":5.342019350018887,"predicted_per_token_ms":187.19512874779542,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1702,"sample_per_second":322.84841072084345,"sample_per_token_ms":3.097428907168038,"sample_time":5271.8240000000005,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139145,"total_time":362115.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1705,"eval_per_second":5.3399499353228705,"eval_per_token_ms":187.267673313783,"eval_time":319291.38300000003,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":319291.38300000003,"predicted_n":1705,"predicted_per_second":5.3399499353228705,"predicted_per_token_ms":187.267673313783,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1706,"sample_per_second":322.7741167700641,"sample_per_token_ms":3.0981418522860493,"sample_time":5285.43,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139146,"total_time":363128.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1709,"eval_per_second":5.33697460943542,"eval_per_token_ms":187.37207372732593,"eval_time":320218.874,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":320218.874,"predicted_n":1709,"predicted_per_second":5.33697460943542,"predicted_per_token_ms":187.37207372732593,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1710,"sample_per_second":322.82369163427893,"sample_per_token_ms":3.097666081871345,"sample_time":5297.009,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139147,"total_time":364135.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1714,"eval_per_second":5.333603735217916,"eval_per_token_ms":187.4904941656943,"eval_time":321358.707,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":321358.707,"predicted_n":1714,"predicted_per_second":5.333603735217916,"predicted_per_token_ms":187.4904941656943,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1715,"sample_per_second":322.8642482910786,"sample_per_token_ms":3.097276967930029,"sample_time":5311.83,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139148,"total_time":365150.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1718,"eval_per_second":5.330972877012274,"eval_per_token_ms":187.58302153667054,"eval_time":322267.631,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":322267.631,"predicted_n":1718,"predicted_per_second":5.330972877012274,"predicted_per_token_ms":187.58302153667054,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1719,"sample_per_second":322.94595849759304,"sample_per_token_ms":3.0964933100639906,"sample_time":5322.872,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139149,"total_time":366155.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1722,"eval_per_second":5.327806786771294,"eval_per_token_ms":187.69449419279906,"eval_time":323209.919,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":323209.919,"predicted_n":1722,"predicted_per_second":5.327806786771294,"predicted_per_token_ms":187.69449419279906,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1723,"sample_per_second":322.7937338673387,"sample_per_token_ms":3.097953569355775,"sample_time":5337.774,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139150,"total_time":367171.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1727,"eval_per_second":5.324624701520408,"eval_per_token_ms":187.80666357845976,"eval_time":324342.108,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":324342.108,"predicted_n":1727,"predicted_per_second":5.324624701520408,"predicted_per_token_ms":187.80666357845976,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1728,"sample_per_second":322.8930620408747,"sample_per_token_ms":3.0970005787037036,"sample_time":5351.617,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139151,"total_time":368182.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1731,"eval_per_second":5.322024282819565,"eval_per_token_ms":187.89842865395724,"eval_time":325252.18,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":325252.18,"predicted_n":1731,"predicted_per_second":5.322024282819565,"predicted_per_token_ms":187.89842865395724,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1732,"sample_per_second":322.7960549550965,"sample_per_token_ms":3.0979312933025405,"sample_time":5365.617,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139152,"total_time":369184.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1735,"eval_per_second":5.319046429806065,"eval_per_token_ms":188.00362305475505,"eval_time":326186.286,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":326186.286,"predicted_n":1735,"predicted_per_second":5.319046429806065,"predicted_per_token_ms":188.00362305475505,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1736,"sample_per_second":322.6208708570583,"sample_per_token_ms":3.099613479262673,"sample_time":5380.929,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139153,"total_time":370189.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1740,"eval_per_second":5.316162105291119,"eval_per_token_ms":188.10562586206896,"eval_time":327303.789,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":327303.789,"predicted_n":1740,"predicted_per_second":5.316162105291119,"predicted_per_token_ms":188.10562586206896,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1741,"sample_per_second":322.6999289170461,"sample_per_token_ms":3.0988541068351525,"sample_time":5395.1050000000005,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139154,"total_time":371200.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1744,"eval_per_second":5.314068130009896,"eval_per_token_ms":188.179747706422,"eval_time":328185.48,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":328185.48,"predicted_n":1744,"predicted_per_second":5.314068130009896,"predicted_per_token_ms":188.179747706422,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1745,"sample_per_second":322.7762381936961,"sample_per_token_ms":3.0981214899713465,"sample_time":5406.222,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139155,"total_time":372215.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1748,"eval_per_second":5.310457728554453,"eval_per_token_ms":188.30768478260867,"eval_time":329161.833,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":329161.833,"predicted_n":1748,"predicted_per_second":5.310457728554453,"predicted_per_token_ms":188.30768478260867,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1749,"sample_per_second":322.79354280031976,"sample_per_token_ms":3.0979554030874787,"sample_time":5418.3240000000005,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139156,"total_time":373228.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1753,"eval_per_second":5.307685643564469,"eval_per_token_ms":188.4060336565887,"eval_time":330275.777,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":330275.777,"predicted_n":1753,"predicted_per_second":5.307685643564469,"predicted_per_token_ms":188.4060336565887,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1754,"sample_per_second":322.90019604128776,"sample_per_token_ms":3.0969321550741165,"sample_time":5432.019,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139157,"total_time":374235.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1757,"eval_per_second":5.305448836366656,"eval_per_token_ms":188.48546670461013,"eval_time":331168.965,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":331168.965,"predicted_n":1757,"predicted_per_second":5.305448836366656,"predicted_per_token_ms":188.48546670461013,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1758,"sample_per_second":322.9309047263822,"sample_per_token_ms":3.0966376564277587,"sample_time":5443.889,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139158,"total_time":375248.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1761,"eval_per_second":5.302368752394816,"eval_per_token_ms":188.59495570698468,"eval_time":332115.717,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":332115.717,"predicted_n":1761,"predicted_per_second":5.302368752394816,"predicted_per_token_ms":188.59495570698468,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1762,"sample_per_second":322.8220140429408,"sample_per_token_ms":3.0976821793416573,"sample_time":5458.116,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139159,"total_time":376263.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1766,"eval_per_second":5.299274533817403,"eval_per_token_ms":188.7050753114383,"eval_time":333253.163,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":333253.163,"predicted_n":1766,"predicted_per_second":5.299274533817403,"predicted_per_token_ms":188.7050753114383,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1767,"sample_per_second":322.9164306166443,"sample_per_token_ms":3.0967764572722127,"sample_time":5472.004,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139160,"total_time":377277.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1770,"eval_per_second":5.2962472506529705,"eval_per_token_ms":188.8129372881356,"eval_time":334198.89900000003,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":334198.89900000003,"predicted_n":1770,"predicted_per_second":5.2962472506529705,"predicted_per_token_ms":188.8129372881356,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1771,"sample_per_second":322.91849763217414,"sample_per_token_ms":3.096756634669678,"sample_time":5484.356,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139161,"total_time":378286.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1774,"eval_per_second":5.293442554976549,"eval_per_token_ms":188.9129785794814,"eval_time":335131.624,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":335131.624,"predicted_n":1774,"predicted_per_second":5.293442554976549,"predicted_per_token_ms":188.9129785794814,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1775,"sample_per_second":322.8856040862219,"sample_per_token_ms":3.0970721126760563,"sample_time":5497.303,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139162,"total_time":379293.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1778,"eval_per_second":5.290636866244973,"eval_per_token_ms":189.01316141732283,"eval_time":336065.401,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":336065.401,"predicted_n":1778,"predicted_per_second":5.290636866244973,"predicted_per_token_ms":189.01316141732283,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1779,"sample_per_second":322.8036559011012,"sample_per_token_ms":3.0978583473861723,"sample_time":5511.09,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139163,"total_time":380303.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1783,"eval_per_second":5.287399443589431,"eval_per_token_ms":189.12889231632082,"eval_time":337216.815,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":337216.815,"predicted_n":1783,"predicted_per_second":5.287399443589431,"predicted_per_token_ms":189.12889231632082,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1784,"sample_per_second":322.8984991011693,"sample_per_token_ms":3.0969484304932737,"sample_time":5524.956,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139164,"total_time":381306.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1787,"eval_per_second":5.284546182173992,"eval_per_token_ms":189.23100783435925,"eval_time":338155.811,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":338155.811,"predicted_n":1787,"predicted_per_second":5.284546182173992,"predicted_per_token_ms":189.23100783435925,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1788,"sample_per_second":322.91026670437566,"sample_per_token_ms":3.0968355704697985,"sample_time":5537.142,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139165,"total_time":382308.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1791,"eval_per_second":5.282284938441635,"eval_per_token_ms":189.3120139586823,"eval_time":339057.817,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":339057.817,"predicted_n":1791,"predicted_per_second":5.282284938441635,"predicted_per_token_ms":189.3120139586823,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1792,"sample_per_second":322.942874613284,"sample_per_token_ms":3.0965228794642856,"sample_time":5548.969,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139166,"total_time":383322.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1795,"eval_per_second":5.279584313943813,"eval_per_token_ms":189.4088512534819,"eval_time":339988.88800000004,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":339988.88800000004,"predicted_n":1795,"predicted_per_second":5.279584313943813,"predicted_per_token_ms":189.4088512534819,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1796,"sample_per_second":322.92220865125853,"sample_per_token_ms":3.0967210467706017,"sample_time":5561.711,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139167,"total_time":384336.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1799,"eval_per_second":5.27569123065454,"eval_per_token_ms":189.54862145636466,"eval_time":340997.97000000003,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":340997.97000000003,"predicted_n":1799,"predicted_per_second":5.27569123065454,"predicted_per_token_ms":189.54862145636466,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1800,"sample_per_second":322.9377286354266,"sample_per_token_ms":3.096572222222222,"sample_time":5573.83,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139168,"total_time":385343.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1804,"eval_per_second":5.272639015275858,"eval_per_token_ms":189.6583470066519,"eval_time":342143.658,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":342143.658,"predicted_n":1804,"predicted_per_second":5.272639015275858,"predicted_per_token_ms":189.6583470066519,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1805,"sample_per_second":323.0225605756781,"sample_per_token_ms":3.0957590027700834,"sample_time":5587.845,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139169,"total_time":386353.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1808,"eval_per_second":5.270296083659887,"eval_per_token_ms":189.74266039823007,"eval_time":343054.73,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":343054.73,"predicted_n":1808,"predicted_per_second":5.270296083659887,"predicted_per_token_ms":189.74266039823007,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1809,"sample_per_second":323.05446298222665,"sample_per_token_ms":3.0954532891100057,"sample_time":5599.675,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139170,"total_time":387367.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849},
{"ctx_size":0.09000000357627869,"cuda_str":"CUDA","eval_count":1812,"eval_per_second":5.267299015603149,"eval_per_token_ms":189.8506230684327,"eval_time":344009.329,"load_time":8104.553,"mem_required":70.44000244140625,"mem_required_per_state":2048.0,"meta":{"arch":"llama","f_norm_eps":"1.0e-05","f_norm_rms_eps":"1.0e-05","format":"GGUF V1 (support until nov 2023)","freq_base":"1000000.0","freq_scale":"1","n_ctx":"4096","n_ctx_train":"16384","n_embd":"4096","n_ff":"11008","n_gqa":"1","n_head":"32","n_head_kv":"32","n_layer":"32","n_merges":"0","n_rot":"128","n_vocab":"32016"},"offloaded":35,"offloaded_total":35,"offloading_repeating":32,"predicted_ms":344009.329,"predicted_n":1812,"predicted_per_second":5.267299015603149,"predicted_per_token_ms":189.8506230684327,"prompt_eval_count":895,"prompt_eval_per_second":25.538463522713453,"prompt_eval_per_token_ms":39.156623463687154,"prompt_eval_time":35045.178,"prompt_ms":35045.178,"prompt_n":895,"prompt_per_second":25.538463522713453,"prompt_per_token_ms":39.156623463687154,"sample_count":1813,"sample_per_second":322.97480279921,"sample_per_token_ms":3.096216767788196,"sample_time":5613.441,"tensor_type":{"f16":1,"f32":65,"q2_K":64,"q3_K":160,"q4_0":1},"time":1694139171,"total_time":388377.70799999987,"vram_per_layer_avg":138.54286193847656,"vram_used":4849}
]