diff --git a/benchmarks/misc/membound.py b/benchmarks/misc/membound.py index 6f1952857c4a5..f70623d04a953 100644 --- a/benchmarks/misc/membound.py +++ b/benchmarks/misc/membound.py @@ -1,38 +1,91 @@ +import os import time from membound_cases import memory_bound_cases_list -from utils import (arch_name, dtype2str, geometric_mean, kibibyte, - md_table_header, size2str) +from utils import (arch_name, datatime_with_format, dtype2str, dump2json, + geometric_mean, md_table_header, scaled_repeat_times, + size2str) import taichi as ti class MemoryBound: suite_name = 'memorybound' - supported_archs = [ti.cpu, ti.cuda] + supported_archs = [ti.x64, ti.cuda] test_cases = memory_bound_cases_list test_dtype_list = [ti.i32, ti.i64, ti.f32, ti.f64] - test_dsize_list = [(4**i) * kibibyte - for i in range(1, 10)] #[4KB,16KB...256MB] + test_dsize_list = [ + (4**i) * 1024 # kibibytes(KiB) = 1024 + for i in range(1, 10) # [4KB,16KB...256MB] + ] basic_repeat_times = 10 evaluator = [geometric_mean] def __init__(self, arch): - self.arch = arch - self.cases_impl = [] + self._arch = arch + self._cases_impl = [] for case in self.test_cases: for dtype in self.test_dtype_list: impl = CaseImpl(case, arch, dtype, self.test_dsize_list, self.evaluator) - self.cases_impl.append(impl) + self._cases_impl.append(impl) def run(self): - for case in self.cases_impl: + for case in self._cases_impl: case.run() - def get_markdown_lines(self): + def save_as_json(self, arch_dir='./'): + #folder of suite + suite_path = os.path.join(arch_dir, self.suite_name) + os.makedirs(suite_path) + #json files + self._save_suite_info_as_json(suite_path) + self._save_cases_info_as_json(suite_path) + + def save_as_markdown(self, arch_dir='./'): + current_time = datatime_with_format() + commit_hash = ti.core.get_commit_hash() #[:8] + file_name = f'{self.suite_name}.md' + file_path = os.path.join(arch_dir, file_name) + with open(file_path, 'w') as f: + lines = [ + f'commit_hash: {commit_hash}\n', f'datatime: {current_time}\n' + ] + lines += self._get_markdown_lines() + for line in lines: + print(line, file=f) + + def _save_suite_info_as_json(self, suite_path='./'): + info_dict = { + 'cases': [func.__name__ for func in self.test_cases], + 'dtype': [dtype2str(dtype) for dtype in self.test_dtype_list], + 'dsize': [size for size in self.test_dsize_list], + 'repeat': [ + scaled_repeat_times(self._arch, size, self.basic_repeat_times) + for size in self.test_dsize_list + ], + 'evaluator': [func.__name__ for func in self.evaluator] + } + info_path = os.path.join(suite_path, '_info.json') + with open(info_path, 'w') as f: + print(dump2json(info_dict), file=f) + + def _save_cases_info_as_json(self, suite_path='./'): + for case in self.test_cases: #for case [fill,saxpy,reduction] + results_dict = {} + for impl in self._cases_impl: #find [ti.i32, ti.i64, ti.f32, ti.f64] + if impl._name != case.__name__: + continue + result_name = dtype2str(impl._test_dtype) + results_dict[result_name] = impl.get_results_dict() + case_path = os.path.join(suite_path, (case.__name__ + '.json')) + with open(case_path, 'w') as f: + case_str = dump2json(results_dict) + print(case_str, file=f) + + def _get_markdown_lines(self): lines = [] - lines += md_table_header(self.suite_name, self.arch, + lines += md_table_header(self.suite_name, self._arch, self.test_dsize_list, self.basic_repeat_times, self.evaluator) @@ -40,7 +93,7 @@ def get_markdown_lines(self): '|' for i in range( len(self.test_dsize_list) + len(MemoryBound.evaluator))) lines += [result_header] - for case in self.cases_impl: + for case in self._cases_impl: lines += case.get_markdown_lines() lines.append('') return lines @@ -48,31 +101,47 @@ def get_markdown_lines(self): class CaseImpl: def __init__(self, func, arch, test_dtype, test_dsize_list, evaluator): - self.func = func - self.name = func.__name__ - self.arch = arch - self.test_dtype = test_dtype - self.test_dsize_list = test_dsize_list - self.min_time_in_us = [] #test results - self.evaluator = evaluator + self._func = func + self._name = func.__name__ + self._arch = arch + self._test_dtype = test_dtype + self._test_dsize_list = test_dsize_list + self._min_time_in_us = [] #test results + self._evaluator = evaluator def run(self): - ti.init(kernel_profiler=True, arch=self.arch) - print("TestCase[%s.%s.%s]" % (self.func.__name__, arch_name( - self.arch), dtype2str[self.test_dtype])) - for test_dsize in self.test_dsize_list: + ti.init(kernel_profiler=True, arch=self._arch) + print("TestCase[%s.%s.%s]" % (self._func.__name__, arch_name( + self._arch), dtype2str(self._test_dtype))) + for test_dsize in self._test_dsize_list: print("test_dsize = %s" % (size2str(test_dsize))) - self.min_time_in_us.append( - self.func(self.arch, self.test_dtype, test_dsize, - MemoryBound.basic_repeat_times)) + self._min_time_in_us.append( + self._func(self._arch, self._test_dtype, test_dsize, + MemoryBound.basic_repeat_times)) time.sleep(0.2) ti.reset() def get_markdown_lines(self): - string = '|' + self.name + '.' + dtype2str[self.test_dtype] + '|' + string = '|' + self._name + '.' + dtype2str(self._test_dtype) + '|' string += ''.join( - str(round(time, 4)) + '|' for time in self.min_time_in_us) + str(round(time, 4)) + '|' for time in self._min_time_in_us) string += ''.join( - str(round(item(self.min_time_in_us), 4)) + '|' - for item in self.evaluator) + str(round(item(self._min_time_in_us), 4)) + '|' + for item in self._evaluator) return [string] + + def get_results_dict(self): + results_dict = {} + for i in range(len(self._test_dsize_list)): + dsize = self._test_dsize_list[i] + repeat = scaled_repeat_times(self._arch, dsize, + MemoryBound.basic_repeat_times) + elapsed_time = self._min_time_in_us[i] + item_name = size2str(dsize).replace('.0', '') + item_dict = { + 'dsize_byte': dsize, + 'repeat': repeat, + 'elapsed_time_ms': elapsed_time + } + results_dict[item_name] = item_dict + return results_dict diff --git a/benchmarks/misc/membound_cases.py b/benchmarks/misc/membound_cases.py index 5792a85a2ce63..6a511624a6eca 100644 --- a/benchmarks/misc/membound_cases.py +++ b/benchmarks/misc/membound_cases.py @@ -26,7 +26,7 @@ def membound_benchmark(func, num_elements, repeat): def fill(arch, dtype, dsize, repeat=10): repeat = scaled_repeat_times(arch, dsize, repeat) - num_elements = dsize // dtype_size[dtype] + num_elements = dsize // dtype_size(dtype) x = ti.field(dtype, shape=num_elements) @@ -41,7 +41,7 @@ def fill_const(n: ti.i32): def saxpy(arch, dtype, dsize, repeat=10): repeat = scaled_repeat_times(arch, dsize, repeat) - num_elements = dsize // dtype_size[dtype] // 3 #z=x+y + num_elements = dsize // dtype_size(dtype) // 3 #z=x+y x = ti.field(dtype, shape=num_elements) y = ti.field(dtype, shape=num_elements) @@ -61,7 +61,7 @@ def saxpy(n: ti.i32): def reduction(arch, dtype, dsize, repeat=10): repeat = scaled_repeat_times(arch, dsize, repeat) - num_elements = dsize // dtype_size[dtype] + num_elements = dsize // dtype_size(dtype) x = ti.field(dtype, shape=num_elements) y = ti.field(dtype, shape=()) diff --git a/benchmarks/misc/run.py b/benchmarks/misc/run.py index 4922e043d14bd..2d1890993c402 100644 --- a/benchmarks/misc/run.py +++ b/benchmarks/misc/run.py @@ -2,32 +2,54 @@ import warnings from membound import MemoryBound -from taichi.core import ti_core as _ti_core -from utils import arch_name, datatime_with_format +from utils import arch_name, datatime_with_format, dump2json import taichi as ti benchmark_suites = [MemoryBound] -benchmark_archs = [ti.cpu, ti.cuda] +benchmark_archs = [ti.x64, ti.cuda] -class CommitInfo: - def __init__(self, pull_request_id, commit_hash): +class BenchmarksInfo: + def __init__(self, pull_request_id: str, commit_hash: str): + """init with commit info""" self.pull_request_id = pull_request_id - self.commit_hash = commit_hash #str - self.archs = [] #['x64','cuda','vulkan', ...] + self.commit_hash = commit_hash self.datetime = [] #[start, end] + self.archs = {} + # "archs": { + # "x64": ["memorybound"], #arch:[suites name] + # "cuda": ["memorybound"] + # } + def add_suites_info(self, arch, suites): + self.archs[arch_name(arch)] = suites.get_suites_name() class BenchmarkSuites: def __init__(self, arch): - self.suites = [] - self.arch = arch + self._suites = [] + self._arch = arch for suite in benchmark_suites: - if self.check_supported(arch, suite): - self.suites.append(suite(arch)) + if self._check_supported(arch, suite): + self._suites.append(suite(arch)) - def check_supported(self, arch, suite): + def run(self): + print(f'Arch [{arch_name(self._arch)}] Running...') + for suite in self._suites: + suite.run() + + def save(self, benchmark_dir='./'): + #folder of archs + arch_dir = os.path.join(benchmark_dir, arch_name(self._arch)) + os.makedirs(arch_dir) + for suite in self._suites: + suite.save_as_json(arch_dir) + suite.save_as_markdown(arch_dir) + + def get_suites_name(self): + return [suite.suite_name for suite in self._suites] + + def _check_supported(self, arch, suite): if arch in suite.supported_archs: return True else: @@ -37,41 +59,35 @@ def check_supported(self, arch, suite): stacklevel=2) return False - def run(self): - print(f'Arch [{arch_name(self.arch)}] Running...') - for suite in self.suites: - suite.run() - - def save_to_markdown(self, arch_dir='./'): - current_time = datatime_with_format() - commit_hash = _ti_core.get_commit_hash() #[:8] - for suite in self.suites: - file_name = f'{suite.suite_name}.md' - path = os.path.join(arch_dir, file_name) - with open(path, 'w') as f: - lines = [ - f'commit_hash: {commit_hash}\n', - f'datatime: {current_time}\n' - ] - lines += suite.get_markdown_lines() - for line in lines: - print(line, file=f) - def main(): benchmark_dir = os.path.join(os.getcwd(), 'results') os.makedirs(benchmark_dir) + pull_request_id = os.environ.get('PULL_REQUEST_NUMBER') + commit_hash = ti.core.get_commit_hash() #[:8] + info = BenchmarksInfo(pull_request_id, commit_hash) + info.datetime.append(datatime_with_format()) #start time + + print(f'pull_request_id = {pull_request_id}') + print(f'commit_hash = {commit_hash}') + for arch in benchmark_archs: - #make dir - arch_dir = os.path.join(benchmark_dir, arch_name(arch)) - os.makedirs(arch_dir) #init & run suites = BenchmarkSuites(arch) suites.run() #save result - suites.save_to_markdown(arch_dir) + suites.save(benchmark_dir) + #add benchmark info + info.add_suites_info(arch, suites) + + info.datetime.append(datatime_with_format()) #end time + #save commit and benchmark info + info_path = os.path.join(benchmark_dir, '_info.json') + info_str = dump2json(info) + with open(info_path, 'w') as f: + print(info_str, file=f) if __name__ == '__main__': diff --git a/benchmarks/misc/utils.py b/benchmarks/misc/utils.py index 3548151ec6fce..62250a216a534 100644 --- a/benchmarks/misc/utils.py +++ b/benchmarks/misc/utils.py @@ -1,18 +1,30 @@ import datetime +import json + +import jsbeautifier import taichi as ti -kibibyte = 1024 -bls2str = {False: "BLS_off", True: "BLS_on"} -dense2str = {False: "Struct_for", True: "Range_for"} +def dtype2str(ti_dtype): + type_str_dict = { + ti.i32: "i32", + ti.i64: "i64", + ti.f32: "f32", + ti.f64: "f64" + } + if ti_dtype not in type_str_dict: + raise RuntimeError('Unsupported ti.dtype: ' + str(type(ti_dtype))) + else: + return type_str_dict[ti_dtype] -dtype2str = {ti.i32: "i32", ti.i64: "i64", ti.f32: "f32", ti.f64: "f64"} -dtype_size = {ti.i32: 4, ti.i64: 8, ti.f32: 4, ti.f64: 8} -# for output string -size_subsection = [(0.0, 'B'), (1024.0, 'KB'), (1048576.0, 'MB'), - (1073741824.0, 'GB'), (float('inf'), 'INF')] #B KB MB GB +def dtype_size(ti_dtype): + dtype_size_dict = {ti.i32: 4, ti.i64: 8, ti.f32: 4, ti.f64: 8} + if ti_dtype not in dtype_size_dict: + raise RuntimeError('Unsupported ti.dtype: ' + str(type(ti_dtype))) + else: + return dtype_size_dict[ti_dtype] def arch_name(arch): @@ -23,7 +35,21 @@ def datatime_with_format(): return datetime.datetime.now().isoformat() +def dump2json(obj): + if type(obj) is dict: + obj2dict = obj + else: + obj2dict = obj.__dict__ + options = jsbeautifier.default_options() + options.indent_size = 4 + return jsbeautifier.beautify(json.dumps(obj2dict), options) + + def size2str(size_in_byte): + # for output string + size_subsection = [(0.0, 'B'), (1024.0, 'KB'), (1048576.0, 'MB'), + (1073741824.0, 'GB'), + (float('inf'), 'INF')] #B KB MB GB for dsize, units in reversed(size_subsection): if size_in_byte >= dsize: return str(round(size_in_byte / dsize, 4)) + units