From 7f4f824f9bc97bfb92e2e68b351830c1df1d621e Mon Sep 17 00:00:00 2001 From: iori-yja Date: Mon, 12 Nov 2018 21:51:00 +0900 Subject: [PATCH 1/4] enhancement: add 'heatmap-cachefile' --- app/util/heatmap.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/app/util/heatmap.py b/app/util/heatmap.py index ca740b2..1e60558 100644 --- a/app/util/heatmap.py +++ b/app/util/heatmap.py @@ -21,7 +21,8 @@ import os import gzip import collections -from os.path import abspath, join +import json +from os.path import abspath, join, isfile from math import ceil, floor from flask import abort @@ -29,6 +30,7 @@ from .regexp import event_regexp, idle_regexp # global defaults +USE_HEATMAPCACHE = True YRATIO = 1000 # milliseconds DEFAULT_ROWS = 50 heatmap_cache = {} @@ -59,6 +61,9 @@ def read_offsets(filename): # use cached heatmap return heatmap_cache[path] + if avail_heatmap_filecache(path, mtime): + return read_heatmap_filecache(path) + # read .gz files via a "gunzip -c" pipe if filename.endswith(".gz"): try: @@ -109,8 +114,31 @@ def read_offsets(filename): heatmap = collections.namedtuple('offsets', ['start', 'end', 'offsets'])(start, end, offsets) heatmap_cache[path] = heatmap heatmap_mtimes[path] = mtime + if USE_HEATMAPCACHE: + write_heatmap_filecache(path, heatmap, mtime) return heatmap +def convert_path_to_filecache(path): + return path + '-heatmap-cache' + + +def write_heatmap_filecache(path, heatmap, mtime): + f = open(convert_path_to_filecache(path), 'wt') + f.write(json.dumps({'start': heatmap.start, 'end': heatmap.end, 'offsets': heatmap.offsets})) + f.close() + + +def read_heatmap_filecache(path): + f = open(convert_path_to_filecache(path), 'rt') + h = json.loads(f.read()) + print('read', h) + f.close() + return collections.namedtuple('offsets', ['start', 'end', 'offsets'])(h['start'], h['end'], h['offsets']) + +def avail_heatmap_filecache(path, mtime): + filename = convert_path_to_filecache(path) + return isfile(filename) + # return a heatmap from the cached offsets def generate_heatmap(filename, rows=None): o = read_offsets(filename) From 1ce15a6fb1aac00ceb9b530c73f9735a4aafd479 Mon Sep 17 00:00:00 2001 From: iori-yja Date: Tue, 13 Nov 2018 16:57:27 +0900 Subject: [PATCH 2/4] add mtime check in file-backed heatmap cache --- app/util/heatmap.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/app/util/heatmap.py b/app/util/heatmap.py index 1e60558..1bdf9ce 100644 --- a/app/util/heatmap.py +++ b/app/util/heatmap.py @@ -30,7 +30,7 @@ from .regexp import event_regexp, idle_regexp # global defaults -USE_HEATMAPCACHE = True +USE_HEATMAP_FILECACHE = True YRATIO = 1000 # milliseconds DEFAULT_ROWS = 50 heatmap_cache = {} @@ -61,8 +61,10 @@ def read_offsets(filename): # use cached heatmap return heatmap_cache[path] - if avail_heatmap_filecache(path, mtime): - return read_heatmap_filecache(path) + if USE_HEATMAP_FILECACHE and avail_heatmap_filecache(path, mtime): + fc = read_heatmap_filecache(path) + if fc.mtime == mtime: + return fc.heatmap # read .gz files via a "gunzip -c" pipe if filename.endswith(".gz"): @@ -114,7 +116,7 @@ def read_offsets(filename): heatmap = collections.namedtuple('offsets', ['start', 'end', 'offsets'])(start, end, offsets) heatmap_cache[path] = heatmap heatmap_mtimes[path] = mtime - if USE_HEATMAPCACHE: + if USE_HEATMAP_FILECACHE: write_heatmap_filecache(path, heatmap, mtime) return heatmap @@ -124,7 +126,7 @@ def convert_path_to_filecache(path): def write_heatmap_filecache(path, heatmap, mtime): f = open(convert_path_to_filecache(path), 'wt') - f.write(json.dumps({'start': heatmap.start, 'end': heatmap.end, 'offsets': heatmap.offsets})) + f.write(json.dumps({'start': heatmap.start, 'end': heatmap.end, 'offsets': heatmap.offsets, 'mtime': mtime})) f.close() @@ -133,9 +135,12 @@ def read_heatmap_filecache(path): h = json.loads(f.read()) print('read', h) f.close() - return collections.namedtuple('offsets', ['start', 'end', 'offsets'])(h['start'], h['end'], h['offsets']) + return collections.namedtuple('cache', ['heatmap', 'mtime'])( + collections.namedtuple('offsets', ['start', 'end', 'offsets'])(h['start'], h['end'], h['offsets']), + h['mtime'] + ) -def avail_heatmap_filecache(path, mtime): +def avail_heatmap_filecache(path): filename = convert_path_to_filecache(path) return isfile(filename) From c4ed2794f29e1716b65b3f5ebb0e402d079b1e32 Mon Sep 17 00:00:00 2001 From: iori-yja Date: Tue, 13 Nov 2018 17:02:52 +0900 Subject: [PATCH 3/4] hide cachefile (eg. examples/.1e8f770c-heatmap-cache) --- app/util/heatmap.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/util/heatmap.py b/app/util/heatmap.py index 1bdf9ce..9cde197 100644 --- a/app/util/heatmap.py +++ b/app/util/heatmap.py @@ -22,7 +22,7 @@ import gzip import collections import json -from os.path import abspath, join, isfile +from os.path import abspath, join, isfile, dirname, basename from math import ceil, floor from flask import abort @@ -61,7 +61,7 @@ def read_offsets(filename): # use cached heatmap return heatmap_cache[path] - if USE_HEATMAP_FILECACHE and avail_heatmap_filecache(path, mtime): + if USE_HEATMAP_FILECACHE and avail_heatmap_filecache(path): fc = read_heatmap_filecache(path) if fc.mtime == mtime: return fc.heatmap @@ -121,7 +121,7 @@ def read_offsets(filename): return heatmap def convert_path_to_filecache(path): - return path + '-heatmap-cache' + return dirname(path) + '/.' + basename(path) + '-heatmap-cache' def write_heatmap_filecache(path, heatmap, mtime): From f78eeaf038e2f822a5bb14769d7f0d234398f0b0 Mon Sep 17 00:00:00 2001 From: iori-yja Date: Tue, 13 Nov 2018 19:33:32 +0900 Subject: [PATCH 4/4] delete debug logic for heatmap filecache --- app/config.py | 3 ++- app/util/heatmap.py | 6 ++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/app/config.py b/app/config.py index a61e8fc..5e436c2 100644 --- a/app/config.py +++ b/app/config.py @@ -5,4 +5,5 @@ STACK_DIR = 'examples' HOST = '0.0.0.0' PORT = 5000 -JSONIFY_PRETTYPRINT_REGULAR = False \ No newline at end of file +USE_HEATMAP_FILECACHE = True +JSONIFY_PRETTYPRINT_REGULAR = False diff --git a/app/util/heatmap.py b/app/util/heatmap.py index 9cde197..7b526a1 100644 --- a/app/util/heatmap.py +++ b/app/util/heatmap.py @@ -30,7 +30,6 @@ from .regexp import event_regexp, idle_regexp # global defaults -USE_HEATMAP_FILECACHE = True YRATIO = 1000 # milliseconds DEFAULT_ROWS = 50 heatmap_cache = {} @@ -61,7 +60,7 @@ def read_offsets(filename): # use cached heatmap return heatmap_cache[path] - if USE_HEATMAP_FILECACHE and avail_heatmap_filecache(path): + if config.USE_HEATMAP_FILECACHE and avail_heatmap_filecache(path): fc = read_heatmap_filecache(path) if fc.mtime == mtime: return fc.heatmap @@ -116,7 +115,7 @@ def read_offsets(filename): heatmap = collections.namedtuple('offsets', ['start', 'end', 'offsets'])(start, end, offsets) heatmap_cache[path] = heatmap heatmap_mtimes[path] = mtime - if USE_HEATMAP_FILECACHE: + if config.USE_HEATMAP_FILECACHE: write_heatmap_filecache(path, heatmap, mtime) return heatmap @@ -133,7 +132,6 @@ def write_heatmap_filecache(path, heatmap, mtime): def read_heatmap_filecache(path): f = open(convert_path_to_filecache(path), 'rt') h = json.loads(f.read()) - print('read', h) f.close() return collections.namedtuple('cache', ['heatmap', 'mtime'])( collections.namedtuple('offsets', ['start', 'end', 'offsets'])(h['start'], h['end'], h['offsets']),