From 0528e0903f994a919b05956e72d24794301f9248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Harter?= Date: Fri, 16 Mar 2018 17:11:18 +0100 Subject: [PATCH 1/3] dist/tools: add lazysponge tool Write stdin to if it is different from the previous content. If data provided in stdin is the same as the data that was in , it is not modified so `last modification date` is not changed. --- dist/tools/lazysponge/lazysponge.py | 102 ++++++++++++++++ dist/tools/lazysponge/lazysponge_test.py | 149 +++++++++++++++++++++++ 2 files changed, 251 insertions(+) create mode 100755 dist/tools/lazysponge/lazysponge.py create mode 100755 dist/tools/lazysponge/lazysponge_test.py diff --git a/dist/tools/lazysponge/lazysponge.py b/dist/tools/lazysponge/lazysponge.py new file mode 100755 index 000000000000..6bfefef79511 --- /dev/null +++ b/dist/tools/lazysponge/lazysponge.py @@ -0,0 +1,102 @@ +#! /usr/bin/env python3 + +# +# Copyright (C) 2018 Gaëtan Harter +# +# This file is subject to the terms and conditions of the GNU Lesser +# General Public License v2.1. See the file LICENSE in the top level +# directory for more details. +# + +""" +lazysponge + +Adaptation of moreutils `sponge` with added functionnality that it does not +modify the output file if the content would be unchanged. + +Description +----------- + +Reads standard input and writes it to the specified file if its content was +different. + +The file is not changed if the content is the same so modification timestamp is +unchanged. + +Note +---- + +It only works with input provided by a `pipe` and not interractive input. +The reason is that `ctrl+c` would not be handled properly in that case. + +Usage +----- + + usage: lazysponge.py [-h] outfile + + Soak up all input from stdin and write it to if it differs from + previous content. If the content is the same, file is not modified. + + positional arguments: + outfile Output file + + optional arguments: + -h, --help show this help message and exit +""" + +import os +import sys +import argparse +import hashlib + +DESCRIPTION = ('Soak up all input from stdin and write it to ' + ' if it differs from previous content.\n' + ' If the content is the same, file is not modified.') +PARSER = argparse.ArgumentParser(description=DESCRIPTION) +PARSER.add_argument('outfile', help='Output file') +PARSER.add_argument('--verbose', '-v', help='Verbose output', default=False, + action='store_true') + + +def _print_hash_debug_info(outfilename, oldbytes, newbytes): + """Print debug information on hashs.""" + oldhash = hashlib.md5(oldbytes).hexdigest() if oldbytes is not None else '' + newhash = hashlib.md5(newbytes).hexdigest() + if oldbytes == newbytes: + msg = 'Keeping old {} ({})'.format(outfilename, oldhash) + else: + msg = 'Replacing {} ({} != {})'.format(outfilename, oldhash, newhash) + print(msg, file=sys.stderr) + + +def main(): + """Write stdin to given if it would change its content.""" + opts = PARSER.parse_args() + + # No support for 'interactive' input as catching Ctrl+c breaks in 'read' + if os.isatty(sys.stdin.fileno()): + print('Interactive input not supported. Use piped input', + file=sys.stderr) + print(' echo message | {}'.format(' '.join(sys.argv)), + file=sys.stderr) + exit(1) + + try: + with open(opts.outfile, 'rb') as outfd: + oldbytes = outfd.read() + except FileNotFoundError: + oldbytes = None + + stdinbytes = sys.stdin.buffer.read() + if opts.verbose: + _print_hash_debug_info(opts.outfile, oldbytes, stdinbytes) + + if oldbytes == stdinbytes: + return + + with open(opts.outfile, 'wb') as outfd: + outfd.write(stdinbytes) + + +if __name__ == '__main__': + main() diff --git a/dist/tools/lazysponge/lazysponge_test.py b/dist/tools/lazysponge/lazysponge_test.py new file mode 100755 index 000000000000..6646161eacce --- /dev/null +++ b/dist/tools/lazysponge/lazysponge_test.py @@ -0,0 +1,149 @@ +#! /usr/bin/env python3 + +# +# Copyright (C) 2018 Gaëtan Harter +# +# This file is subject to the terms and conditions of the GNU Lesser +# General Public License v2.1. See the file LICENSE in the top level +# directory for more details. +# + +"""Test script for lazysponge.""" + +import os +import sys +import shutil +import tempfile +from io import StringIO, BytesIO + +import unittest +from unittest import mock + +import lazysponge + + +class TestLazysponge(unittest.TestCase): + """Test the lazysponge script. + + Tested using mocks for stdin. + """ + + def setUp(self): + self.isatty_ret = False + self.isatty = mock.patch.object( + os, 'isatty', lambda _: self.isatty_ret).start() + + self.tmpdir = tempfile.mkdtemp() + self.outfile = os.path.join(self.tmpdir, 'outfile') + + self.argv = ['lazysponge', self.outfile] + mock.patch.object(sys, 'argv', self.argv).start() + + self.stdin = mock.Mock() + self.stdin.fileno.return_value = 0 + mock.patch.object(sys, 'stdin', self.stdin).start() + self.stdin.buffer = BytesIO() + + def tearDown(self): + shutil.rmtree(self.tmpdir, ignore_errors=True) + mock.patch.stopall() + + def test_write_one_file(self): + """Test a simple case where we write one file without quiet output.""" + first_input = b'First input\n' + + # Write input once + self.stdin.buffer.write(first_input) + self.stdin.buffer.seek(0) + stderr = StringIO() + with mock.patch('sys.stderr', stderr): + lazysponge.main() + self.assertEqual(stderr.getvalue(), '') + # no errors + os.stat(self.outfile) + with open(self.outfile, 'rb') as outfd: + self.assertEqual(outfd.read(), first_input) + + def test_write_two_times_and_update(self): + """Test writing two times the same output plus a new one.""" + first_input = b'First input\n' + updated_input = b'Second input\n' + stderr = StringIO() + + self.argv.append('--verbose') + + # File does not exist + with self.assertRaises(OSError): + os.stat(self.outfile) + + # Write input once + self.stdin.buffer.write(first_input) + self.stdin.buffer.seek(0) + with mock.patch('sys.stderr', stderr): + lazysponge.main() + first_stat = os.stat(self.outfile) + with open(self.outfile, 'rb') as outfd: + self.assertEqual(outfd.read(), first_input) + self._truncate(self.stdin.buffer) + + # compare stderr verbose output + errmsg = 'Replacing %s ( != 96022020c795ee69653958a3cb4bb083)\n' + self.assertEqual(stderr.getvalue(), errmsg % self.outfile) + self._truncate(stderr) + + # Re-Write the same input + self.stdin.buffer.write(first_input) + self.stdin.buffer.seek(0) + with mock.patch('sys.stderr', stderr): + lazysponge.main() + second_stat = os.stat(self.outfile) + with open(self.outfile, 'rb') as outfd: + self.assertEqual(outfd.read(), first_input) + self._truncate(self.stdin.buffer) + + # File has not been modified + self.assertEqual(first_stat, second_stat) + # compare stderr verbose output + errmsg = 'Keeping old %s (96022020c795ee69653958a3cb4bb083)\n' + self.assertEqual(stderr.getvalue(), errmsg % self.outfile) + self._truncate(stderr) + + # Update with a new input + self.stdin.buffer.write(updated_input) + self.stdin.buffer.seek(0) + with mock.patch('sys.stderr', stderr): + lazysponge.main() + third_stat = os.stat(self.outfile) + with open(self.outfile, 'rb') as outfd: + self.assertEqual(outfd.read(), updated_input) + self._truncate(self.stdin.buffer) + + # File is newer + self.assertGreater(third_stat, second_stat) + # compare stderr verbose output + errmsg = ('Replacing %s (96022020c795ee69653958a3cb4bb083' + ' != 1015f2c7f2fc3d575b7aeb1e92c0f6bf)\n') + self.assertEqual(stderr.getvalue(), errmsg % self.outfile) + self._truncate(stderr) + + @staticmethod + def _truncate(filefd): + filefd.seek(0) + filefd.truncate(0) + + def test_no_tty_detection(self): + """Test detecting that 'stdin' is not a tty.""" + self.isatty_ret = True + stderr = StringIO() + + with mock.patch('sys.stderr', stderr): + with self.assertRaises(SystemExit): + lazysponge.main() + + not_a_tty = ('Interactive input not supported. Use piped input\n' + ' echo message | {}\n'.format(' '.join(self.argv))) + self.assertEqual(stderr.getvalue(), not_a_tty) + + +if __name__ == '__main__': + unittest.main() From 94f2a499a326e34cc8ca52343f588d4517936c71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Harter?= Date: Tue, 7 Aug 2018 16:13:17 +0200 Subject: [PATCH 2/3] make: add LAZYSPONGE and LAZYSPONGE_FLAGS variables --- Makefile.include | 5 +++++ makefiles/vars.inc.mk | 3 +++ 2 files changed, 8 insertions(+) diff --git a/Makefile.include b/Makefile.include index 5fe885dc90d8..c16e016de87f 100644 --- a/Makefile.include +++ b/Makefile.include @@ -186,6 +186,11 @@ ifeq (,$(UNZIP_HERE)) endif endif +# Tool saving stdin to a file only on content update. +# It keeps the file timestamp if it would end up the same. +LAZYSPONGE ?= $(RIOTTOOLS)/lazysponge/lazysponge.py +LAZYSPONGE_FLAGS ?= $(if $(filter 1,$(QUIET)),,--verbose) + ifeq (, $(APPLICATION)) $(error An application name must be specified as APPLICATION.) endif diff --git a/makefiles/vars.inc.mk b/makefiles/vars.inc.mk index abe85369911e..7fc732cd1056 100644 --- a/makefiles/vars.inc.mk +++ b/makefiles/vars.inc.mk @@ -88,3 +88,6 @@ export DLCACHE # directory used to cache http downloads export DOWNLOAD_TO_FILE # Use `$(DOWNLOAD_TO_FILE) $(DESTINATION) $(URL)` to download `$(URL)` to `$(DESTINATION)`. export DOWNLOAD_TO_STDOUT # Use `$(DOWNLOAD_TO_STDOUT) $(URL)` to download `$(URL)` output `$(URL)` to stdout, e.g. to be piped into `tar xz`. export UNZIP_HERE # Use `cd $(SOME_FOLDER) && $(UNZIP_HERE) $(SOME_FILE)` to extract the contents of the zip file `$(SOME_FILE)` into `$(SOME_FOLDER)`. + +export LAZYSPONGE # Command saving stdin to a file only on content update. +export LAZYSPONGE_FLAGS # Parameters supplied to LAZYSPONGE. From cc63d2d21cf524f95fbefcb5900619ed648c0af7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Harter?= Date: Fri, 16 Mar 2018 18:13:12 +0100 Subject: [PATCH 3/3] genconfigheader: use lazysponge for file management Remove file management from `genconfigheader` script and use `lazysponge` in Makefile.include. Use --verbose option when in non QUIET building mode. --- Makefile.include | 3 +- dist/tools/genconfigheader/genconfigheader.sh | 60 ++++--------------- 2 files changed, 12 insertions(+), 51 deletions(-) diff --git a/Makefile.include b/Makefile.include index c16e016de87f..06ca9e5b6e2f 100644 --- a/Makefile.include +++ b/Makefile.include @@ -680,7 +680,8 @@ include $(RIOTTOOLS)/desvirt/Makefile.desvirt # The script will only touch the file if anything has changed since last time. $(RIOTBUILD_CONFIG_HEADER_C): FORCE @mkdir -p '$(dir $@)' - $(Q)'$(RIOTTOOLS)/genconfigheader/genconfigheader.sh' '$@' $(CFLAGS_WITH_MACROS) + $(Q)'$(RIOTTOOLS)/genconfigheader/genconfigheader.sh' $(CFLAGS_WITH_MACROS) \ + | '$(LAZYSPONGE)' $(LAZYSPONGE_FLAGS) '$@' CFLAGS_WITH_MACROS := $(CFLAGS) diff --git a/dist/tools/genconfigheader/genconfigheader.sh b/dist/tools/genconfigheader/genconfigheader.sh index bf17aef45c66..581ceb55e0ce 100755 --- a/dist/tools/genconfigheader/genconfigheader.sh +++ b/dist/tools/genconfigheader/genconfigheader.sh @@ -7,43 +7,18 @@ # Public License v2.1. See the file LICENSE in the top level directory for more # details. # - -DEBUG=0 -if [ "${QUIET}" != "1" ]; then - DEBUG=1 -fi - -if [ $# -lt 1 ]; then - echo "Usage: $0 [CFLAGS]..." - echo "Extract all macros from CFLAGS and generate a header file" - exit 1 -fi -OUTPUTFILE="$1" -shift - -MD5SUM=md5sum -if [ "$(uname -s)" = "Darwin" -o "$(uname -s)" = "FreeBSD" ]; then - MD5SUM="md5 -r" -fi - -# atomically update the file -TMPFILE= -trap '[ -n "${TMPFILE}" ] && rm -f "${TMPFILE}"' EXIT -# Create temporary output file -TMPFILE=$(mktemp ${OUTPUTFILE}.XXXXXX) - -if [ -z "${TMPFILE}" ]; then - echo "Error creating temporary file, aborting" - exit 1 -fi +# Usage: $0 [CFLAGS]... +# +# Extract all macros from CFLAGS and generate a header file format" +# # exit on any errors below this line set -e -echo "/* DO NOT edit this file, your changes will be overwritten and won't take any effect! */" > "${TMPFILE}" -echo "/* Generated from CFLAGS: $@ */" >> "${TMPFILE}" +echo "/* DO NOT edit this file, your changes will be overwritten and won't take any effect! */" +echo "/* Generated from CFLAGS: $@ */" -[ -n "${LTOFLAGS}" ] && echo "/* LTOFLAGS=${LTOFLAGS} */" >> "${TMPFILE}" +[ -n "${LTOFLAGS}" ] && echo "/* LTOFLAGS=${LTOFLAGS} */" for arg in "$@"; do case ${arg} in @@ -54,34 +29,19 @@ for arg in "$@"; do # key=value pairs key=${d%%=*} value=${d#*=} - echo "#define $key $value" >> "${TMPFILE}" + echo "#define $key $value" else # simple #define - echo "#define $d 1" >> "${TMPFILE}" + echo "#define $d 1" fi ;; -U*) # Strip leading -U d=${arg#-U} - echo "#undef $d" >> "${TMPFILE}" + echo "#undef $d" ;; *) continue ;; esac done - -# Only replace old file if the new file differs. This allows make to check the -# date of the config header for dependency calculations. -NEWMD5=$(${MD5SUM} ${TMPFILE} | cut -c -32) -OLDMD5=$(${MD5SUM} ${OUTPUTFILE} 2>/dev/null | cut -c -32) -if [ "${NEWMD5}" != "${OLDMD5}" ]; then - if [ "${DEBUG}" -eq 1 ]; then echo "Replacing ${OUTPUTFILE} (${NEWMD5} != ${OLDMD5})"; fi - # Set mode according to umask - chmod +rw "${TMPFILE}" - mv -f "${TMPFILE}" "${OUTPUTFILE}" -else - if [ "${DEBUG}" -eq 1 ]; then echo "Keeping old ${OUTPUTFILE}"; fi -fi - -# $TMPFILE will be deleted by the EXIT trap above if it still exists when we exit