RFC: Feature/asset versioning

gulpfile.js

@@ -10,6 +10,7 @@ const copy = require('./lib/gulp/copy.js');
 const {css, cssCompile} = require('./lib/gulp/css.js');
 const {img, imgCompile, svgSprite} = require('./lib/gulp/image.js');
 const {js, jsCompile, jsConcat} = require('./lib/gulp/js.js');
+const {versionAssets} = require('./lib/gulp/version.js');
 
 // Watch files
 async function watchFiles() {
@@ -107,7 +108,7 @@ function setEnvironment(cb) {
 }
 
 // Define complex tasks
-const build = gulp.series(setEnvironment, clean, copy, gulp.parallel(css, js, img));
+const build = gulp.series(setEnvironment, clean, copy, gulp.parallel(css, js, img), versionAssets);
 const watch = gulp.series(setEnvironment, clean, copy, watchFiles);
 
 // Export tasks
@@ -118,6 +119,7 @@ exports.js = js;
 exports['js-concat'] = jsConcat;
 exports['svg-sprite'] = svgSprite;
 exports.clean = clean;
+exports.versionAssets = versionAssets;
 exports.setEnvironment = setEnvironment;
 exports.build = build;
 exports.watch = watch;

lib/gulp/configs.js

@@ -29,6 +29,13 @@ function watchConfigPaths(paths, name, cwd) {
 function processConfig(config, cwd) {
   config.cwd = cwd;
 
+  // Is a manifest defined for this config, and is it a global
+  const {manifest} = config;
+  if (typeof manifest === 'string') {
+    config.manifestSource = path.join(cwd, config.manifest_source || config.dest_base_path);
+    config.manifestPath = path.join(config.manifest_source, manifest);
+  }
+
   ['css', 'js', 'img', 'js-concat', 'svg-sprite', 'copy'].forEach(key => {
     if (config[key]) {
       config[key] = config[key].map(value => {
@@ -52,6 +59,11 @@ function processConfig(config, cwd) {
           conf.name = value.name;
         }
 
+        // Check whether a specific manifest is defined for this task
+        if (typeof manifest === 'object' && manifest[key] !== undefined) {
+          conf.manifestPath = path.join(dest, manifest[key]);
+        }
+
         return conf;
       });
     }
@@ -75,7 +87,10 @@ async function configs() {
   // Search for configs if config_search is enabled
   if (config[0].config_search.enabled) {
     await new Promise((resolve, reject) => {
-      glob('**/{.buildozerrc,.buildozerrc.json,.buildozerrc.yaml,.buildozerrc.yml,.buildozerrc.js,buildozerrc.config.js}', {cwd, ignore: config[0].config_search.ignore}, (error, files) => {
+      glob('**/{.buildozerrc,.buildozerrc.json,.buildozerrc.yaml,.buildozerrc.yml,.buildozerrc.js,buildozerrc.config.js}', {
+        cwd,
+        ignore: config[0].config_search.ignore
+      }, (error, files) => {
         if (error) {
           reject(error);
         } else {

lib/gulp/version.js

@@ -0,0 +1,100 @@
+const configs = require('./configs.js');
+const {glob} = require('glob');
+const hasha = require('hasha');
+const fs = require('fs');
+
+const extensionsToVersion = ['.css', '.js'];
+
+// Version assets before copying them
+async function versionAssets(resolve) {
+  const versionTasks = [];
+  await configs.then(configurations => {
+    configurations.forEach(config => {
+      if (config.manifestPath === undefined) {
+        // Generate manifest per defined task
+        for (const subTask of Object.values(config)) {
+          if (Array.isArray(subTask)) {
+            versionTasks.concat(subTask.filter(subTaskConfig => subTaskConfig.manifestPath !== undefined).map(subTaskConfig => {
+              return generateManifest(subTaskConfig.dest, subTaskConfig.manifestPath);
+            }));
+          }
+        }
+      } else {
+        // Generate one big manifest
+        versionTasks.push(generateManifest(config.manifestSource, config.manifestPath));
+      }
+    }
+    );
+  });
+  await Promise.all(versionTasks);
+  resolve();
+}
+
+async function generateManifest(srcDir, manifestPath) {
+  const entries = {};
+  createManifest(manifestPath);
+
+  glob(`${srcDir}/**/*+(${extensionsToVersion.join('|')})`, {}, async (err, files) => {
+    if (err) {
+      throw err;
+    }
+
+    const subTasks = [];
+
+    for (const fullPath of files) {
+      subTasks.push(generateVersionedFileAndLog(fullPath, srcDir, entries));
+    }
+
+    await Promise.all(subTasks);
+
+    dumpToManifest(entries, manifestPath);
+  });
+}
+
+async function generateVersionedFileAndLog(fullPath, srcDir, entryLog) {
+  const versionedFile = await generateVersionedFile(fullPath);
+  // Trim down absolute paths and record them
+  const relativeSource = fullPath.replace(srcDir, '');
+  entryLog[relativeSource] = versionedFile.replace(srcDir, '');
+}
+
+async function generateVersionedFile(fullPath) {
+  const hash = await hasha.fromFile(fullPath, {algorithm: 'md5'});
+  let versionedFileName = fullPath;
+  for (const extension of extensionsToVersion) {
+    versionedFileName = versionedFileName.replace(extension, `.${hash}${extension}`);
+  }
+
+  fs.copyFile(fullPath, versionedFileName, err => {
+    if (err) {
+      throw err;
+    }
+  });
+  return versionedFileName;
+}
+
+function createManifest(manifestPath) {
+  fs.writeFile(manifestPath, '{}', err => {
+    if (err) {
+      throw err;
+    }
+  });
+}
+
+function dumpToManifest(newEntries, targetPath) {
+  fs.readFile(targetPath, (err, data) => {
+    if (err) {
+      throw err;
+    }
+
+    const currentEntries = JSON.parse(data);
+    Object.assign(currentEntries, newEntries);
+    fs.writeFile(targetPath, JSON.stringify(currentEntries), err => {
+      if (err) {
+        throw err;
+      }
+    });
+  });
+}
+
+module.exports = {versionAssets};

package.json

@@ -32,6 +32,7 @@
     "gulp-stylelint": "^13.0.0",
     "gulp-svg-sprite": "^1.5.0",
     "gulp-terser": "^2.0.0",
+    "hasha": "^5.2.2",
     "plugin-error": "^1.0.1",
     "postcss": "^8.2.1",
     "postcss-load-config": "^3.0.0",
@@ -60,6 +61,7 @@
     "test-copy": "node bin/buildozer build --cwd=test/copy",
     "test-sass-module": "node bin/buildozer build --cwd=test/sass-module",
     "test-config-search": "node bin/buildozer build --cwd=test/config-search",
+    "test-versioning": "node bin/buildozer build --cwd=test/versioning",
     "test-stylelint": "node bin/buildozer build --cwd=test/stylelint",
     "test-eslint": "node bin/buildozer build --cwd=test/eslint",
     "test-fails": "node test/fail-tests.js",

test/versioning/.buildozerrc

@@ -0,0 +1,2 @@
+manifest_source: ./dist
+manifest: ./manifest.json

test/versioning/css/dir/_import.scss

@@ -0,0 +1,3 @@
+body {
+  background: blue;
+}

test/versioning/css/main.scss

@@ -0,0 +1,7 @@
+@import "dir/import";
+
+body {
+  background: red;
+  appearance: none;
+  font-size: 40px;
+}

test/versioning/dist/css/main.890ad1afcf5a0f80d220a0cdef8f38b3.css

@@ -0,0 +1 @@
+body{background:red;-webkit-appearance:none;-moz-appearance:none;appearance:none;font-size:40px}

test/versioning/dist/css/main.css

@@ -0,0 +1 @@
+body{background:red;-webkit-appearance:none;-moz-appearance:none;appearance:none;font-size:40px}

test/versioning/dist/js/concat/test.01d0f06fa45f22e5815640c3cefcefd5.js

@@ -0,0 +1 @@
+"use strict";var q=3,test=function(t){console.log(t)};test(q);

test/versioning/dist/js/concat/test.js

@@ -0,0 +1 @@
+"use strict";var q=3,test=function(t){console.log(t)};test(q);

test/versioning/dist/js/concat/test2.b798d2b5b7ef0fe5124303a842f51457.js

@@ -0,0 +1 @@
+"use strict";var b=3,test2=function(t){console.log(t)};test2(b);

test/versioning/dist/js/concat/test2.js

@@ -0,0 +1 @@
+"use strict";var b=3,test2=function(t){console.log(t)};test2(b);

test/versioning/dist/js/test.01d0f06fa45f22e5815640c3cefcefd5.js

@@ -0,0 +1 @@
+"use strict";var q=3,test=function(t){console.log(t)};test(q);

test/versioning/dist/js/test.js

@@ -0,0 +1 @@
+"use strict";var q=3,test=function(t){console.log(t)};test(q);

test/versioning/dist/manifest.json

@@ -0,0 +1 @@
+{"/css/main.css":"/css/main.890ad1afcf5a0f80d220a0cdef8f38b3.css","/js/all.js":"/js/all.904701738b552057566438714a768e36.js","/js/concat/test.js":"/js/concat/test.01d0f06fa45f22e5815640c3cefcefd5.js","/js/concat/test2.js":"/js/concat/test2.b798d2b5b7ef0fe5124303a842f51457.js","/js/test.js":"/js/test.01d0f06fa45f22e5815640c3cefcefd5.js"}

test/versioning/js/concat/test.js

@@ -0,0 +1,7 @@
+const q = 3;
+
+const test = value => {
+  console.log(value);
+};
+
+test(q);

test/versioning/js/concat/test2.js

@@ -0,0 +1,7 @@
+const b = 3;
+
+const test2 = value => {
+  console.log(value);
+};
+
+test2(b);

test/versioning/js/test.js

@@ -0,0 +1,7 @@
+const q = 3;
+
+const test = value => {
+  console.log(value);
+};
+
+test(q);