From b81693380b69e8bc401e637c74e3ee4f869a046b Mon Sep 17 00:00:00 2001 From: Jan Decavele Date: Tue, 26 Jan 2021 13:00:53 +0100 Subject: [PATCH 1/3] Asset versioning: first draft --- gulpfile.js | 4 ++- lib/.buildozerrc | 1 + lib/gulp/configs.js | 11 ++++++ lib/gulp/version.js | 88 +++++++++++++++++++++++++++++++++++++++++++++ package-lock.json | 26 +++++++++++--- package.json | 1 + 6 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 lib/gulp/version.js diff --git a/gulpfile.js b/gulpfile.js index 6594471..654c07c 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -10,6 +10,7 @@ const copy = require('./lib/gulp/copy.js'); const {css, cssCompile} = require('./lib/gulp/css.js'); const {img, imgCompile, svgSprite} = require('./lib/gulp/image.js'); const {js, jsCompile, jsConcat} = require('./lib/gulp/js.js'); +const {versionAssets} = require('./lib/gulp/version.js'); // Watch files async function watchFiles() { @@ -107,7 +108,7 @@ function setEnvironment(cb) { } // Define complex tasks -const build = gulp.series(setEnvironment, clean, copy, gulp.parallel(css, js, img)); +const build = gulp.series(setEnvironment, clean, copy, gulp.parallel(css, js, img), versionAssets); const watch = gulp.series(setEnvironment, clean, copy, watchFiles); // Export tasks @@ -118,6 +119,7 @@ exports.js = js; exports['js-concat'] = jsConcat; exports['svg-sprite'] = svgSprite; exports.clean = clean; +exports.versionAssets = versionAssets; exports.setEnvironment = setEnvironment; exports.build = build; exports.watch = watch; diff --git a/lib/.buildozerrc b/lib/.buildozerrc index cafa57f..37ae9fe 100644 --- a/lib/.buildozerrc +++ b/lib/.buildozerrc @@ -1,5 +1,6 @@ src_base_path: ./ dest_base_path: ./ +manifest: ./manifest.json css: - src: css/**/*.css dest: dist/css diff --git a/lib/gulp/configs.js b/lib/gulp/configs.js index db5b16a..4801582 100644 --- a/lib/gulp/configs.js +++ b/lib/gulp/configs.js @@ -29,6 +29,12 @@ function watchConfigPaths(paths, name, cwd) { function processConfig(config, cwd) { config.cwd = cwd; + // Is a manifest defined for this config, and is it a global + const {manifest} = config; + if (typeof manifest === 'string') { + config.manifest_path = path.join(cwd, config.dest_base_path, manifest); + } + ['css', 'js', 'img', 'js-concat', 'svg-sprite', 'copy'].forEach(key => { if (config[key]) { config[key] = config[key].map(value => { @@ -52,6 +58,11 @@ function processConfig(config, cwd) { conf.name = value.name; } + // Check whether a specific manifest is defined for this task + if (typeof manifest === 'object' && manifest[key] !== undefined) { + conf.manifest_path = path.join(dest, manifest[key]); + } + return conf; }); } diff --git a/lib/gulp/version.js b/lib/gulp/version.js new file mode 100644 index 0000000..f89ebd1 --- /dev/null +++ b/lib/gulp/version.js @@ -0,0 +1,88 @@ +const configs = require('./configs.js'); +const path = require('path'); +const {glob} = require("glob"); +const hasha = require('hasha'); +const fs = require('fs'); + +const extensionsToVersion = ['.css', '.js'] + +// Version assets before copying them +async function versionAssets(resolve) { + const versionTasks = []; + await configs.then(configurations => { + configurations.forEach(config => { + if (config.manifest_path !== undefined) { + // Generate one big manifest + const dest = path.join(config.cwd, config.dest_base_path) + versionTasks.push(generateManifest(dest, config.manifest_path)); + } else { + // Generate manifest per defined task + for (const subTask of Object.values(config)) { + if (Array.isArray(subTask)) { + versionTasks.concat(subTask.filter((subTaskConfig) => subTaskConfig.manifest_path !== undefined).map((subTaskConfig) => { + return generateManifest(subTaskConfig.dest, subTaskConfig.manifest_path) + })) + } + } + + } + } + ); + }); + await Promise.all(versionTasks); + resolve(); +} + +async function generateManifest(srcDir, manifestPath) { + const entries = {}; + createManifest(manifestPath); + + glob(`${srcDir}/**/*+(${extensionsToVersion.join('|')})`, {}, async (err, files) => { + if (err) { + throw err; + } + + for (const fullPath of files) { + const versionedFile = await generateVersionedFile(fullPath) + // Trim down absolute paths and record them + const relativeSource = fullPath.replace(srcDir, ''); + entries[relativeSource] = versionedFile.replace(srcDir, ''); + } + dumpToManifest(entries, manifestPath); + }); +} + +async function generateVersionedFile(fullPath) { + const hash = await hasha.fromFile(fullPath, {algorithm: 'md5'}); + let versionedFileName = fullPath; + for (const extension of extensionsToVersion) { + versionedFileName = versionedFileName.replace(extension, `.${hash}${extension}`); + } + fs.copyFile(fullPath, versionedFileName, () => { + }); + return versionedFileName; +} + +function createManifest(manifestPath) { + fs.writeFile(manifestPath, '{}', (err) => { + if (err) + throw err; + }) + +} + +function dumpToManifest(newEntries, targetPath) { + fs.readFile(targetPath, (err, data) => { + if (err) + throw err; + + const currentEntries = JSON.parse(data); + Object.assign(currentEntries,newEntries); + fs.writeFile(targetPath, JSON.stringify(currentEntries), (err) => { + if (err) + throw err; + }); + }) +} + +module.exports = {versionAssets}; diff --git a/package-lock.json b/package-lock.json index 1a963f3..9aa9d9d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8249,12 +8249,19 @@ } }, "hasha": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/hasha/-/hasha-2.2.0.tgz", - "integrity": "sha1-eNfL/B5tZjA/55g3NlmEUXsvbuE=", + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/hasha/-/hasha-5.2.2.tgz", + "integrity": "sha512-Hrp5vIK/xr5SkeN2onO32H0MgNZ0f17HRNH39WfL0SYUNOTZ5Lz1TJ8Pajo/87dYGEFlLMm7mIc/k/s6Bvz9HQ==", "requires": { - "is-stream": "^1.0.1", - "pinkie-promise": "^2.0.0" + "is-stream": "^2.0.0", + "type-fest": "^0.8.0" + }, + "dependencies": { + "is-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", + "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==" + } } }, "he": { @@ -11283,6 +11290,15 @@ "klaw": "^1.0.0" } }, + "hasha": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/hasha/-/hasha-2.2.0.tgz", + "integrity": "sha1-eNfL/B5tZjA/55g3NlmEUXsvbuE=", + "requires": { + "is-stream": "^1.0.1", + "pinkie-promise": "^2.0.0" + } + }, "jsonfile": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", diff --git a/package.json b/package.json index c4e0cae..2a2b97d 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "gulp-stylelint": "^13.0.0", "gulp-svg-sprite": "^1.5.0", "gulp-terser": "^2.0.0", + "hasha": "^5.2.2", "plugin-error": "^1.0.1", "postcss": "^8.2.1", "postcss-load-config": "^3.0.0", From bd134be999de907ab426f3f429d5f9feea286b08 Mon Sep 17 00:00:00 2001 From: Jan Decavele Date: Tue, 26 Jan 2021 19:07:56 +0100 Subject: [PATCH 2/3] Removed manifest from default config --- lib/.buildozerrc | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/.buildozerrc b/lib/.buildozerrc index 37ae9fe..cafa57f 100644 --- a/lib/.buildozerrc +++ b/lib/.buildozerrc @@ -1,6 +1,5 @@ src_base_path: ./ dest_base_path: ./ -manifest: ./manifest.json css: - src: css/**/*.css dest: dist/css From a19a693ddbef31339ef94381c72770975063ec53 Mon Sep 17 00:00:00 2001 From: Jan Decavele Date: Tue, 26 Jan 2021 19:56:18 +0100 Subject: [PATCH 3/3] Fixed style and added tests --- lib/gulp/configs.js | 10 ++- lib/gulp/version.js | 72 +++++++++++-------- package.json | 1 + test/versioning/.buildozerrc | 2 + test/versioning/css/dir/_import.scss | 3 + test/versioning/css/main.scss | 7 ++ .../main.890ad1afcf5a0f80d220a0cdef8f38b3.css | 1 + test/versioning/dist/css/main.css | 1 + .../all.904701738b552057566438714a768e36.js | 1 + test/versioning/dist/js/all.js | 1 + .../test.01d0f06fa45f22e5815640c3cefcefd5.js | 1 + test/versioning/dist/js/concat/test.js | 1 + .../test2.b798d2b5b7ef0fe5124303a842f51457.js | 1 + test/versioning/dist/js/concat/test2.js | 1 + .../test.01d0f06fa45f22e5815640c3cefcefd5.js | 1 + test/versioning/dist/js/test.js | 1 + test/versioning/dist/manifest.json | 1 + test/versioning/js/concat/test.js | 7 ++ test/versioning/js/concat/test2.js | 7 ++ test/versioning/js/test.js | 7 ++ 20 files changed, 94 insertions(+), 33 deletions(-) create mode 100644 test/versioning/.buildozerrc create mode 100644 test/versioning/css/dir/_import.scss create mode 100644 test/versioning/css/main.scss create mode 100644 test/versioning/dist/css/main.890ad1afcf5a0f80d220a0cdef8f38b3.css create mode 100644 test/versioning/dist/css/main.css create mode 100644 test/versioning/dist/js/all.904701738b552057566438714a768e36.js create mode 100644 test/versioning/dist/js/all.js create mode 100644 test/versioning/dist/js/concat/test.01d0f06fa45f22e5815640c3cefcefd5.js create mode 100644 test/versioning/dist/js/concat/test.js create mode 100644 test/versioning/dist/js/concat/test2.b798d2b5b7ef0fe5124303a842f51457.js create mode 100644 test/versioning/dist/js/concat/test2.js create mode 100644 test/versioning/dist/js/test.01d0f06fa45f22e5815640c3cefcefd5.js create mode 100644 test/versioning/dist/js/test.js create mode 100644 test/versioning/dist/manifest.json create mode 100644 test/versioning/js/concat/test.js create mode 100644 test/versioning/js/concat/test2.js create mode 100644 test/versioning/js/test.js diff --git a/lib/gulp/configs.js b/lib/gulp/configs.js index 4801582..1f267cb 100644 --- a/lib/gulp/configs.js +++ b/lib/gulp/configs.js @@ -32,7 +32,8 @@ function processConfig(config, cwd) { // Is a manifest defined for this config, and is it a global const {manifest} = config; if (typeof manifest === 'string') { - config.manifest_path = path.join(cwd, config.dest_base_path, manifest); + config.manifestSource = path.join(cwd, config.manifest_source || config.dest_base_path); + config.manifestPath = path.join(config.manifest_source, manifest); } ['css', 'js', 'img', 'js-concat', 'svg-sprite', 'copy'].forEach(key => { @@ -60,7 +61,7 @@ function processConfig(config, cwd) { // Check whether a specific manifest is defined for this task if (typeof manifest === 'object' && manifest[key] !== undefined) { - conf.manifest_path = path.join(dest, manifest[key]); + conf.manifestPath = path.join(dest, manifest[key]); } return conf; @@ -86,7 +87,10 @@ async function configs() { // Search for configs if config_search is enabled if (config[0].config_search.enabled) { await new Promise((resolve, reject) => { - glob('**/{.buildozerrc,.buildozerrc.json,.buildozerrc.yaml,.buildozerrc.yml,.buildozerrc.js,buildozerrc.config.js}', {cwd, ignore: config[0].config_search.ignore}, (error, files) => { + glob('**/{.buildozerrc,.buildozerrc.json,.buildozerrc.yaml,.buildozerrc.yml,.buildozerrc.js,buildozerrc.config.js}', { + cwd, + ignore: config[0].config_search.ignore + }, (error, files) => { if (error) { reject(error); } else { diff --git a/lib/gulp/version.js b/lib/gulp/version.js index f89ebd1..ceb03d1 100644 --- a/lib/gulp/version.js +++ b/lib/gulp/version.js @@ -1,32 +1,29 @@ const configs = require('./configs.js'); -const path = require('path'); -const {glob} = require("glob"); +const {glob} = require('glob'); const hasha = require('hasha'); const fs = require('fs'); -const extensionsToVersion = ['.css', '.js'] +const extensionsToVersion = ['.css', '.js']; // Version assets before copying them async function versionAssets(resolve) { const versionTasks = []; await configs.then(configurations => { configurations.forEach(config => { - if (config.manifest_path !== undefined) { - // Generate one big manifest - const dest = path.join(config.cwd, config.dest_base_path) - versionTasks.push(generateManifest(dest, config.manifest_path)); - } else { - // Generate manifest per defined task - for (const subTask of Object.values(config)) { - if (Array.isArray(subTask)) { - versionTasks.concat(subTask.filter((subTaskConfig) => subTaskConfig.manifest_path !== undefined).map((subTaskConfig) => { - return generateManifest(subTaskConfig.dest, subTaskConfig.manifest_path) - })) - } + if (config.manifestPath === undefined) { + // Generate manifest per defined task + for (const subTask of Object.values(config)) { + if (Array.isArray(subTask)) { + versionTasks.concat(subTask.filter(subTaskConfig => subTaskConfig.manifestPath !== undefined).map(subTaskConfig => { + return generateManifest(subTaskConfig.dest, subTaskConfig.manifestPath); + })); } - } + } else { + // Generate one big manifest + versionTasks.push(generateManifest(config.manifestSource, config.manifestPath)); } + } ); }); await Promise.all(versionTasks); @@ -42,47 +39,62 @@ async function generateManifest(srcDir, manifestPath) { throw err; } + const subTasks = []; + for (const fullPath of files) { - const versionedFile = await generateVersionedFile(fullPath) - // Trim down absolute paths and record them - const relativeSource = fullPath.replace(srcDir, ''); - entries[relativeSource] = versionedFile.replace(srcDir, ''); + subTasks.push(generateVersionedFileAndLog(fullPath, srcDir, entries)); } + + await Promise.all(subTasks); + dumpToManifest(entries, manifestPath); }); } +async function generateVersionedFileAndLog(fullPath, srcDir, entryLog) { + const versionedFile = await generateVersionedFile(fullPath); + // Trim down absolute paths and record them + const relativeSource = fullPath.replace(srcDir, ''); + entryLog[relativeSource] = versionedFile.replace(srcDir, ''); +} + async function generateVersionedFile(fullPath) { const hash = await hasha.fromFile(fullPath, {algorithm: 'md5'}); let versionedFileName = fullPath; for (const extension of extensionsToVersion) { versionedFileName = versionedFileName.replace(extension, `.${hash}${extension}`); } - fs.copyFile(fullPath, versionedFileName, () => { + + fs.copyFile(fullPath, versionedFileName, err => { + if (err) { + throw err; + } }); return versionedFileName; } function createManifest(manifestPath) { - fs.writeFile(manifestPath, '{}', (err) => { - if (err) + fs.writeFile(manifestPath, '{}', err => { + if (err) { throw err; - }) - + } + }); } function dumpToManifest(newEntries, targetPath) { fs.readFile(targetPath, (err, data) => { - if (err) + if (err) { throw err; + } const currentEntries = JSON.parse(data); - Object.assign(currentEntries,newEntries); - fs.writeFile(targetPath, JSON.stringify(currentEntries), (err) => { - if (err) + Object.assign(currentEntries, newEntries); + fs.writeFile(targetPath, JSON.stringify(currentEntries), err => { + if (err) { throw err; + } }); - }) + }); } module.exports = {versionAssets}; diff --git a/package.json b/package.json index 2a2b97d..45ff07a 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "test-copy": "node bin/buildozer build --cwd=test/copy", "test-sass-module": "node bin/buildozer build --cwd=test/sass-module", "test-config-search": "node bin/buildozer build --cwd=test/config-search", + "test-versioning": "node bin/buildozer build --cwd=test/versioning", "test-stylelint": "node bin/buildozer build --cwd=test/stylelint", "test-eslint": "node bin/buildozer build --cwd=test/eslint", "test-fails": "node test/fail-tests.js", diff --git a/test/versioning/.buildozerrc b/test/versioning/.buildozerrc new file mode 100644 index 0000000..54bd968 --- /dev/null +++ b/test/versioning/.buildozerrc @@ -0,0 +1,2 @@ +manifest_source: ./dist +manifest: ./manifest.json diff --git a/test/versioning/css/dir/_import.scss b/test/versioning/css/dir/_import.scss new file mode 100644 index 0000000..ae844dc --- /dev/null +++ b/test/versioning/css/dir/_import.scss @@ -0,0 +1,3 @@ +body { + background: blue; +} diff --git a/test/versioning/css/main.scss b/test/versioning/css/main.scss new file mode 100644 index 0000000..eda7309 --- /dev/null +++ b/test/versioning/css/main.scss @@ -0,0 +1,7 @@ +@import "dir/import"; + +body { + background: red; + appearance: none; + font-size: 40px; +} diff --git a/test/versioning/dist/css/main.890ad1afcf5a0f80d220a0cdef8f38b3.css b/test/versioning/dist/css/main.890ad1afcf5a0f80d220a0cdef8f38b3.css new file mode 100644 index 0000000..cd96dfb --- /dev/null +++ b/test/versioning/dist/css/main.890ad1afcf5a0f80d220a0cdef8f38b3.css @@ -0,0 +1 @@ +body{background:red;-webkit-appearance:none;-moz-appearance:none;appearance:none;font-size:40px} \ No newline at end of file diff --git a/test/versioning/dist/css/main.css b/test/versioning/dist/css/main.css new file mode 100644 index 0000000..cd96dfb --- /dev/null +++ b/test/versioning/dist/css/main.css @@ -0,0 +1 @@ +body{background:red;-webkit-appearance:none;-moz-appearance:none;appearance:none;font-size:40px} \ No newline at end of file diff --git a/test/versioning/dist/js/all.904701738b552057566438714a768e36.js b/test/versioning/dist/js/all.904701738b552057566438714a768e36.js new file mode 100644 index 0000000..82717ac --- /dev/null +++ b/test/versioning/dist/js/all.904701738b552057566438714a768e36.js @@ -0,0 +1 @@ +"use strict";var q=3,test=function(t){console.log(t)};test(q);var b=3,test2=function(t){console.log(t)};test2(b); \ No newline at end of file diff --git a/test/versioning/dist/js/all.js b/test/versioning/dist/js/all.js new file mode 100644 index 0000000..82717ac --- /dev/null +++ b/test/versioning/dist/js/all.js @@ -0,0 +1 @@ +"use strict";var q=3,test=function(t){console.log(t)};test(q);var b=3,test2=function(t){console.log(t)};test2(b); \ No newline at end of file diff --git a/test/versioning/dist/js/concat/test.01d0f06fa45f22e5815640c3cefcefd5.js b/test/versioning/dist/js/concat/test.01d0f06fa45f22e5815640c3cefcefd5.js new file mode 100644 index 0000000..ef81d7c --- /dev/null +++ b/test/versioning/dist/js/concat/test.01d0f06fa45f22e5815640c3cefcefd5.js @@ -0,0 +1 @@ +"use strict";var q=3,test=function(t){console.log(t)};test(q); \ No newline at end of file diff --git a/test/versioning/dist/js/concat/test.js b/test/versioning/dist/js/concat/test.js new file mode 100644 index 0000000..ef81d7c --- /dev/null +++ b/test/versioning/dist/js/concat/test.js @@ -0,0 +1 @@ +"use strict";var q=3,test=function(t){console.log(t)};test(q); \ No newline at end of file diff --git a/test/versioning/dist/js/concat/test2.b798d2b5b7ef0fe5124303a842f51457.js b/test/versioning/dist/js/concat/test2.b798d2b5b7ef0fe5124303a842f51457.js new file mode 100644 index 0000000..bcdb326 --- /dev/null +++ b/test/versioning/dist/js/concat/test2.b798d2b5b7ef0fe5124303a842f51457.js @@ -0,0 +1 @@ +"use strict";var b=3,test2=function(t){console.log(t)};test2(b); \ No newline at end of file diff --git a/test/versioning/dist/js/concat/test2.js b/test/versioning/dist/js/concat/test2.js new file mode 100644 index 0000000..bcdb326 --- /dev/null +++ b/test/versioning/dist/js/concat/test2.js @@ -0,0 +1 @@ +"use strict";var b=3,test2=function(t){console.log(t)};test2(b); \ No newline at end of file diff --git a/test/versioning/dist/js/test.01d0f06fa45f22e5815640c3cefcefd5.js b/test/versioning/dist/js/test.01d0f06fa45f22e5815640c3cefcefd5.js new file mode 100644 index 0000000..ef81d7c --- /dev/null +++ b/test/versioning/dist/js/test.01d0f06fa45f22e5815640c3cefcefd5.js @@ -0,0 +1 @@ +"use strict";var q=3,test=function(t){console.log(t)};test(q); \ No newline at end of file diff --git a/test/versioning/dist/js/test.js b/test/versioning/dist/js/test.js new file mode 100644 index 0000000..ef81d7c --- /dev/null +++ b/test/versioning/dist/js/test.js @@ -0,0 +1 @@ +"use strict";var q=3,test=function(t){console.log(t)};test(q); \ No newline at end of file diff --git a/test/versioning/dist/manifest.json b/test/versioning/dist/manifest.json new file mode 100644 index 0000000..a94ddd0 --- /dev/null +++ b/test/versioning/dist/manifest.json @@ -0,0 +1 @@ +{"/css/main.css":"/css/main.890ad1afcf5a0f80d220a0cdef8f38b3.css","/js/all.js":"/js/all.904701738b552057566438714a768e36.js","/js/concat/test.js":"/js/concat/test.01d0f06fa45f22e5815640c3cefcefd5.js","/js/concat/test2.js":"/js/concat/test2.b798d2b5b7ef0fe5124303a842f51457.js","/js/test.js":"/js/test.01d0f06fa45f22e5815640c3cefcefd5.js"} \ No newline at end of file diff --git a/test/versioning/js/concat/test.js b/test/versioning/js/concat/test.js new file mode 100644 index 0000000..8b745c4 --- /dev/null +++ b/test/versioning/js/concat/test.js @@ -0,0 +1,7 @@ +const q = 3; + +const test = value => { + console.log(value); +}; + +test(q); diff --git a/test/versioning/js/concat/test2.js b/test/versioning/js/concat/test2.js new file mode 100644 index 0000000..36d3727 --- /dev/null +++ b/test/versioning/js/concat/test2.js @@ -0,0 +1,7 @@ +const b = 3; + +const test2 = value => { + console.log(value); +}; + +test2(b); diff --git a/test/versioning/js/test.js b/test/versioning/js/test.js new file mode 100644 index 0000000..8b745c4 --- /dev/null +++ b/test/versioning/js/test.js @@ -0,0 +1,7 @@ +const q = 3; + +const test = value => { + console.log(value); +}; + +test(q);