From b148e714e041bb58262e262ae92baf6b8b9a5fd3 Mon Sep 17 00:00:00 2001 From: showitz Date: Mon, 5 Oct 2020 11:14:49 +0200 Subject: [PATCH 1/2] Develop (#5) * Feature/zipper stream arguments (#4) * update(zipper): extend sonfig * update(zipper): extend sonfig * update(zipper): handle glob * update(zipper): handle glob * update(zipper): handle streams * Update: updates Co-authored-by: Mykola Fant Co-authored-by: Sina Hwz * Update: version1.0.6 now Co-authored-by: Mykola Fant Co-authored-by: Mykola Fant --- .gitignore | 1 + modules/zipper.js | 119 +++++++++++++++++++++++++++++----------------- package.json | 2 +- 3 files changed, 78 insertions(+), 44 deletions(-) diff --git a/.gitignore b/.gitignore index 3c3629e..eb79dd5 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ node_modules +.idea diff --git a/modules/zipper.js b/modules/zipper.js index b83a685..2187de4 100644 --- a/modules/zipper.js +++ b/modules/zipper.js @@ -2,67 +2,100 @@ const fs = require('fs-extra'); const archiver = require('archiver'); const { ensurePath, pathExists } = require('./file_handler'); +/** + * @typedef {object} ZippableEntity + * @property {string} name + * @property {string|Buffer|ReadableStream} entity + */ + +/** + * @typedef {object} ZipperConfig + * @property {string} [directory] - if you want to create a ZIP from directory + * @property {string} [zipName] - name of the ZIP you want to create, ignored if `outputStream === true` + * @property {string} [dest] - dest of the created ZIP, ignored if `outputStream === true` + * @property {string} [glob] - e.g. if you want a directory with some excluded files you may pass src/**\/!(banana.txt) + * or to exclude a subdirectrory src/!(bananas)/**\/* + * or for a structure like this https://prnt.sc/usx263 => 'modules/{!(excl)/**\/*,**.*} + * Note: "\" is used for escaping comment end in the examples above + * @property {ZippableEntity[]} [entities] - array of ZippableEntity (with strings, buffers or streams) + * @property {boolean} [outputStream=false] - by default the handler will create a file in the fs and resolve the path + */ + /** * Creates a zip file from source * common-mods/modules/zipper -* @param {String} source +* @param {String|ZipperConfig} source * @param {String} zipName * @param {String} dest * @param {Object} options * @param {Array} options.excludedPaths -* @returns {} +* @returns {string|ReadableStream} */ -// TODO: add 'exclude' parameter module.exports.zipper = async (source, zipName, dest) => new Promise(async (resolve, reject) => { - if (pathExists(source)) { - const nameWithDotZip = zipName.endsWith('.zip') ? zipName : `${zipName}.zip`; - const zipPath = `${dest}/${nameWithDotZip}`; - await ensurePath(dest); - const outputZip = fs.createWriteStream(zipPath); - const archive = archiver('zip', { - zlib: { level: 9 }, - }); + const config = typeof source === 'string' + ? { directory: source, zipName, dest } + : { ...source }; + + const archive = archiver('zip', { + zlib: { level: 9 }, + }); - archive.on('error', (err) => { - console.log('[ERR][zipper] ', err); + archive.on('error', (err) => { + console.log('[ERR][zipper] ', err); + reject(err); + }); + + // good practice to catch warnings (ie stat failures and other non-blocking errors) + archive.on('warning', (err) => { + if (err.code === 'ENOENT') { + // log warning + console.log('[DEV] Archive Warning', err); + } else { + console.error('[DEV] Archive error', err); + } + }); + + // handle directory + if (config.directory) { + if (pathExists(config.directory)) { + archive.directory(config.directory, false); + } else { + const err = { + errMessage: ` [ERR][zipper] path not found: ${config.directory}`, + }; reject(err); - }); + } + } - outputZip.on('close', () => { - console.log(`${archive.pointer()} total bytes`); - console.log('[DEV][zipper] archiver has been finalized and the output file descriptor has closed.'); + if (config.glob) { + archive.glob(config.glob); + } - setTimeout(() => { - resolve(zipPath); - }, 1000); - }); + if (config.entities) { + config.entities + .forEach(({ name, entity }) => archive.append(entity, { name })); + } - // good practice to catch warnings (ie stat failures and other non-blocking errors) - archive.on('warning', (err) => { - if (err.code === 'ENOENT') { - // log warning - console.log('[DEV] Archive Warning', err); - } else { - console.error('[DEV] Archive error', err); - } - }); + // finalize archive input + archive.finalize(); - console.log(' :::: source', source); - archive.directory(source, false); + // handle archive output + if (config.outputStream) { + console.log('[DEV][zipper] Stream ready!'); + resolve(archive); + } else { + const nameWithDotZip = config.zipName.endsWith('.zip') ? config.zipName : `${config.zipName}.zip`; + const zipPath = `${config.dest}/${nameWithDotZip}`; + await ensurePath(config.dest); + const outputZip = fs.createWriteStream(zipPath); - // TODO: useful when trying to exclude paths - // TODO: issue is that, it adds all glob matches files under 'source' - // archive.glob(`${source}/**/*`, { - // // ignore: ['mydir/**', 'file.txt'], - // }); + outputZip.on('close', () => { + console.log(`${archive.pointer()} total bytes`); + console.log('[DEV][zipper] archiver has been finalized and the output file descriptor has closed.'); + resolve(zipPath); + }); archive.pipe(outputZip); - archive.finalize(); - } else { - const err = { - errMessage: ` [ERR][zipper] path not found: ${source}`, - }; - reject(err); } }); diff --git a/package.json b/package.json index 3888486..c33b5a6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "common-mods", - "version": "1.0.5", + "version": "1.0.6", "description": "", "main": "index.js", "scripts": { From e019002f6cfee2d1c6d9ab1c5b0a8f56dcb5e316 Mon Sep 17 00:00:00 2001 From: Mykola Fant Date: Wed, 7 Oct 2020 00:02:02 +0300 Subject: [PATCH 2/2] update(file_handler): add getBufferFromStream --- modules/file_handler.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/modules/file_handler.js b/modules/file_handler.js index 4422bd9..239d448 100644 --- a/modules/file_handler.js +++ b/modules/file_handler.js @@ -216,6 +216,20 @@ const readJson = async (path) => { } }; +/** + * Converts a stream to a buffer + * @param {ReadableStream} stream + * @returns {Promise} + */ +const getBufferFromStream = async (stream) => new Promise((res, rej) => { + const buffers = []; + stream.on('data', (chunk) => buffers.push(chunk)); + stream.on('end', () => { + res(Buffer.concat(buffers)); + }); + stream.on('error', (error) => rej(error)); +}); + module.exports = { cleanPath, copyFiles, @@ -230,4 +244,5 @@ module.exports = { renameFile, writeFile, writeJson, + getBufferFromStream, };