From 2eeee4ec15d85d6b365e4546e0f6421689cd918f Mon Sep 17 00:00:00 2001 From: jmz1984 Date: Fri, 23 May 2025 00:32:14 +0000 Subject: [PATCH 1/6] Start draft PR From d06ac1a6e3b5dda654817e6a8902009ee4ab31be Mon Sep 17 00:00:00 2001 From: jmz1984 Date: Fri, 23 May 2025 00:32:48 +0000 Subject: [PATCH 2/6] Add custom Logger utility for web scraper --- lib/Logger.js | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 lib/Logger.js diff --git a/lib/Logger.js b/lib/Logger.js new file mode 100644 index 0000000..7331baa --- /dev/null +++ b/lib/Logger.js @@ -0,0 +1,74 @@ +/** + * Logging utility for Osmosis web scraper + * Provides configurable logging with different log levels + */ +class Logger { + /** + * Creates a new Logger instance + * @param {Object} options - Logger configuration options + * @param {string} [options.level='info'] - Logging level (debug, info, warn, error) + * @param {boolean} [options.enabled=true] - Enable/disable logging + */ + constructor(options = {}) { + this.level = options.level || 'info'; + this.enabled = options.enabled !== false; + this.logLevels = ['debug', 'info', 'warn', 'error']; + } + + /** + * Check if a log level is enabled + * @param {string} level - Log level to check + * @returns {boolean} Whether the log level is enabled + */ + isLevelEnabled(level) { + const currentLevelIndex = this.logLevels.indexOf(this.level); + const checkLevelIndex = this.logLevels.indexOf(level); + return checkLevelIndex >= currentLevelIndex; + } + + /** + * Debug level logging + * @param {string} message - Log message + * @param {Object} [metadata] - Optional additional logging metadata + */ + debug(message, metadata = {}) { + if (this.enabled && this.isLevelEnabled('debug')) { + console.log(`[DEBUG] ${message}`, metadata); + } + } + + /** + * Info level logging + * @param {string} message - Log message + * @param {Object} [metadata] - Optional additional logging metadata + */ + info(message, metadata = {}) { + if (this.enabled && this.isLevelEnabled('info')) { + console.log(`[INFO] ${message}`, metadata); + } + } + + /** + * Warning level logging + * @param {string} message - Log message + * @param {Object} [metadata] - Optional additional logging metadata + */ + warn(message, metadata = {}) { + if (this.enabled && this.isLevelEnabled('warn')) { + console.warn(`[WARN] ${message}`, metadata); + } + } + + /** + * Error level logging + * @param {string} message - Log message + * @param {Object} [metadata] - Optional additional logging metadata + */ + error(message, metadata = {}) { + if (this.enabled && this.isLevelEnabled('error')) { + console.error(`[ERROR] ${message}`, metadata); + } + } +} + +module.exports = Logger; \ No newline at end of file From aa41659d8182aec01443d826403080eb491192f0 Mon Sep 17 00:00:00 2001 From: jmz1984 Date: Fri, 23 May 2025 00:33:02 +0000 Subject: [PATCH 3/6] Add tests for Logger utility --- test/Logger.test.js | 79 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 test/Logger.test.js diff --git a/test/Logger.test.js b/test/Logger.test.js new file mode 100644 index 0000000..91ba4d6 --- /dev/null +++ b/test/Logger.test.js @@ -0,0 +1,79 @@ +const Logger = require('../lib/Logger'); + +describe('Logger', () => { + let consoleSpy; + + beforeEach(() => { + consoleSpy = { + log: jest.spyOn(console, 'log').mockImplementation(), + warn: jest.spyOn(console, 'warn').mockImplementation(), + error: jest.spyOn(console, 'error').mockImplementation() + }; + }); + + afterEach(() => { + consoleSpy.log.mockRestore(); + consoleSpy.warn.mockRestore(); + consoleSpy.error.mockRestore(); + }); + + test('should log debug message when level is debug', () => { + const logger = new Logger({ level: 'debug' }); + logger.debug('Test debug message', { data: 'test' }); + expect(consoleSpy.log).toHaveBeenCalledWith( + expect.stringContaining('[DEBUG]'), + { data: 'test' } + ); + }); + + test('should not log debug message when level is info', () => { + const logger = new Logger({ level: 'info' }); + logger.debug('Test debug message'); + expect(consoleSpy.log).not.toHaveBeenCalled(); + }); + + test('should log info message', () => { + const logger = new Logger(); + logger.info('Test info message', { data: 'test' }); + expect(consoleSpy.log).toHaveBeenCalledWith( + expect.stringContaining('[INFO]'), + { data: 'test' } + ); + }); + + test('should log warn message', () => { + const logger = new Logger(); + logger.warn('Test warn message', { data: 'test' }); + expect(consoleSpy.warn).toHaveBeenCalledWith( + expect.stringContaining('[WARN]'), + { data: 'test' } + ); + }); + + test('should log error message', () => { + const logger = new Logger(); + logger.error('Test error message', { data: 'test' }); + expect(consoleSpy.error).toHaveBeenCalledWith( + expect.stringContaining('[ERROR]'), + { data: 'test' } + ); + }); + + test('should respect log level configuration', () => { + const logger = new Logger({ level: 'warn' }); + logger.debug('Debug message'); + logger.info('Info message'); + logger.warn('Warn message'); + logger.error('Error message'); + + expect(consoleSpy.log).not.toHaveBeenCalled(); + expect(consoleSpy.warn).toHaveBeenCalledWith(expect.stringContaining('[WARN]'), {}); + expect(consoleSpy.error).toHaveBeenCalledWith(expect.stringContaining('[ERROR]'), {}); + }); + + test('should disable logging when enabled is false', () => { + const logger = new Logger({ enabled: false }); + logger.info('Test message'); + expect(consoleSpy.log).not.toHaveBeenCalled(); + }); +}); \ No newline at end of file From a574702a19ea9ce3654664d41a5b4dc83cf10121 Mon Sep 17 00:00:00 2001 From: jmz1984 Date: Fri, 23 May 2025 00:33:32 +0000 Subject: [PATCH 4/6] Update package.json with Jest testing configuration --- package.json | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 5aa2568..88af3ac 100644 --- a/package.json +++ b/package.json @@ -25,10 +25,16 @@ }, "devDependencies": { "jscs": ">=3.0.2", - "nodeunit": "0.11.3" + "nodeunit": "0.11.3", + "jest": "^29.7.0" }, "scripts": { - "test": "node ./node_modules/.bin/nodeunit test" + "test": "jest", + "test:watch": "jest --watch" + }, + "jest": { + "testEnvironment": "node", + "verbose": true }, "license": "MIT", "main": "index", @@ -39,4 +45,4 @@ "bugs": { "url": "https://github.com/rchipka/node-osmosis/issues" } -} +} \ No newline at end of file From 0a8b1c9a076e4638a3b86e3df3500676dc7767da Mon Sep 17 00:00:00 2001 From: jmz1984 Date: Fri, 23 May 2025 00:34:52 +0000 Subject: [PATCH 5/6] Enhance Logger with detailed page fetch and error logging --- lib/Logger.js | 110 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 96 insertions(+), 14 deletions(-) diff --git a/lib/Logger.js b/lib/Logger.js index 7331baa..f1127d7 100644 --- a/lib/Logger.js +++ b/lib/Logger.js @@ -1,6 +1,9 @@ +const fs = require('fs'); +const path = require('path'); + /** - * Logging utility for Osmosis web scraper - * Provides configurable logging with different log levels + * Advanced logging utility for web scraping activities + * Supports console and file-based logging with detailed error tracking */ class Logger { /** @@ -8,13 +11,46 @@ class Logger { * @param {Object} options - Logger configuration options * @param {string} [options.level='info'] - Logging level (debug, info, warn, error) * @param {boolean} [options.enabled=true] - Enable/disable logging + * @param {string} [options.logFile] - Path to log file for persistent logging */ constructor(options = {}) { this.level = options.level || 'info'; this.enabled = options.enabled !== false; + this.logFile = options.logFile || path.join(process.cwd(), 'scraper.log'); this.logLevels = ['debug', 'info', 'warn', 'error']; } + /** + * Generate a timestamp for logging + * @returns {string} Formatted timestamp + */ + _getTimestamp() { + return new Date().toISOString(); + } + + /** + * Write log message to file + * @param {string} level - Log level + * @param {string} message - Log message + * @param {Object} [metadata] - Additional logging metadata + */ + _writeToFile(level, message, metadata = {}) { + if (!this.logFile) return; + + const logEntry = JSON.stringify({ + timestamp: this._getTimestamp(), + level, + message, + metadata + }) + '\n'; + + try { + fs.appendFileSync(this.logFile, logEntry); + } catch (error) { + console.error('Failed to write to log file:', error); + } + } + /** * Check if a log level is enabled * @param {string} level - Log level to check @@ -26,15 +62,58 @@ class Logger { return checkLevelIndex >= currentLevelIndex; } + /** + * Log a successful page fetch + * @param {string} url - URL of the fetched page + * @param {Object} [details] - Additional fetch details + */ + logPageFetch(url, details = {}) { + if (!this.enabled || !this.isLevelEnabled('info')) return; + + const message = `Page fetched successfully: ${url}`; + const metadata = { + url, + ...details + }; + + console.log(`[INFO] ${message}`, metadata); + this._writeToFile('info', message, metadata); + } + + /** + * Log a network or parsing error + * @param {string} type - Error type (network, parsing) + * @param {string} url - URL associated with the error + * @param {Error} error - The error object + * @param {Object} [details] - Additional error details + */ + logError(type, url, error, details = {}) { + if (!this.enabled || !this.isLevelEnabled('error')) return; + + const message = `${type.toUpperCase()} Error: ${error.message}`; + const metadata = { + type, + url, + errorName: error.name, + errorMessage: error.message, + stack: error.stack, + ...details + }; + + console.error(`[ERROR] ${message}`, metadata); + this._writeToFile('error', message, metadata); + } + /** * Debug level logging * @param {string} message - Log message * @param {Object} [metadata] - Optional additional logging metadata */ debug(message, metadata = {}) { - if (this.enabled && this.isLevelEnabled('debug')) { - console.log(`[DEBUG] ${message}`, metadata); - } + if (!this.enabled || !this.isLevelEnabled('debug')) return; + + console.log(`[DEBUG] ${message}`, metadata); + this._writeToFile('debug', message, metadata); } /** @@ -43,9 +122,10 @@ class Logger { * @param {Object} [metadata] - Optional additional logging metadata */ info(message, metadata = {}) { - if (this.enabled && this.isLevelEnabled('info')) { - console.log(`[INFO] ${message}`, metadata); - } + if (!this.enabled || !this.isLevelEnabled('info')) return; + + console.log(`[INFO] ${message}`, metadata); + this._writeToFile('info', message, metadata); } /** @@ -54,9 +134,10 @@ class Logger { * @param {Object} [metadata] - Optional additional logging metadata */ warn(message, metadata = {}) { - if (this.enabled && this.isLevelEnabled('warn')) { - console.warn(`[WARN] ${message}`, metadata); - } + if (!this.enabled || !this.isLevelEnabled('warn')) return; + + console.warn(`[WARN] ${message}`, metadata); + this._writeToFile('warn', message, metadata); } /** @@ -65,9 +146,10 @@ class Logger { * @param {Object} [metadata] - Optional additional logging metadata */ error(message, metadata = {}) { - if (this.enabled && this.isLevelEnabled('error')) { - console.error(`[ERROR] ${message}`, metadata); - } + if (!this.enabled || !this.isLevelEnabled('error')) return; + + console.error(`[ERROR] ${message}`, metadata); + this._writeToFile('error', message, metadata); } } From 610ec799691aa91d1579321a4df0e8a4918ed384 Mon Sep 17 00:00:00 2001 From: jmz1984 Date: Fri, 23 May 2025 00:35:10 +0000 Subject: [PATCH 6/6] Update Logger tests to cover new logging methods --- test/Logger.test.js | 127 ++++++++++++++++++++++++++++++-------------- 1 file changed, 86 insertions(+), 41 deletions(-) diff --git a/test/Logger.test.js b/test/Logger.test.js index 91ba4d6..3d8f21f 100644 --- a/test/Logger.test.js +++ b/test/Logger.test.js @@ -1,9 +1,15 @@ +const fs = require('fs'); +const path = require('path'); const Logger = require('../lib/Logger'); describe('Logger', () => { let consoleSpy; + let testLogFile; beforeEach(() => { + // Create a unique test log file for each test + testLogFile = path.join(process.cwd(), `test-${Date.now()}.log`); + consoleSpy = { log: jest.spyOn(console, 'log').mockImplementation(), warn: jest.spyOn(console, 'warn').mockImplementation(), @@ -12,68 +18,107 @@ describe('Logger', () => { }); afterEach(() => { + // Restore console methods consoleSpy.log.mockRestore(); consoleSpy.warn.mockRestore(); consoleSpy.error.mockRestore(); - }); - test('should log debug message when level is debug', () => { - const logger = new Logger({ level: 'debug' }); - logger.debug('Test debug message', { data: 'test' }); - expect(consoleSpy.log).toHaveBeenCalledWith( - expect.stringContaining('[DEBUG]'), - { data: 'test' } - ); + // Remove test log file if it exists + if (fs.existsSync(testLogFile)) { + fs.unlinkSync(testLogFile); + } }); - test('should not log debug message when level is info', () => { - const logger = new Logger({ level: 'info' }); - logger.debug('Test debug message'); - expect(consoleSpy.log).not.toHaveBeenCalled(); - }); + test('logPageFetch logs successful page fetch', () => { + const logger = new Logger({ + level: 'info', + logFile: testLogFile + }); + const testUrl = 'https://example.com'; + + logger.logPageFetch(testUrl, { statusCode: 200 }); - test('should log info message', () => { - const logger = new Logger(); - logger.info('Test info message', { data: 'test' }); + // Check console output expect(consoleSpy.log).toHaveBeenCalledWith( expect.stringContaining('[INFO]'), - { data: 'test' } + expect.objectContaining({ + url: testUrl, + statusCode: 200 + }) ); - }); - test('should log warn message', () => { - const logger = new Logger(); - logger.warn('Test warn message', { data: 'test' }); - expect(consoleSpy.warn).toHaveBeenCalledWith( - expect.stringContaining('[WARN]'), - { data: 'test' } - ); + // Check log file content + const logContent = fs.readFileSync(testLogFile, 'utf-8'); + const logEntry = JSON.parse(logContent.trim()); + + expect(logEntry).toMatchObject({ + level: 'info', + message: `Page fetched successfully: ${testUrl}`, + metadata: { + url: testUrl, + statusCode: 200 + } + }); }); - test('should log error message', () => { - const logger = new Logger(); - logger.error('Test error message', { data: 'test' }); + test('logError logs network and parsing errors', () => { + const logger = new Logger({ + level: 'error', + logFile: testLogFile + }); + const testUrl = 'https://example.com'; + const testError = new Error('Connection timeout'); + + logger.logError('network', testUrl, testError, { retryCount: 1 }); + + // Check console output expect(consoleSpy.error).toHaveBeenCalledWith( expect.stringContaining('[ERROR]'), - { data: 'test' } + expect.objectContaining({ + type: 'network', + url: testUrl, + errorMessage: 'Connection timeout', + retryCount: 1 + }) ); + + // Check log file content + const logContent = fs.readFileSync(testLogFile, 'utf-8'); + const logEntry = JSON.parse(logContent.trim()); + + expect(logEntry).toMatchObject({ + level: 'error', + message: 'NETWORK Error: Connection timeout', + metadata: { + type: 'network', + url: testUrl, + errorName: 'Error', + errorMessage: 'Connection timeout', + retryCount: 1 + } + }); + expect(logEntry.metadata.stack).toBeDefined(); }); - test('should respect log level configuration', () => { - const logger = new Logger({ level: 'warn' }); + test('logger respects log level configuration', () => { + const logger = new Logger({ + level: 'warn', + logFile: testLogFile + }); + + // These should not log logger.debug('Debug message'); logger.info('Info message'); - logger.warn('Warn message'); - logger.error('Error message'); - expect(consoleSpy.log).not.toHaveBeenCalled(); - expect(consoleSpy.warn).toHaveBeenCalledWith(expect.stringContaining('[WARN]'), {}); - expect(consoleSpy.error).toHaveBeenCalledWith(expect.stringContaining('[ERROR]'), {}); - }); + // This should log + logger.warn('Warning message', { context: 'test' }); - test('should disable logging when enabled is false', () => { - const logger = new Logger({ enabled: false }); - logger.info('Test message'); + // Check console output expect(consoleSpy.log).not.toHaveBeenCalled(); + expect(consoleSpy.error).not.toHaveBeenCalled(); + expect(consoleSpy.warn).toHaveBeenCalledWith( + expect.stringContaining('[WARN]'), + { context: 'test' } + ); }); -}); \ No newline at end of file +});