From 49c8353c57a1e7ea1e0801bcc7da2a6c753864e1 Mon Sep 17 00:00:00 2001 From: m-idler Date: Tue, 31 Oct 2023 11:44:47 +0100 Subject: [PATCH 1/4] make `User-Agent` configurable (#3253) --- CHANGELOG.md | 1 + js/server_functions.js | 25 +++++++++++++++++-- modules/default/calendar/calendarfetcher.js | 4 +-- modules/default/newsfeed/newsfeedfetcher.js | 4 +-- tests/unit/functions/server_functions_spec.js | 19 +++++++++++++- 5 files changed, 46 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7be7d3d059..df2032f19b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ _This release is scheduled to be released on 2024-01-01._ - Added node 21 to the test matrix - Added transform object to calendar:customEvents +- Added configuration option for `User-Agent`, used by calendar & news module ### Removed diff --git a/js/server_functions.js b/js/server_functions.js index 5693ad41c4..425fd38776 100644 --- a/js/server_functions.js +++ b/js/server_functions.js @@ -68,7 +68,7 @@ async function cors(req, res) { * @returns {object} An object specifying name and value of the headers. */ function getHeadersToSend(url) { - const headersToSend = { "User-Agent": `Mozilla/5.0 MagicMirror/${global.version}` }; + const headersToSend = { "User-Agent": getUserAgent() }; const headersToSendMatch = new RegExp("sendheaders=(.+?)(&|$)", "g").exec(url); if (headersToSendMatch) { const headers = headersToSendMatch[1].split(","); @@ -127,4 +127,25 @@ function getVersion(req, res) { res.send(global.version); } -module.exports = { cors, getConfig, getHtml, getVersion, getStartup }; +/** + * Gets the preferred `User-Agent` + * @returns {string} `User-Agent` to be used + */ +function getUserAgent() { + const defaultUserAgent = `Mozilla/5.0 (Node.js ${Number(process.version.match(/^v(\d+\.\d+)/)[1])}) MagicMirror/${global.version}`; + + if (typeof config === "undefined") { + return defaultUserAgent; + } + + switch (typeof config.userAgent) { + case "function": + return config.userAgent(); + case "string": + return config.userAgent; + default: + return defaultUserAgent; + } +} + +module.exports = { cors, getConfig, getHtml, getVersion, getStartup, getUserAgent }; diff --git a/modules/default/calendar/calendarfetcher.js b/modules/default/calendar/calendarfetcher.js index 51db30d7c7..23ad1c5a3e 100644 --- a/modules/default/calendar/calendarfetcher.js +++ b/modules/default/calendar/calendarfetcher.js @@ -9,6 +9,7 @@ const https = require("https"); const ical = require("node-ical"); const Log = require("logger"); const NodeHelper = require("node_helper"); +const { getUserAgent } = require("../../../js/server_functions"); const CalendarFetcherUtils = require("./calendarfetcherutils"); /** @@ -36,10 +37,9 @@ const CalendarFetcher = function (url, reloadInterval, excludedEvents, maximumEn const fetchCalendar = () => { clearTimeout(reloadTimer); reloadTimer = null; - const nodeVersion = Number(process.version.match(/^v(\d+\.\d+)/)[1]); let httpsAgent = null; let headers = { - "User-Agent": `Mozilla/5.0 (Node.js ${nodeVersion}) MagicMirror/${global.version}` + "User-Agent": getUserAgent() }; if (selfSignedCert) { diff --git a/modules/default/newsfeed/newsfeedfetcher.js b/modules/default/newsfeed/newsfeedfetcher.js index 0f1b5d6c6d..f4012c32bc 100644 --- a/modules/default/newsfeed/newsfeedfetcher.js +++ b/modules/default/newsfeed/newsfeedfetcher.js @@ -11,6 +11,7 @@ const iconv = require("iconv-lite"); const { htmlToText } = require("html-to-text"); const Log = require("logger"); const NodeHelper = require("node_helper"); +const { getUserAgent } = require("../../../js/server_functions"); /** * Responsible for requesting an update on the set interval and broadcasting the data. @@ -100,9 +101,8 @@ const NewsfeedFetcher = function (url, reloadInterval, encoding, logFeedWarnings } }); - const nodeVersion = Number(process.version.match(/^v(\d+\.\d+)/)[1]); const headers = { - "User-Agent": `Mozilla/5.0 (Node.js ${nodeVersion}) MagicMirror/${global.version}`, + "User-Agent": getUserAgent(), "Cache-Control": "max-age=0, no-cache, no-store, must-revalidate", Pragma: "no-cache" }; diff --git a/tests/unit/functions/server_functions_spec.js b/tests/unit/functions/server_functions_spec.js index 6242c9a9b5..54638ae16f 100644 --- a/tests/unit/functions/server_functions_spec.js +++ b/tests/unit/functions/server_functions_spec.js @@ -1,4 +1,5 @@ -const { cors } = require("../../../js/server_functions"); +const { expect } = require("playwright/test"); +const { cors, getUserAgent } = require("../../../js/server_functions"); describe("server_functions tests", () => { describe("The cors method", () => { @@ -142,5 +143,21 @@ describe("server_functions tests", () => { expect(corsResponse.set.mock.calls[2][0]).toBe("header2"); expect(corsResponse.set.mock.calls[2][1]).toBe("value2"); }); + + test("Gets User-Agent from configuration", async () => { + config = {}; + let userAgent; + + userAgent = getUserAgent(); + expect(userAgent).toContain("Mozilla/5.0 (Node.js "); + + config.userAgent = "Mozilla/5.0 (Foo)"; + userAgent = getUserAgent(); + expect(userAgent).toBe("Mozilla/5.0 (Foo)"); + + config.userAgent = () => "Mozilla/5.0 (Bar)"; + userAgent = getUserAgent(); + expect(userAgent).toBe("Mozilla/5.0 (Bar)"); + }); }); }); From 7c9aca6d818ab8026be173cd4116e42d971f8d34 Mon Sep 17 00:00:00 2001 From: veeck Date: Mon, 25 Aug 2025 15:10:02 +0200 Subject: [PATCH 2/4] eslint fixes --- js/server_functions.js | 2 +- tests/unit/functions/server_functions_spec.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/js/server_functions.js b/js/server_functions.js index 3f8df2d9dc..1f206ccd88 100644 --- a/js/server_functions.js +++ b/js/server_functions.js @@ -133,7 +133,7 @@ function getVersion (req, res) { * Gets the preferred `User-Agent` * @returns {string} `User-Agent` to be used */ -function getUserAgent() { +function getUserAgent () { const defaultUserAgent = `Mozilla/5.0 (Node.js ${Number(process.version.match(/^v(\d+\.\d+)/)[1])}) MagicMirror/${global.version}`; if (typeof config === "undefined") { diff --git a/tests/unit/functions/server_functions_spec.js b/tests/unit/functions/server_functions_spec.js index 85165c4a0d..01487ac84a 100644 --- a/tests/unit/functions/server_functions_spec.js +++ b/tests/unit/functions/server_functions_spec.js @@ -144,7 +144,7 @@ describe("server_functions tests", () => { expect(corsResponse.set.mock.calls[2][1]).toBe("value2"); }); - test("Gets User-Agent from configuration", async () => { + it("Gets User-Agent from configuration", async () => { config = {}; let userAgent; From 0bc14bb0f5d955563d2b329d5e87806c164a4bc5 Mon Sep 17 00:00:00 2001 From: Karsten Hassel Date: Tue, 26 Aug 2025 22:08:43 +0200 Subject: [PATCH 3/4] use server_functions as import --- js/app.js | 3 +-- js/server.js | 2 +- modules/default/calendar/calendarfetcher.js | 2 +- modules/default/newsfeed/newsfeedfetcher.js | 2 +- package.json | 3 +++ tests/unit/functions/server_functions_spec.js | 2 +- 6 files changed, 8 insertions(+), 6 deletions(-) diff --git a/js/app.js b/js/app.js index 22797195dc..5e7989aac1 100644 --- a/js/app.js +++ b/js/app.js @@ -9,10 +9,9 @@ const Log = require("logger"); const Server = require(`${__dirname}/server`); const Utils = require(`${__dirname}/utils`); const defaultModules = require(`${__dirname}/../modules/default/defaultmodules`); -const { getEnvVarsAsObj } = require(`${__dirname}/server_functions`); - // used to control fetch timeout for node_helpers const { setGlobalDispatcher, Agent } = require("undici"); +const { getEnvVarsAsObj } = require("#server_functions"); // common timeout value, provide environment override in case const fetch_timeout = process.env.mmFetchTimeout !== undefined ? process.env.mmFetchTimeout : 30000; diff --git a/js/server.js b/js/server.js index a15c811d23..95fd2b8247 100644 --- a/js/server.js +++ b/js/server.js @@ -7,7 +7,7 @@ const ipfilter = require("express-ipfilter").IpFilter; const helmet = require("helmet"); const socketio = require("socket.io"); const Log = require("logger"); -const { cors, getConfig, getHtml, getVersion, getStartup, getEnvVars } = require("./server_functions"); +const { cors, getConfig, getHtml, getVersion, getStartup, getEnvVars } = require("#server_functions"); const vendor = require(`${__dirname}/vendor`); diff --git a/modules/default/calendar/calendarfetcher.js b/modules/default/calendar/calendarfetcher.js index f67fb9c5a0..e9af5a1608 100644 --- a/modules/default/calendar/calendarfetcher.js +++ b/modules/default/calendar/calendarfetcher.js @@ -2,8 +2,8 @@ const https = require("node:https"); const ical = require("node-ical"); const Log = require("logger"); const NodeHelper = require("node_helper"); -const { getUserAgent } = require("../../../js/server_functions"); const CalendarFetcherUtils = require("./calendarfetcherutils"); +const { getUserAgent } = require("#server_functions"); const { scheduleTimer } = require("#module_functions"); /** diff --git a/modules/default/newsfeed/newsfeedfetcher.js b/modules/default/newsfeed/newsfeedfetcher.js index 43c42c6174..9ddebe52f1 100644 --- a/modules/default/newsfeed/newsfeedfetcher.js +++ b/modules/default/newsfeed/newsfeedfetcher.js @@ -5,7 +5,7 @@ const iconv = require("iconv-lite"); const { htmlToText } = require("html-to-text"); const Log = require("logger"); const NodeHelper = require("node_helper"); -const { getUserAgent } = require("../../../js/server_functions"); +const { getUserAgent } = require("#server_functions"); const { scheduleTimer } = require("#module_functions"); /** diff --git a/package.json b/package.json index a2e4002529..740da7968a 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,9 @@ "imports": { "#module_functions": { "default": "./js/module_functions.js" + }, + "#server_functions": { + "default": "./js/server_functions.js" } }, "main": "js/electron.js", diff --git a/tests/unit/functions/server_functions_spec.js b/tests/unit/functions/server_functions_spec.js index 01487ac84a..b7b6b06f69 100644 --- a/tests/unit/functions/server_functions_spec.js +++ b/tests/unit/functions/server_functions_spec.js @@ -1,5 +1,5 @@ const { expect } = require("playwright/test"); -const { cors, getUserAgent } = require("../../../js/server_functions"); +const { cors, getUserAgent } = require("#server_functions"); describe("server_functions tests", () => { describe("The cors method", () => { From 22831c9791f3bb1bd1e64c6557d71fbc0fd385eb Mon Sep 17 00:00:00 2001 From: Kristjan ESPERANTO <35647502+KristjanESPERANTO@users.noreply.github.com> Date: Wed, 27 Aug 2025 11:22:42 +0200 Subject: [PATCH 4/4] Add PR id --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8639fa68d1..27d004500b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ Thanks to: @dathbe. ### Added -- Added configuration option for `User-Agent`, used by calendar & news module +- Added configuration option for `User-Agent`, used by calendar & news module (#3255) ### Changed