diff --git a/src/request-handler.ts b/src/request-handler.ts
index faf440b..250251e 100644
--- a/src/request-handler.ts
+++ b/src/request-handler.ts
@@ -7,7 +7,7 @@ import { ContentCrawlerStatus, ContentCrawlerTypes } from './const.js';
import { addResultToResponse, responseData, sendResponseIfFinished } from './responses.js';
import type { ContentCrawlerUserData, Output } from './types.js';
import { addTimeMeasureEvent, isActorStandby, transformTimeMeasuresToRelative } from './utils.js';
-import { processHtml } from './website-content-crawler/html-processing.js';
+import { extractTitle, processHtml } from './website-content-crawler/html-processing.js';
import { htmlToMarkdown } from './website-content-crawler/markdown.js';
let ACTOR_TIMEOUT_AT: number | undefined;
@@ -136,7 +136,7 @@ async function handleContent(
searchResult: request.userData.searchResult!,
metadata: {
author: $('meta[name=author]').first().attr('content') ?? undefined,
- title: $('title').first().text(),
+ title: extractTitle($),
description: $('meta[name=description]').first().attr('content') ?? undefined,
languageCode: $html.first().attr('lang') ?? undefined,
url: request.url,
diff --git a/src/website-content-crawler/html-processing.ts b/src/website-content-crawler/html-processing.ts
index eadfe42..626cb6d 100644
--- a/src/website-content-crawler/html-processing.ts
+++ b/src/website-content-crawler/html-processing.ts
@@ -4,6 +4,28 @@ import { log } from 'crawlee';
import type { ContentScraperSettings } from '../types.js';
import { readableText } from './text-extractor.js';
+const SKIP_CHILD_OF_ELEMENT_SELECTORS = ['.crawlee-iframe-replacement *', 'svg *'].join(', ');
+const TITLE_SELECTORS = [
+ `head > title:not(${SKIP_CHILD_OF_ELEMENT_SELECTORS})`,
+ `title:not(${SKIP_CHILD_OF_ELEMENT_SELECTORS})`,
+];
+
+/**
+ * Extracts the page title (source: Website Content Crawler).
+ *
+ * Prefers the `` in `<head>` and ignores `<title>` elements nested in SVGs
+ * (used there as tooltips) or in Crawlee iframe replacement nodes.
+ */
+export function extractTitle($: CheerioAPI): string {
+ for (const selector of TITLE_SELECTORS) {
+ const title = $(selector).first().text().trim();
+ if (title) {
+ return title;
+ }
+ }
+ return '';
+}
+
/**
* Process HTML with the selected HTML transformer (source: Website Content Crawler).
*/
@@ -18,12 +40,13 @@ export async function processHtml(
$body.find(settings.removeElementsCssSelector).remove();
}
const simplifiedBody = $body.html()?.trim();
+ const title = extractTitle($);
const simplified = typeof simplifiedBody === 'string'
? `<html lang="">
<head>
<title>
- ${$('title').text()}
+ ${title}
diff --git a/tests/cheerio-crawler.content.test.ts b/tests/cheerio-crawler.content.test.ts
index b47706e..484bb60 100644
--- a/tests/cheerio-crawler.content.test.ts
+++ b/tests/cheerio-crawler.content.test.ts
@@ -42,6 +42,9 @@ describe('Cheerio Crawler Content Tests', () => {
expect(pushDataSpy).toHaveBeenCalledTimes(1);
expect(pushDataSpy).toHaveBeenCalledWith(expect.objectContaining({
text: expect.stringContaining('hello world'),
+ metadata: expect.objectContaining({
+ title: 'Test Page',
+ }),
}));
successUrls.add(context.request.url);
},
diff --git a/tests/html-processing.test.ts b/tests/html-processing.test.ts
new file mode 100644
index 0000000..bca408b
--- /dev/null
+++ b/tests/html-processing.test.ts
@@ -0,0 +1,47 @@
+import { load } from 'cheerio';
+import type { CheerioAPI } from 'crawlee';
+import { describe, expect, it } from 'vitest';
+
+import { extractTitle } from '../src/website-content-crawler/html-processing.js';
+
+// The `cheerio` version bundled with Crawlee differs from the top-level one, so the types don't match.
+const parse = (html: string) => load(html) as unknown as CheerioAPI;
+
+describe('extractTitle', () => {
+ it('should extract the title from somewhere else if not in head', () => {
+ const $ = parse(`
+
+
+
The part with the content.
+ Title in body
+
+ `);
+ expect(extractTitle($)).toBe('Title in body');
+ });
+
+ it('should ignore titles in SVGs anywhere in the html', () => {
+ const $ = parse(`
+
+
+
The part with the content.
+
+
+ `);
+ expect(extractTitle($)).toBe('');
+ });
+
+ it('should ignore titles in .crawlee-iframe-replacement anywhere in the html', () => {
+ const $ = parse(`
+
Title in head
+
+
Title in .crawlee-iframe-replacement
+
+ `);
+ expect(extractTitle($)).toBe('');
+ });
+
+ it('should trim surrounding whitespace', () => {
+ const $ = parse('\n Test Title \n');
+ expect(extractTitle($)).toBe('Test Title');
+ });
+});