chore: enrich page metadata

actors/apify_rag-web-browser/README.md

@@ -63,7 +63,17 @@ the web page content directly like this:
         "url": "https://openai.com/index/introducing-chatgpt-search/",
         "title": "Introducing ChatGPT search | OpenAI",
         "description": "Get fast, timely answers with links to relevant web sources",
-        "languageCode": "en-US"
+        "languageCode": "en-US",
+        "canonicalUrl": "https://openai.com/index/introducing-chatgpt-search/",
+        "openGraph": [
+            { "property": "og:title", "content": "Introducing ChatGPT search" },
+            { "property": "og:description", "content": "Get fast, timely answers with links to relevant web sources" },
+            { "property": "og:url", "content": "https://openai.com/index/introducing-chatgpt-search/" }
+        ],
+        "headers": {
+            "content-type": "text/html; charset=utf-8",
+            "cache-control": "public, max-age=0, must-revalidate"
+        }
     },
     "markdown": "# Introducing ChatGPT search | OpenAI\n\nGet fast, timely answers with links to relevant web sources.\n\nChatGPT can now search the web in a much better way than before. ..."
 }]

actors/apify_url-to-markdown/README.md

@@ -11,7 +11,7 @@ Additionally, you can select the **Scraping mode**:
 It will return an output with the following data:
 - URL
 - Markdown of the page
-- Basic metadata
+- Page metadata, including the canonical URL, description, keywords, Open Graph tags, JSON-LD structured data, and HTTP response headers
 
 This Actor doesn't support pagination or crawling to discover new URLs. If you are looking to convert a whole website to Markdown, use the [Website Content Crawler](https://apify.com/apify/website-content-crawler) instead.
 
@@ -35,9 +35,23 @@ This Actor doesn't support pagination or crawling to discover new URLs. If you a
   "metadata": {
     "title": "Apify: Full-stack web scraping and data extraction platform",
     "description": "Cloud platform for web scraping, browser automation, AI agents, and data for AI. Use 38,000+ ready-made tools, code templates, or order a custom solution.",
+    "keywords": "web scraper,web crawler,scraping,data extraction,API",
     "languageCode": "en",
     "url": "https://apify.com",
-    "redirectedUrl": "https://apify.com/"
+    "redirectedUrl": "https://apify.com/",
+    "canonicalUrl": "https://apify.com/",
+    "openGraph": [
+      { "property": "og:title", "content": "Apify: Full-stack web scraping and data extraction platform" },
+      { "property": "og:url", "content": "https://apify.com/" },
+      { "property": "og:site_name", "content": "Apify" }
+    ],
+    "jsonLd": [
+      { "@context": "https://schema.org", "@type": "Organization", "name": "Apify" }
+    ],
+    "headers": {
+      "content-type": "text/html; charset=utf-8",
+      "cache-control": "public, max-age=0, must-revalidate"
+    }
   },
   "query": "https://apify.com",
   "markdown": "Apify: Full-stack web scraping and data extraction platform\n\n"

src/request-handler.ts

@@ -1,3 +1,5 @@
+import type { IncomingHttpHeaders } from 'node:http';
+
 import type { PlaywrightBlocker } from '@ghostery/adblocker-playwright';
 import { Actor } from 'apify';
 import { load } from 'cheerio';
@@ -8,7 +10,13 @@ import { blockMediaRequests, SKIPPED_MEDIA_FILE_MESSAGE } from './media.js';
 import { addResultToResponse, responseData, sendResponseIfFinished } from './responses.js';
 import type { ContentCrawlerUserData, Output } from './types.js';
 import { addTimeMeasureEvent, isActorStandby, transformTimeMeasuresToRelative } from './utils.js';
-import { extractTitle, processHtml } from './website-content-crawler/html-processing.js';
+import {
+    extractCanonicalUrl,
+    extractJsonLd,
+    extractOpenGraphProperties,
+    extractTitle,
+    processHtml,
+} from './website-content-crawler/html-processing.js';
 import { htmlToMarkdown } from './website-content-crawler/markdown.js';
 
 let ACTOR_TIMEOUT_AT: number | undefined;
@@ -76,6 +84,14 @@ function isValidContentType(contentType: string | undefined) {
     return ['text', 'html', 'xml'].some((type) => contentType?.includes(type));
 }
 
+/** Playwright exposes the headers through a method, but the context types also allow a plain object. */
+function getPlaywrightResponseHeaders(response: PlaywrightCrawlingContext['response']): IncomingHttpHeaders | undefined {
+    if (!response) return undefined;
+
+    const { headers }: { headers: IncomingHttpHeaders | (() => IncomingHttpHeaders) } = response;
+    return typeof headers === 'function' ? response.headers() : headers;
+}
+
 /**
  * Stores an empty result for a page we haven't extracted any content from.
  */
@@ -127,6 +143,7 @@ async function handleContent(
     $: CheerioCrawlingContext['$'],
     crawlerType: ContentCrawlerTypes,
     statusCode: number | undefined,
+    headers: IncomingHttpHeaders | undefined,
     context: PlaywrightCrawlingContext<ContentCrawlerUserData> | CheerioCrawlingContext<ContentCrawlerUserData>,
 ) {
     const { request } = context;
@@ -153,9 +170,14 @@ async function handleContent(
             author: $('meta[name=author]').first().attr('content') ?? undefined,
             title: extractTitle($),
             description: $('meta[name=description]').first().attr('content') ?? undefined,
+            keywords: $('meta[name=keywords]').first().attr('content') ?? undefined,
             languageCode: $html.first().attr('lang') ?? undefined,
             url: request.url,
             redirectedUrl: request.loadedUrl,
+            canonicalUrl: extractCanonicalUrl($, request.loadedUrl ?? request.url),
+            openGraph: extractOpenGraphProperties($),
+            jsonLd: extractJsonLd($),
+            headers,
         },
         query: request.userData.query,
         text: settings.outputFormats.includes('text') ? text : undefined,
@@ -231,14 +253,13 @@ export async function requestHandlerPlaywright(
     const $ = await context.parseWithCheerio();
     addTimeMeasureEvent(request.userData, 'playwright-parse-with-cheerio');
 
-    const headers = response?.headers instanceof Function ? response.headers() : response?.headers;
+    const headers = getPlaywrightResponseHeaders(response);
     const statusCode = response?.status();
 
-    // @ts-expect-error false-positive?
     const isValidResponse = await checkValidResponse($, headers?.['content-type'], statusCode, context);
     if (!isValidResponse) return;
 
-    await handleContent($, ContentCrawlerTypes.PLAYWRIGHT, statusCode, context);
+    await handleContent($, ContentCrawlerTypes.PLAYWRIGHT, statusCode, headers, context);
 }
 
 export async function requestHandlerCheerio(
@@ -263,7 +284,7 @@ export async function requestHandlerCheerio(
     const isValidResponse = await checkValidResponse($, response.headers['content-type'], statusCode, context);
     if (!isValidResponse) return;
 
-    await handleContent($, ContentCrawlerTypes.CHEERIO, statusCode, context);
+    await handleContent($, ContentCrawlerTypes.CHEERIO, statusCode, response.headers, context);
 }
 
 export async function failedRequestHandler(request: Request, err: Error, crawlerType: ContentCrawlerTypes) {

src/types.ts

@@ -1,3 +1,5 @@
+import type { IncomingHttpHeaders } from 'node:http';
+
 import type { ProxyConfigurationOptions } from 'apify';
 import type { CheerioCrawlerOptions, PlaywrightCrawlerOptions } from 'crawlee';
 
@@ -45,6 +47,11 @@ export type Input = UrlToMarkdownInput | RagWebBrowserInput;
 
 export type SearchResultType = 'ORGANIC' | 'SUGGESTED';
 
+export type OpenGraphProperty = {
+    property: string;
+    content: string;
+};
+
 export type OrganicResult = {
     description?: string;
     title?: string;
@@ -141,9 +148,14 @@ export type Output = {
         title?: string | null;
         url: string;
         redirectedUrl?: string | null;
+        canonicalUrl?: string;
         description?: string | null;
         author?: string | null;
+        keywords?: string | null;
         languageCode?: string | null;
+        openGraph?: OpenGraphProperty[];
+        jsonLd?: unknown[];
+        headers?: IncomingHttpHeaders;
     };
 };
 

src/website-content-crawler/html-processing.ts

@@ -1,7 +1,7 @@
 import type { CheerioAPI } from 'crawlee';
 import { log } from 'crawlee';
 
-import type { ContentScraperSettings } from '../types.js';
+import type { ContentScraperSettings, OpenGraphProperty } from '../types.js';
 import { readableText } from './text-extractor.js';
 
 const SKIP_CHILD_OF_ELEMENT_SELECTORS = ['.crawlee-iframe-replacement *', 'svg *'].join(', ');
@@ -10,6 +10,9 @@ const TITLE_SELECTORS = [
     `title:not(${SKIP_CHILD_OF_ELEMENT_SELECTORS})`,
 ];
 
+const OPEN_GRAPH_PREFIXES = ['og:', 'article:', 'book:', 'profile:', 'video:', 'website:', 'twitter:'];
+const OPEN_GRAPH_SELECTOR = OPEN_GRAPH_PREFIXES.map((prefix) => `meta[property^="${prefix}"]`).join(', ');
+
 /**
  * Extracts the page title (source: Website Content Crawler).
  *
@@ -26,6 +29,47 @@ export function extractTitle($: CheerioAPI): string {
     return '';
 }
 
+export function extractCanonicalUrl($: CheerioAPI, baseUrl: string): string | undefined {
+    const href = $('html > head > link[rel="canonical"]').first().attr('href');
+    if (!href) return undefined;
+
+    try {
+        const url = new URL(href, baseUrl);
+        if (url.protocol === 'http:' || url.protocol === 'https:') return url.href;
+    } catch {
+        // Handled by the log below.
+    }
+
+    log.debug(`Ignoring the canonical link of ${baseUrl}, which is not an HTTP(S) URL: ${href}`);
+    return undefined;
+}
+
+export function extractOpenGraphProperties($: CheerioAPI): OpenGraphProperty[] | undefined {
+    const properties = $(OPEN_GRAPH_SELECTOR).get().flatMap((element) => {
+        const property = $(element).attr('property');
+        const content = $(element).attr('content');
+        return property && content ? [{ property, content }] : [];
+    });
+
+    return properties.length > 0 ? properties : undefined;
+}
+
+/**
+ * Extracts the JSON-LD structured data of the page (source: Website Content Crawler).
+ */
+export function extractJsonLd($: CheerioAPI): unknown[] | undefined {
+    const items = $('script[type="application/ld+json"]').get().flatMap((element) => {
+        try {
+            return [JSON.parse($(element).text())];
+        } catch {
+            log.debug('Skipping a JSON-LD script that does not contain valid JSON.');
+            return [];
+        }
+    });
+
+    return items.length > 0 ? items : undefined;
+}
+
 /**
  * Process HTML with the selected HTML transformer (source: Website Content Crawler).
  */