diff --git a/actors/apify_rag-web-browser/README.md b/actors/apify_rag-web-browser/README.md index 94d7551..06366db 100644 --- a/actors/apify_rag-web-browser/README.md +++ b/actors/apify_rag-web-browser/README.md @@ -63,7 +63,17 @@ the web page content directly like this: "url": "https://openai.com/index/introducing-chatgpt-search/", "title": "Introducing ChatGPT search | OpenAI", "description": "Get fast, timely answers with links to relevant web sources", - "languageCode": "en-US" + "languageCode": "en-US", + "canonicalUrl": "https://openai.com/index/introducing-chatgpt-search/", + "openGraph": [ + { "property": "og:title", "content": "Introducing ChatGPT search" }, + { "property": "og:description", "content": "Get fast, timely answers with links to relevant web sources" }, + { "property": "og:url", "content": "https://openai.com/index/introducing-chatgpt-search/" } + ], + "headers": { + "content-type": "text/html; charset=utf-8", + "cache-control": "public, max-age=0, must-revalidate" + } }, "markdown": "# Introducing ChatGPT search | OpenAI\n\nGet fast, timely answers with links to relevant web sources.\n\nChatGPT can now search the web in a much better way than before. ..." }] diff --git a/actors/apify_url-to-markdown/README.md b/actors/apify_url-to-markdown/README.md index 95bd910..b543e13 100644 --- a/actors/apify_url-to-markdown/README.md +++ b/actors/apify_url-to-markdown/README.md @@ -11,7 +11,7 @@ Additionally, you can select the **Scraping mode**: It will return an output with the following data: - URL - Markdown of the page -- Basic metadata +- Page metadata, including the canonical URL, description, keywords, Open Graph tags, JSON-LD structured data, and HTTP response headers This Actor doesn't support pagination or crawling to discover new URLs. If you are looking to convert a whole website to Markdown, use the [Website Content Crawler](https://apify.com/apify/website-content-crawler) instead. @@ -35,9 +35,23 @@ This Actor doesn't support pagination or crawling to discover new URLs. If you a "metadata": { "title": "Apify: Full-stack web scraping and data extraction platform", "description": "Cloud platform for web scraping, browser automation, AI agents, and data for AI. Use 38,000+ ready-made tools, code templates, or order a custom solution.", + "keywords": "web scraper,web crawler,scraping,data extraction,API", "languageCode": "en", "url": "https://apify.com", - "redirectedUrl": "https://apify.com/" + "redirectedUrl": "https://apify.com/", + "canonicalUrl": "https://apify.com/", + "openGraph": [ + { "property": "og:title", "content": "Apify: Full-stack web scraping and data extraction platform" }, + { "property": "og:url", "content": "https://apify.com/" }, + { "property": "og:site_name", "content": "Apify" } + ], + "jsonLd": [ + { "@context": "https://schema.org", "@type": "Organization", "name": "Apify" } + ], + "headers": { + "content-type": "text/html; charset=utf-8", + "cache-control": "public, max-age=0, must-revalidate" + } }, "query": "https://apify.com", "markdown": "Apify: Full-stack web scraping and data extraction platform\n\n" diff --git a/src/request-handler.ts b/src/request-handler.ts index 960cebe..4c033e9 100644 --- a/src/request-handler.ts +++ b/src/request-handler.ts @@ -1,3 +1,5 @@ +import type { IncomingHttpHeaders } from 'node:http'; + import type { PlaywrightBlocker } from '@ghostery/adblocker-playwright'; import { Actor } from 'apify'; import { load } from 'cheerio'; @@ -8,7 +10,13 @@ import { blockMediaRequests, SKIPPED_MEDIA_FILE_MESSAGE } from './media.js'; import { addResultToResponse, responseData, sendResponseIfFinished } from './responses.js'; import type { ContentCrawlerUserData, Output } from './types.js'; import { addTimeMeasureEvent, isActorStandby, transformTimeMeasuresToRelative } from './utils.js'; -import { extractTitle, processHtml } from './website-content-crawler/html-processing.js'; +import { + extractCanonicalUrl, + extractJsonLd, + extractOpenGraphProperties, + extractTitle, + processHtml, +} from './website-content-crawler/html-processing.js'; import { htmlToMarkdown } from './website-content-crawler/markdown.js'; let ACTOR_TIMEOUT_AT: number | undefined; @@ -76,6 +84,14 @@ function isValidContentType(contentType: string | undefined) { return ['text', 'html', 'xml'].some((type) => contentType?.includes(type)); } +/** Playwright exposes the headers through a method, but the context types also allow a plain object. */ +function getPlaywrightResponseHeaders(response: PlaywrightCrawlingContext['response']): IncomingHttpHeaders | undefined { + if (!response) return undefined; + + const { headers }: { headers: IncomingHttpHeaders | (() => IncomingHttpHeaders) } = response; + return typeof headers === 'function' ? response.headers() : headers; +} + /** * Stores an empty result for a page we haven't extracted any content from. */ @@ -127,6 +143,7 @@ async function handleContent( $: CheerioCrawlingContext['$'], crawlerType: ContentCrawlerTypes, statusCode: number | undefined, + headers: IncomingHttpHeaders | undefined, context: PlaywrightCrawlingContext | CheerioCrawlingContext, ) { const { request } = context; @@ -153,9 +170,14 @@ async function handleContent( author: $('meta[name=author]').first().attr('content') ?? undefined, title: extractTitle($), description: $('meta[name=description]').first().attr('content') ?? undefined, + keywords: $('meta[name=keywords]').first().attr('content') ?? undefined, languageCode: $html.first().attr('lang') ?? undefined, url: request.url, redirectedUrl: request.loadedUrl, + canonicalUrl: extractCanonicalUrl($, request.loadedUrl ?? request.url), + openGraph: extractOpenGraphProperties($), + jsonLd: extractJsonLd($), + headers, }, query: request.userData.query, text: settings.outputFormats.includes('text') ? text : undefined, @@ -231,14 +253,13 @@ export async function requestHandlerPlaywright( const $ = await context.parseWithCheerio(); addTimeMeasureEvent(request.userData, 'playwright-parse-with-cheerio'); - const headers = response?.headers instanceof Function ? response.headers() : response?.headers; + const headers = getPlaywrightResponseHeaders(response); const statusCode = response?.status(); - // @ts-expect-error false-positive? const isValidResponse = await checkValidResponse($, headers?.['content-type'], statusCode, context); if (!isValidResponse) return; - await handleContent($, ContentCrawlerTypes.PLAYWRIGHT, statusCode, context); + await handleContent($, ContentCrawlerTypes.PLAYWRIGHT, statusCode, headers, context); } export async function requestHandlerCheerio( @@ -263,7 +284,7 @@ export async function requestHandlerCheerio( const isValidResponse = await checkValidResponse($, response.headers['content-type'], statusCode, context); if (!isValidResponse) return; - await handleContent($, ContentCrawlerTypes.CHEERIO, statusCode, context); + await handleContent($, ContentCrawlerTypes.CHEERIO, statusCode, response.headers, context); } export async function failedRequestHandler(request: Request, err: Error, crawlerType: ContentCrawlerTypes) { diff --git a/src/types.ts b/src/types.ts index 2938e1f..d5c5863 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,3 +1,5 @@ +import type { IncomingHttpHeaders } from 'node:http'; + import type { ProxyConfigurationOptions } from 'apify'; import type { CheerioCrawlerOptions, PlaywrightCrawlerOptions } from 'crawlee'; @@ -45,6 +47,11 @@ export type Input = UrlToMarkdownInput | RagWebBrowserInput; export type SearchResultType = 'ORGANIC' | 'SUGGESTED'; +export type OpenGraphProperty = { + property: string; + content: string; +}; + export type OrganicResult = { description?: string; title?: string; @@ -141,9 +148,14 @@ export type Output = { title?: string | null; url: string; redirectedUrl?: string | null; + canonicalUrl?: string; description?: string | null; author?: string | null; + keywords?: string | null; languageCode?: string | null; + openGraph?: OpenGraphProperty[]; + jsonLd?: unknown[]; + headers?: IncomingHttpHeaders; }; }; diff --git a/src/website-content-crawler/html-processing.ts b/src/website-content-crawler/html-processing.ts index 626cb6d..9c5cf0e 100644 --- a/src/website-content-crawler/html-processing.ts +++ b/src/website-content-crawler/html-processing.ts @@ -1,7 +1,7 @@ import type { CheerioAPI } from 'crawlee'; import { log } from 'crawlee'; -import type { ContentScraperSettings } from '../types.js'; +import type { ContentScraperSettings, OpenGraphProperty } from '../types.js'; import { readableText } from './text-extractor.js'; const SKIP_CHILD_OF_ELEMENT_SELECTORS = ['.crawlee-iframe-replacement *', 'svg *'].join(', '); @@ -10,6 +10,9 @@ const TITLE_SELECTORS = [ `title:not(${SKIP_CHILD_OF_ELEMENT_SELECTORS})`, ]; +const OPEN_GRAPH_PREFIXES = ['og:', 'article:', 'book:', 'profile:', 'video:', 'website:', 'twitter:']; +const OPEN_GRAPH_SELECTOR = OPEN_GRAPH_PREFIXES.map((prefix) => `meta[property^="${prefix}"]`).join(', '); + /** * Extracts the page title (source: Website Content Crawler). * @@ -26,6 +29,47 @@ export function extractTitle($: CheerioAPI): string { return ''; } +export function extractCanonicalUrl($: CheerioAPI, baseUrl: string): string | undefined { + const href = $('html > head > link[rel="canonical"]').first().attr('href'); + if (!href) return undefined; + + try { + const url = new URL(href, baseUrl); + if (url.protocol === 'http:' || url.protocol === 'https:') return url.href; + } catch { + // Handled by the log below. + } + + log.debug(`Ignoring the canonical link of ${baseUrl}, which is not an HTTP(S) URL: ${href}`); + return undefined; +} + +export function extractOpenGraphProperties($: CheerioAPI): OpenGraphProperty[] | undefined { + const properties = $(OPEN_GRAPH_SELECTOR).get().flatMap((element) => { + const property = $(element).attr('property'); + const content = $(element).attr('content'); + return property && content ? [{ property, content }] : []; + }); + + return properties.length > 0 ? properties : undefined; +} + +/** + * Extracts the JSON-LD structured data of the page (source: Website Content Crawler). + */ +export function extractJsonLd($: CheerioAPI): unknown[] | undefined { + const items = $('script[type="application/ld+json"]').get().flatMap((element) => { + try { + return [JSON.parse($(element).text())]; + } catch { + log.debug('Skipping a JSON-LD script that does not contain valid JSON.'); + return []; + } + }); + + return items.length > 0 ? items : undefined; +} + /** * Process HTML with the selected HTML transformer (source: Website Content Crawler). */