From 816b548a1e9e2d9a135219974ef72c09c0642bf4 Mon Sep 17 00:00:00 2001 From: ghostcoder42 Date: Tue, 1 Sep 2026 17:40:07 +0800 Subject: [PATCH] fix(scraper): extract real artist slugs and parse current uploader markup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tapping an artist on the video page led to a model list without the current video — usually a 404 rendered as an error screen. Two defects: 1. The artist regex targeted the old markup (class="item btn_link" with the name span directly inside). The site now renders vote-chip pills (class="item btn_link video_meta_pill" with a wrap_image div before the span), so artist extraction never matched and the chip disappeared entirely. 2. Navigation derived the model slug from the display name (toLowerCase + dashes), which 404s for names like "OpenNSFW (VA)" whose real site slug is "opennsfw". parseVideoDetail now collects every model pill into an artists[] (name + slug taken from the site's own href), which the post page renders as one chip per artist. The sidebar's "Top Artists" links (class="item", no btn_link) are excluded, and each name is bound to its own anchor so adjacent pills can't bleed into each other. The uploader regex is tightened the same way (name bound to the member anchor, avatar alt first, plain text fallback for avatar-less members) instead of lazily scanning for the next alt attribute in the whole document. Artist chips promote a known role suffix from the model's display name to the label prefix — "OpenNSFW (VA)" renders as "VA: OpenNSFW", "HentAudio (Audio)" as "Audio: HentAudio"; plain names keep the "Artist: Name" prefix. Unknown parentheticals stay part of the name. --- __mocks__/fixtures/video-detail.html | 22 ++++++++++++- src/api/common/r34.test.ts | 5 ++- src/app/post/[id].tsx | 13 +++++--- src/lib/r34/artists.test.ts | 26 +++++++++++++++ src/lib/r34/artists.ts | 18 +++++++++++ src/lib/r34/offline-detail.test.ts | 3 +- src/lib/r34/offline-detail.ts | 3 +- src/lib/r34/scraper.test.ts | 40 ++++++++++++++++++++++- src/lib/r34/scraper.ts | 48 +++++++++++++++++++++------- src/lib/r34/types.ts | 9 +++++- 10 files changed, 165 insertions(+), 22 deletions(-) create mode 100644 src/lib/r34/artists.test.ts create mode 100644 src/lib/r34/artists.ts diff --git a/__mocks__/fixtures/video-detail.html b/__mocks__/fixtures/video-detail.html index 2b67914..a12b78a 100644 --- a/__mocks__/fixtures/video-detail.html +++ b/__mocks__/fixtures/video-detail.html @@ -16,8 +16,28 @@
TestUser
TestUser +
+ + + + +
Starfire
+ Starfire + + + + + + +
OpenNSFW (VA)
+ OpenNSFW (VA) + + + + +
1
+
Jackerman
 -Starfire test tag animation Test Category diff --git a/src/api/common/r34.test.ts b/src/api/common/r34.test.ts index 0439879..1d6b586 100644 --- a/src/api/common/r34.test.ts +++ b/src/api/common/r34.test.ts @@ -65,7 +65,10 @@ describe('r34Client', () => { expect(result.uploader).toBe('TestUser'); expect(result.uploaderMemberId).toBe('99999'); - expect(result.artist).toBe('Starfire'); + expect(result.artists).toEqual([ + { name: 'Starfire', slug: 'starfire' }, + { name: 'OpenNSFW (VA)', slug: 'opennsfw' }, + ]); expect(result.formats.length).toBeGreaterThan(0); }); }); diff --git a/src/app/post/[id].tsx b/src/app/post/[id].tsx index f073146..4b9fe86 100644 --- a/src/app/post/[id].tsx +++ b/src/app/post/[id].tsx @@ -9,6 +9,7 @@ import { useVideoDetail } from '@/api/video-queries'; import { ActivityIndicator, Button, FocusAwareStatusBar, Text } from '@/components/ui'; import { useVideoDownload } from '@/lib/hooks'; import { useTranslate } from '@/lib/i18n'; +import { artistChipLabel } from '@/lib/r34/artists'; import { toOfflineDetail } from '@/lib/r34/offline-detail'; import type { VideoDetail, VideoFormat } from '@/lib/r34/types'; import { baseIdOf, useDownloadedStore } from '@/lib/stores/downloaded-store'; @@ -312,23 +313,27 @@ export default function Post(): React.ReactElement | null { - {data.artist ? ( + {(data.artists ?? []).map((artist) => ( - Artist: {data.artist} + {artistChipLabel(artist.name)} - ) : null} + ))} {data.uploader && data.uploaderMemberId ? ( { + it('uses the generic Artist prefix for plain names', () => { + expect(artistChipLabel('Jackerman')).toBe('Artist: Jackerman'); + expect(artistChipLabel('General Butch')).toBe('Artist: General Butch'); + }); + + it('promotes known role suffixes to the prefix and strips them', () => { + expect(artistChipLabel('OpenNSFW (VA)')).toBe('VA: OpenNSFW'); + expect(artistChipLabel('MizzPeachy (VA)')).toBe('VA: MizzPeachy'); + expect(artistChipLabel('HentAudio (Audio)')).toBe('Audio: HentAudio'); + expect(artistChipLabel('Someone (Artist)')).toBe('Artist: Someone'); + }); + + it('keeps unknown parentheticals as part of the name', () => { + // Only VA/Audio/Artist are known roles — anything else stays verbatim so + // meaningful suffixes aren't silently dropped. + expect(artistChipLabel('Studio X (official)')).toBe('Artist: Studio X (official)'); + }); + + it('handles degenerate inputs without matching', () => { + expect(artistChipLabel('(VA)')).toBe('Artist: (VA)'); + expect(artistChipLabel('')).toBe('Artist: '); + }); +}); diff --git a/src/lib/r34/artists.ts b/src/lib/r34/artists.ts new file mode 100644 index 0000000..14d5eff --- /dev/null +++ b/src/lib/r34/artists.ts @@ -0,0 +1,18 @@ +/** + * Chip label for a site model (artist). The site appends a role to some model + * display names in parentheses — "(VA)" for voice actors, "(Audio)" for audio + * work — e.g. "OpenNSFW (VA)". Promote a known role to the label prefix and + * strip it from the name: "VA: OpenNSFW". Models without a known role keep + * the generic "Artist: Name" (with their name untouched, parenthetical + * included — it may carry meaning we don't want to silently drop). + */ +const ROLE_PREFIXES = new Set(['VA', 'Audio', 'Artist']); + +export function artistChipLabel(name: string): string { + const match = name.match(/^(.+?)\s*\(([^()]+)\)$/); + const role = match?.[2].trim(); + if (match && role && ROLE_PREFIXES.has(role)) { + return `${role}: ${match[1].trim()}`; + } + return `Artist: ${name}`; +} diff --git a/src/lib/r34/offline-detail.test.ts b/src/lib/r34/offline-detail.test.ts index 47c16a6..148f382 100644 --- a/src/lib/r34/offline-detail.test.ts +++ b/src/lib/r34/offline-detail.test.ts @@ -37,7 +37,8 @@ describe('toOfflineDetail', () => { expect(detail.tags).toEqual([]); expect(detail.categories).toEqual([]); - expect(detail.artist).toBeUndefined(); + expect(detail.artists).toEqual([]); + expect(detail.uploader).toBeUndefined(); expect(detail.description).toBeUndefined(); }); }); diff --git a/src/lib/r34/offline-detail.ts b/src/lib/r34/offline-detail.ts index 8cd2d65..3a89043 100644 --- a/src/lib/r34/offline-detail.ts +++ b/src/lib/r34/offline-detail.ts @@ -5,7 +5,7 @@ import type { VideoDetail } from './types'; /** * Build a minimal VideoDetail from a downloaded item so the post page can * render (and play the local file) with no network. Rich metadata - * (tags/categories/artist) is unavailable offline by design. + * (tags/categories/artists) is unavailable offline by design. */ export function toOfflineDetail(meta: DownloadMetadata, id: string): VideoDetail { return { @@ -19,5 +19,6 @@ export function toOfflineDetail(meta: DownloadMetadata, id: string): VideoDetail formats: [{ url: meta.uri, quality: meta.quality, ext: 'mp4' }], tags: [], categories: [], + artists: [], }; } diff --git a/src/lib/r34/scraper.test.ts b/src/lib/r34/scraper.test.ts index 49ad20e..f8a2532 100644 --- a/src/lib/r34/scraper.test.ts +++ b/src/lib/r34/scraper.test.ts @@ -98,11 +98,46 @@ describe('scraper', () => { expect(detail.tags).toContainEqual({ id: '67890', name: 'animation' }); expect(detail.uploader).toBe('TestUser'); expect(detail.uploaderMemberId).toBe('99999'); - expect(detail.artist).toBe('Starfire'); + expect(detail.artists).toEqual([ + { name: 'Starfire', slug: 'starfire' }, + { name: 'OpenNSFW (VA)', slug: 'opennsfw' }, + ]); expect(detail.categories).toContain('Test Category'); expect(detail.description).toBe('A test video description for Starfire.'); }); + it('does not treat sidebar "Top Artists" links as video artists', () => { + const html = loadFixture('video-detail.html'); + const detail = parseVideoDetail(html, '4406161', 'starfire-quick-hot-encounter'); + + // The fixture contains a sidebar-style link (class="item", no btn_link) + // to /models/jackerman/ that must not bleed into the artist list. + expect(detail.artists.some((a) => a.slug === 'jackerman')).toBe(false); + }); + + it('navigates artists by their real slug, not a name-derived one', () => { + const html = loadFixture('video-detail.html'); + const detail = parseVideoDetail(html, '4406161', 'starfire-quick-hot-encounter'); + + // "OpenNSFW (VA)" used to become the slug "opennsfw-(va)" (a 404 on the + // site); the real slug from the href is "opennsfw". + const derived = detail.artists[1].name.toLowerCase().replace(/\s+/g, '-'); + expect(derived).not.toBe(detail.artists[1].slug); + expect(detail.artists[1].slug).toBe('opennsfw'); + }); + + it('falls back to the anchor text when the uploader has no avatar', () => { + const html = ` + Minimal - Rule34Video +
Uploaded by
+ NoAvatarUser + `; + const detail = parseVideoDetail(html, '1', 'x'); + + expect(detail.uploaderMemberId).toBe('4242'); + expect(detail.uploader).toBe('NoAvatarUser'); + }); + it('does not expose removed uploaderUrl field', () => { const html = loadFixture('video-detail.html'); const detail = parseVideoDetail(html, '4406161', 'starfire-quick-hot-encounter') as Record< @@ -122,6 +157,9 @@ describe('scraper', () => { expect(detail.title).toBe('Minimal'); expect(detail.formats).toEqual([]); expect(detail.tags).toEqual([]); + expect(detail.artists).toEqual([]); + expect(detail.uploader).toBeUndefined(); + expect(detail.uploaderMemberId).toBeUndefined(); }); }); diff --git a/src/lib/r34/scraper.ts b/src/lib/r34/scraper.ts index 21adbff..cb9a4aa 100644 --- a/src/lib/r34/scraper.ts +++ b/src/lib/r34/scraper.ts @@ -159,19 +159,43 @@ export function parseVideoDetail(html: string, videoId: string, slug: string): V categories.push(match[1].trim()); } - // Extract uploader (a site member). Capture the member id from the profile - // link and the name from the avatar's alt attribute. + // Extract the uploader (a site member). Anchor on the "Uploaded by" label, + // then bind the name to the member anchor's own inner HTML — the avatar's + // alt text first, plain text (tags stripped) as a fallback for members + // without an avatar. This can't bleed into unrelated elements further down + // the page the way a lazy document-wide scan could. const uploaderMatch = html.match( - /Uploaded by<\/div>\s*]+href="(?https:\/\/rule34video\.com\/members\/(?\d+)\/)"[\s\S]*?alt="(?[^"]*)"/i + /Uploaded by<\/div>\s*]*href="https:\/\/rule34video\.com\/members\/(\d+)\/"[^>]*>([\s\S]*?)<\/a>/i ); - const uploader = uploaderMatch?.groups?.name?.trim(); - const uploaderMemberId = uploaderMatch?.groups?.id; - - // Extract artist - const artistMatch = html.match( - /]+class="item btn_link"[^>]+href="(?[^"]+)"[^>]*>\s*\s*(?[^<]+)\s*<\/span>\s*<\/a>/i - ); - const artist = artistMatch?.groups?.name?.trim(); + const uploaderMemberId = uploaderMatch?.[1]; + const uploaderInner = uploaderMatch?.[2] ?? ''; + const uploader = + uploaderInner.match(/alt="([^"]*)"/)?.[1]?.trim() || + uploaderInner + .replace(/<[^>]*>/g, ' ') + .replace(/\s+/g, ' ') + .trim() || + undefined; + + // Extract artists (site "models"). Each artist is a pill anchor (class + // contains "item btn_link") linking to /models/{slug}/ — the sidebar's "Top + // Artists" links (class="item", no btn_link) must not match. The slug comes + // from the href itself: deriving it from the display name breaks for names + // like "OpenNSFW (VA)", whose real site slug is "opennsfw". + // NOTE: positional capture groups (match[1..n]) — Hermes' matchAll does not + // populate `.groups` for named captures the way V8 does. + const artists: { name: string; slug: string }[] = []; + const artistRegex = + /]*class="[^"]*\bitem btn_link\b[^"]*"[^>]*href="https:\/\/rule34video\.com\/models\/([^\/"]+)\/"[^>]*>([\s\S]*?)<\/a>/g; + for (const match of html.matchAll(artistRegex)) { + const slug = match[1]; + if (artists.some((a) => a.slug === slug)) continue; + const name = + match[2].match(/\s*([^<]+?)\s*<\/span>/)?.[1]?.trim() || + match[2].match(/alt="([^"]*)"/)?.[1]?.trim() || + slug; + artists.push({ name, slug }); + } // Extract description const descriptionMatch = html.match(/