diff --git a/lib/test/lib/util.ts b/lib/test/lib/util.ts index 1dd650ce..4ec866b4 100644 --- a/lib/test/lib/util.ts +++ b/lib/test/lib/util.ts @@ -29,6 +29,7 @@ const media_files = resource_file_mapper([ 'cat_cronch.mp4', 'blink.gif', 'music_snippet.mp3', + 'nyan_cat.webp', ] as const) if (!import.meta.dirname) throw new Error(`unexpected value in import.meta.dirname`) diff --git a/lib/test/resources/nyan_cat.webp b/lib/test/resources/nyan_cat.webp new file mode 100644 index 00000000..7a1d3fe1 Binary files /dev/null and b/lib/test/resources/nyan_cat.webp differ diff --git a/packages/core/src/lib/file_processor.ts b/packages/core/src/lib/file_processor.ts index cce63817..e6b71fe8 100644 --- a/packages/core/src/lib/file_processor.ts +++ b/packages/core/src/lib/file_processor.ts @@ -6,6 +6,7 @@ import z from 'zod' import { Context } from '~/context.ts' import { CODECS } from './codecs.ts' import * as errors from './errors.ts' +import { get_webp_file_info } from './file_processor_webp_shim.ts' interface FileInfoBase { @@ -243,6 +244,9 @@ class FileProcessor { const output = await cmd.output() if (!output.success) { + if (path.extname(this.#filepath) === '.webp') { + return get_webp_file_info(this.#filepath) + } const command = ffprobe_command.join(' ') const stdout = this.#decoder.decode(output.stdout) const stderr = this.#decoder.decode(output.stderr) @@ -341,7 +345,7 @@ class FileProcessor { if (media_type !== 'AUDIO' && width === 0 || height === 0) { if (path.extname(this.#filepath) === '.webp') { - throw new errors.InvalidFileError(`webp decoding is not yet supported by ffmpeg. See here https://trac.ffmpeg.org/ticket/4907`, new Error(`width: ${width} and height: ${height} values cannot be zero`)) + return get_webp_file_info(this.#filepath) } throw new errors.InvalidFileError(`Invalid file`, new Error(`width: ${width} and height: ${height} values cannot be zero`)) } @@ -392,6 +396,18 @@ class FileProcessor { // TODO make previews a supported field // assuming that 1/4 of the way into a video is a good preview position + // FFmpeg cannot decode animated WebP files, so we skip thumbnail generation. + // See https://trac.ffmpeg.org/ticket/4907 + if (file_info.codec === 'webp' && file_info.animated) { + const tmp_folder = await Deno.makeTempDir({prefix: 'forager-thumbnails-'}) + const thumbnail_destination_folder = path.join(this.#ctx.config.thumbnails.folder, this.get_storage_folder(checksum)) + return { + source_folder: tmp_folder, + destination_folder: thumbnail_destination_folder, + thumbnails: [], + } + } + const tmp_folder = await Deno.makeTempDir({prefix: 'forager-thumbnails-'}) const thumbnail_timestamps: number[] = [] const tmp_thumbnail_filepath = path.join(tmp_folder, `%0${this.#THUMBNAILS_FILENAME_ZERO_PAD_SIZE}d.jpg`) @@ -665,3 +681,4 @@ class FileProcessor { } export { FileProcessor } +export type { FileInfo } diff --git a/packages/core/src/lib/file_processor_webp_shim.ts b/packages/core/src/lib/file_processor_webp_shim.ts new file mode 100644 index 00000000..fea4bb46 --- /dev/null +++ b/packages/core/src/lib/file_processor_webp_shim.ts @@ -0,0 +1,293 @@ +/** + * Pure TypeScript parser for WebP files (RIFF-based container format). + * This is a shim for animated WebP support until FFmpeg's libwebp decoding + * is fully functional. See https://trac.ffmpeg.org/ticket/4907 + * + * Implements parsing based on the WebP Container Specification: + * https://developers.google.com/speed/webp/docs/riff_container + */ + +import * as path from '@std/path' +import { CODECS } from './codecs.ts' +import * as errors from './errors.ts' +import type { FileInfo } from './file_processor.ts' + + +interface WebPFrame { + x_offset: number + y_offset: number + width: number + height: number + duration_ms: number + blending: 'alpha' | 'no_alpha' + disposal: 'none' | 'background' +} + +interface WebPInfoBase { + canvas_width: number + canvas_height: number + has_alpha: boolean +} + +interface WebPStillInfo extends WebPInfoBase { + animated: false +} + +interface WebPAnimatedInfo extends WebPInfoBase { + animated: true + loop_count: number + background_color: { r: number; g: number; b: number; a: number } + frames: WebPFrame[] + total_duration_ms: number + framecount: number +} + +type WebPInfo = WebPStillInfo | WebPAnimatedInfo + + +class WebPParseError extends errors.FileProcessingError { + override name = 'WebPParseError' + constructor(message: string) { + super(message) + } +} + + +function read_uint24_le(data: DataView, offset: number): number { + return data.getUint8(offset) | (data.getUint8(offset + 1) << 8) | (data.getUint8(offset + 2) << 16) +} + + +function ascii(data: DataView, offset: number, length: number): string { + const bytes = new Uint8Array(data.buffer, data.byteOffset + offset, length) + return String.fromCharCode(...bytes) +} + + +/** + * Parse a VP8 (lossy) bitstream header to extract width and height. + * The VP8 bitstream begins with a 3-byte frame tag, followed (for keyframes) + * by a 3-byte start code (0x9D 0x01 0x2A) and then width/height. + */ +function parse_vp8_dimensions(data: DataView, offset: number, size: number): { width: number; height: number } { + if (size < 10) { + throw new WebPParseError(`VP8 chunk too small to contain dimensions (size: ${size})`) + } + + const frame_tag_byte0 = data.getUint8(offset) + const is_keyframe = (frame_tag_byte0 & 0x01) === 0 + + if (!is_keyframe) { + throw new WebPParseError('VP8 chunk is not a keyframe, cannot extract dimensions') + } + + const start_code_0 = data.getUint8(offset + 3) + const start_code_1 = data.getUint8(offset + 4) + const start_code_2 = data.getUint8(offset + 5) + if (start_code_0 !== 0x9D || start_code_1 !== 0x01 || start_code_2 !== 0x2A) { + throw new WebPParseError(`Invalid VP8 start code: 0x${start_code_0.toString(16)} 0x${start_code_1.toString(16)} 0x${start_code_2.toString(16)}`) + } + + const width = data.getUint16(offset + 6, true) & 0x3FFF + const height = data.getUint16(offset + 8, true) & 0x3FFF + + return { width, height } +} + + +/** + * Parse a VP8L (lossless) bitstream header to extract width and height. + * The VP8L bitstream starts with a 1-byte signature (0x2F), followed by + * a 4-byte packed field containing width-1 (14 bits) and height-1 (14 bits). + */ +function parse_vp8l_dimensions(data: DataView, offset: number, size: number): { width: number; height: number } { + if (size < 5) { + throw new WebPParseError(`VP8L chunk too small to contain dimensions (size: ${size})`) + } + + const signature = data.getUint8(offset) + if (signature !== 0x2F) { + throw new WebPParseError(`Invalid VP8L signature: 0x${signature.toString(16)}`) + } + + const bits = data.getUint32(offset + 1, true) + const width = (bits & 0x3FFF) + 1 + const height = ((bits >> 14) & 0x3FFF) + 1 + + return { width, height } +} + + +/** + * Parse a WebP file buffer and extract metadata including animation info. + */ +export function parse_webp(buffer: ArrayBuffer): WebPInfo { + const data = new DataView(buffer) + + if (buffer.byteLength < 12) { + throw new WebPParseError(`File too small to be a valid WebP (${buffer.byteLength} bytes)`) + } + + const riff_tag = ascii(data, 0, 4) + if (riff_tag !== 'RIFF') { + throw new WebPParseError(`Missing RIFF tag, found "${riff_tag}"`) + } + + const file_size = data.getUint32(4, true) + 8 + const webp_tag = ascii(data, 8, 4) + if (webp_tag !== 'WEBP') { + throw new WebPParseError(`Missing WEBP tag, found "${webp_tag}"`) + } + + let canvas_width = 0 + let canvas_height = 0 + let has_alpha = false + let is_animated = false + let loop_count = 0 + let background_color = { r: 0, g: 0, b: 0, a: 0 } + const frames: WebPFrame[] = [] + let found_vp8x = false + + let pos = 12 + const end = Math.min(file_size, buffer.byteLength) + + while (pos + 8 <= end) { + const chunk_fourcc = ascii(data, pos, 4) + const chunk_size = data.getUint32(pos + 4, true) + const chunk_data_offset = pos + 8 + const padded_size = chunk_size + (chunk_size & 1) + + if (chunk_data_offset + chunk_size > end) { + break + } + + switch (chunk_fourcc) { + case 'VP8X': { + if (chunk_size < 10) { + throw new WebPParseError(`VP8X chunk too small (${chunk_size} bytes)`) + } + found_vp8x = true + const flags = data.getUint8(chunk_data_offset) + is_animated = (flags & 0x02) !== 0 + has_alpha = (flags & 0x10) !== 0 + + canvas_width = read_uint24_le(data, chunk_data_offset + 4) + 1 + canvas_height = read_uint24_le(data, chunk_data_offset + 7) + 1 + break + } + + case 'ANIM': { + if (chunk_size < 6) { + throw new WebPParseError(`ANIM chunk too small (${chunk_size} bytes)`) + } + background_color = { + b: data.getUint8(chunk_data_offset), + g: data.getUint8(chunk_data_offset + 1), + r: data.getUint8(chunk_data_offset + 2), + a: data.getUint8(chunk_data_offset + 3), + } + loop_count = data.getUint16(chunk_data_offset + 4, true) + break + } + + case 'ANMF': { + if (chunk_size < 16) { + throw new WebPParseError(`ANMF chunk too small (${chunk_size} bytes)`) + } + const frame_x = read_uint24_le(data, chunk_data_offset) * 2 + const frame_y = read_uint24_le(data, chunk_data_offset + 3) * 2 + const frame_width = read_uint24_le(data, chunk_data_offset + 6) + 1 + const frame_height = read_uint24_le(data, chunk_data_offset + 9) + 1 + const duration_ms = read_uint24_le(data, chunk_data_offset + 12) + const frame_flags = data.getUint8(chunk_data_offset + 15) + const blending = (frame_flags & 0x02) !== 0 ? 'no_alpha' as const : 'alpha' as const + const disposal = (frame_flags & 0x01) !== 0 ? 'background' as const : 'none' as const + + frames.push({ x_offset: frame_x, y_offset: frame_y, width: frame_width, height: frame_height, duration_ms, blending, disposal }) + break + } + + case 'VP8 ': { + if (!found_vp8x) { + const dims = parse_vp8_dimensions(data, chunk_data_offset, chunk_size) + canvas_width = dims.width + canvas_height = dims.height + } + break + } + + case 'VP8L': { + if (!found_vp8x) { + const dims = parse_vp8l_dimensions(data, chunk_data_offset, chunk_size) + canvas_width = dims.width + canvas_height = dims.height + has_alpha = true + } + break + } + } + + pos = chunk_data_offset + padded_size + } + + if (canvas_width === 0 || canvas_height === 0) { + throw new WebPParseError(`Could not determine WebP dimensions (width: ${canvas_width}, height: ${canvas_height})`) + } + + if (is_animated) { + if (frames.length === 0) { + throw new WebPParseError('VP8X indicates animation but no ANMF frames found') + } + const total_duration_ms = frames.reduce((sum, f) => sum + f.duration_ms, 0) + return { + canvas_width, + canvas_height, + has_alpha, + animated: true, + loop_count, + background_color, + frames, + total_duration_ms, + framecount: frames.length, + } + } + + return { + canvas_width, + canvas_height, + has_alpha, + animated: false, + } +} + + +/** + * Parse a WebP file from disk and return FileInfo compatible with FileProcessor. + * This is the primary entry point used by FileProcessor as a fallback when + * ffprobe cannot handle WebP files. + */ +export async function get_webp_file_info(filepath: string): Promise { + const resolved_filepath = path.resolve(filepath) + const buffer = await Deno.readFile(resolved_filepath) + const webp_info = parse_webp(buffer.buffer as ArrayBuffer) + + const filename = path.basename(resolved_filepath) + const codec_info = CODECS.get_codec('webp') + + const duration = webp_info.animated ? webp_info.total_duration_ms / 1000 : 0 + const framecount = webp_info.animated ? webp_info.framecount : 0 + const framerate = webp_info.animated && duration > 0 ? framecount / duration : 0 + + return { + filepath: resolved_filepath, + filename, + ...codec_info, + width: webp_info.canvas_width, + height: webp_info.canvas_height, + animated: webp_info.animated, + duration, + framerate, + framecount, + audio: false, + } as FileInfo +} diff --git a/packages/core/test/media.test.ts b/packages/core/test/media.test.ts index eb4aa188..9efb68f6 100644 --- a/packages/core/test/media.test.ts +++ b/packages/core/test/media.test.ts @@ -1213,6 +1213,25 @@ test('video media', async ctx => { }) }) +test('animated webp media', async ctx => { + using forager = new Forager(ctx.get_test_config()) + forager.init() + + const media_nyan = await forager.media.create(ctx.resources.media_files['nyan_cat.webp'], {title: 'Nyan Cat'}, ['meme']) + ctx.assert.equals(media_nyan.media_file.media_type, 'IMAGE') + ctx.assert.equals(media_nyan.media_file.content_type, 'image/webp') + ctx.assert.equals(media_nyan.media_file.codec, 'webp') + ctx.assert.equals(media_nyan.media_file.width, 400) + ctx.assert.equals(media_nyan.media_file.height, 400) + ctx.assert.equals(media_nyan.media_file.animated, true) + ctx.assert.equals(media_nyan.media_file.duration, 0.84) + ctx.assert.equals(media_nyan.media_file.framecount, 12) + ctx.assert.equals(media_nyan.media_file.audio, false) + ctx.assert.equals(media_nyan.media_file.filepath, ctx.resources.media_files['nyan_cat.webp']) + // ffmpeg cannot generate thumbnails for animated webp + ctx.assert.equals(media_nyan.thumbnails.total, 0) +}) + test('search query.animated', async ctx => { using forager = new Forager(ctx.get_test_config()) forager.init() diff --git a/packages/core/test/webp_parser.test.ts b/packages/core/test/webp_parser.test.ts new file mode 100644 index 00000000..429176db --- /dev/null +++ b/packages/core/test/webp_parser.test.ts @@ -0,0 +1,249 @@ +import { test } from 'forager-test' +import { parse_webp } from '~/lib/file_processor_webp_shim.ts' + + +function build_riff_webp(chunks: Uint8Array): ArrayBuffer { + const riff_header = new Uint8Array(12) + const view = new DataView(riff_header.buffer) + riff_header.set(new TextEncoder().encode('RIFF'), 0) + view.setUint32(4, chunks.byteLength + 4, true) + riff_header.set(new TextEncoder().encode('WEBP'), 8) + + const result = new Uint8Array(riff_header.byteLength + chunks.byteLength) + result.set(riff_header, 0) + result.set(chunks, 12) + return result.buffer as ArrayBuffer +} + + +function build_chunk(fourcc: string, payload: Uint8Array): Uint8Array { + const header = new Uint8Array(8) + const view = new DataView(header.buffer) + header.set(new TextEncoder().encode(fourcc), 0) + view.setUint32(4, payload.byteLength, true) + + const padded_size = payload.byteLength + (payload.byteLength & 1) + const result = new Uint8Array(8 + padded_size) + result.set(header, 0) + result.set(payload, 8) + return result +} + + +function build_vp8x_chunk(opts: { animated?: boolean; alpha?: boolean; width: number; height: number }): Uint8Array { + const payload = new Uint8Array(10) + const view = new DataView(payload.buffer) + let flags = 0 + if (opts.animated) flags |= 0x02 + if (opts.alpha) flags |= 0x10 + view.setUint8(0, flags) + + const w = opts.width - 1 + const h = opts.height - 1 + payload[4] = w & 0xFF + payload[5] = (w >> 8) & 0xFF + payload[6] = (w >> 16) & 0xFF + payload[7] = h & 0xFF + payload[8] = (h >> 8) & 0xFF + payload[9] = (h >> 16) & 0xFF + + return build_chunk('VP8X', payload) +} + + +function build_anim_chunk(loop_count: number): Uint8Array { + const payload = new Uint8Array(6) + const view = new DataView(payload.buffer) + view.setUint16(4, loop_count, true) + return build_chunk('ANIM', payload) +} + + +function build_anmf_chunk(opts: { width: number; height: number; duration_ms: number }): Uint8Array { + const min_payload_size = 16 + const dummy_frame_data = new Uint8Array(8) + const payload = new Uint8Array(min_payload_size + dummy_frame_data.byteLength) + + const w = opts.width - 1 + const h = opts.height - 1 + payload[6] = w & 0xFF + payload[7] = (w >> 8) & 0xFF + payload[8] = (w >> 16) & 0xFF + payload[9] = h & 0xFF + payload[10] = (h >> 8) & 0xFF + payload[11] = (h >> 16) & 0xFF + payload[12] = opts.duration_ms & 0xFF + payload[13] = (opts.duration_ms >> 8) & 0xFF + payload[14] = (opts.duration_ms >> 16) & 0xFF + + payload.set(dummy_frame_data, min_payload_size) + return build_chunk('ANMF', payload) +} + + +function concat_uint8(...arrays: Uint8Array[]): Uint8Array { + const total = arrays.reduce((sum, a) => sum + a.byteLength, 0) + const result = new Uint8Array(total) + let offset = 0 + for (const a of arrays) { + result.set(a, offset) + offset += a.byteLength + } + return result +} + + +test('webp_parser - synthetic buffers', async (ctx) => { + await ctx.subtest('parse non-animated VP8X webp', () => { + const vp8x = build_vp8x_chunk({ width: 800, height: 600 }) + const buffer = build_riff_webp(vp8x) + const info = parse_webp(buffer) + + ctx.assert.equals(info.canvas_width, 800) + ctx.assert.equals(info.canvas_height, 600) + ctx.assert.equals(info.animated, false) + ctx.assert.equals(info.has_alpha, false) + }) + + await ctx.subtest('parse non-animated VP8X with alpha', () => { + const vp8x = build_vp8x_chunk({ width: 1920, height: 1080, alpha: true }) + const buffer = build_riff_webp(vp8x) + const info = parse_webp(buffer) + + ctx.assert.equals(info.canvas_width, 1920) + ctx.assert.equals(info.canvas_height, 1080) + ctx.assert.equals(info.animated, false) + ctx.assert.equals(info.has_alpha, true) + }) + + await ctx.subtest('parse animated webp', () => { + const vp8x = build_vp8x_chunk({ animated: true, width: 320, height: 240 }) + const anim = build_anim_chunk(0) + const frame1 = build_anmf_chunk({ width: 320, height: 240, duration_ms: 100 }) + const frame2 = build_anmf_chunk({ width: 320, height: 240, duration_ms: 100 }) + const frame3 = build_anmf_chunk({ width: 320, height: 240, duration_ms: 100 }) + + const chunks = concat_uint8(vp8x, anim, frame1, frame2, frame3) + const buffer = build_riff_webp(chunks) + const info = parse_webp(buffer) + + ctx.assert.equals(info.animated, true) + if (!info.animated) throw new Error('unreachable') + ctx.assert.equals(info.canvas_width, 320) + ctx.assert.equals(info.canvas_height, 240) + ctx.assert.equals(info.framecount, 3) + ctx.assert.equals(info.total_duration_ms, 300) + ctx.assert.equals(info.loop_count, 0) + }) + + await ctx.subtest('parse animated webp with varying frame durations', () => { + const vp8x = build_vp8x_chunk({ animated: true, width: 640, height: 480 }) + const anim = build_anim_chunk(5) + const frame1 = build_anmf_chunk({ width: 640, height: 480, duration_ms: 50 }) + const frame2 = build_anmf_chunk({ width: 640, height: 480, duration_ms: 200 }) + const frame3 = build_anmf_chunk({ width: 640, height: 480, duration_ms: 150 }) + const frame4 = build_anmf_chunk({ width: 640, height: 480, duration_ms: 100 }) + + const chunks = concat_uint8(vp8x, anim, frame1, frame2, frame3, frame4) + const buffer = build_riff_webp(chunks) + const info = parse_webp(buffer) + + ctx.assert.equals(info.animated, true) + if (!info.animated) throw new Error('unreachable') + ctx.assert.equals(info.framecount, 4) + ctx.assert.equals(info.total_duration_ms, 500) + ctx.assert.equals(info.loop_count, 5) + }) + + await ctx.subtest('rejects non-RIFF data', () => { + const garbage = new Uint8Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) + ctx.assert.throws(() => parse_webp(garbage.buffer as ArrayBuffer), Error, 'Missing RIFF tag') + }) + + await ctx.subtest('rejects file too small', () => { + const tiny = new Uint8Array(4) + ctx.assert.throws(() => parse_webp(tiny.buffer as ArrayBuffer), Error, 'too small') + }) + + await ctx.subtest('rejects RIFF without WEBP tag', () => { + const data = new Uint8Array(12) + const view = new DataView(data.buffer) + data.set(new TextEncoder().encode('RIFF'), 0) + view.setUint32(4, 4, true) + data.set(new TextEncoder().encode('AVI '), 8) + ctx.assert.throws(() => parse_webp(data.buffer as ArrayBuffer), Error, 'Missing WEBP tag') + }) + + await ctx.subtest('VP8X animated flag set but no ANMF frames', () => { + const vp8x = build_vp8x_chunk({ animated: true, width: 100, height: 100 }) + const buffer = build_riff_webp(vp8x) + ctx.assert.throws(() => parse_webp(buffer), Error, 'no ANMF frames found') + }) +}) + + +test('webp_parser - VP8 lossy still image', async (ctx) => { + await ctx.subtest('parse VP8 dimensions', () => { + const frame_tag = new Uint8Array(3) + frame_tag[0] = 0x00 + + const start_code = new Uint8Array([0x9D, 0x01, 0x2A]) + const dims = new Uint8Array(4) + const dims_view = new DataView(dims.buffer) + dims_view.setUint16(0, 400, true) + dims_view.setUint16(2, 300, true) + + const padding = new Uint8Array(3) + const vp8_payload = concat_uint8(frame_tag, start_code, dims, padding) + const vp8_chunk = build_chunk('VP8 ', vp8_payload) + const buffer = build_riff_webp(vp8_chunk) + const info = parse_webp(buffer) + + ctx.assert.equals(info.canvas_width, 400) + ctx.assert.equals(info.canvas_height, 300) + ctx.assert.equals(info.animated, false) + }) +}) + + +test('webp_parser - VP8L lossless still image', async (ctx) => { + await ctx.subtest('parse VP8L dimensions', () => { + const payload = new Uint8Array(5) + const view = new DataView(payload.buffer) + payload[0] = 0x2F + + const width = 512 + const height = 256 + const bits = ((width - 1) & 0x3FFF) | (((height - 1) & 0x3FFF) << 14) + view.setUint32(1, bits, true) + + const vp8l_chunk = build_chunk('VP8L', payload) + const buffer = build_riff_webp(vp8l_chunk) + const info = parse_webp(buffer) + + ctx.assert.equals(info.canvas_width, 512) + ctx.assert.equals(info.canvas_height, 256) + ctx.assert.equals(info.animated, false) + ctx.assert.equals(info.has_alpha, true) + }) +}) + + +test('webp_parser - animated webp file from disk', async (ctx) => { + await ctx.subtest('parse_webp extracts correct metadata', async () => { + const buffer = await Deno.readFile(ctx.resources.media_files['nyan_cat.webp']) + const info = parse_webp(buffer.buffer as ArrayBuffer) + + ctx.assert.equals(info.animated, true) + if (!info.animated) throw new Error('unreachable') + ctx.assert.equals(info.canvas_width, 400) + ctx.assert.equals(info.canvas_height, 400) + ctx.assert.equals(info.has_alpha, true) + ctx.assert.equals(info.framecount, 12) + ctx.assert.equals(info.total_duration_ms, 840) + ctx.assert.equals(info.loop_count, 0) + ctx.assert.equals(info.frames.length, 12) + ctx.assert.equals(info.frames[0].duration_ms, 70) + }) + +})