diff --git a/app/client/package-lock.json b/app/client/package-lock.json index 7b825519..7af010cb 100644 --- a/app/client/package-lock.json +++ b/app/client/package-lock.json @@ -1,12 +1,12 @@ { "name": "fireshare", - "version": "1.8.3", + "version": "1.8.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "fireshare", - "version": "1.8.3", + "version": "1.8.4", "dependencies": { "@emotion/react": "^11.9.0", "@emotion/styled": "^11.8.1", diff --git a/app/client/package.json b/app/client/package.json index 5cd0f99c..66dfd47d 100644 --- a/app/client/package.json +++ b/app/client/package.json @@ -1,6 +1,6 @@ { "name": "fireshare", - "version": "1.8.3", + "version": "1.8.4", "private": true, "dependencies": { "@emotion/react": "^11.9.0", diff --git a/app/client/src/common/utils.js b/app/client/src/common/utils.js index dd716cd9..e844d2b2 100644 --- a/app/client/src/common/utils.js +++ b/app/client/src/common/utils.js @@ -215,9 +215,62 @@ export const getImageUrl = (imageId) => { return `${baseUrl}/api/image?id=${imageId}` } +// Transcodes are H.264 unless an admin has switched the encoder to AV1, and nothing +// records which one a given transcode used, so they are described as H.264. The +// description is only used to ask whether the device decodes that kind of file in +// hardware. +const TRANSCODE_CODEC = 'avc1.640033' + +// Rough bits per pixel per frame, for a file that does not record its bitrate. The +// browser decides on codec, size and frame rate; the bitrate only has to be plausible. +const ESTIMATED_BITS_PER_PIXEL = 0.1 + +const describeMedia = (contentType, width, height, framerate, bitrate) => { + if (!width || !height) return null + const fps = framerate > 0 ? framerate : 30 + return { + contentType, + width, + height, + framerate: fps, + bitrate: bitrate > 0 ? bitrate : Math.round(width * height * fps * ESTIMATED_BITS_PER_PIXEL), + } +} + +/** + * What the source file is, in the terms navigator.mediaCapabilities asks for, so the + * player can check it will play smoothly before starting on it. + * + * Null when that is unknown or must not matter: the editor always needs the uncut + * original, and an .mkv's source is Fireshare's own H.264 conversion rather than the + * file the stored codec describes. + */ +const getSourceMedia = (videoInfo, extension, { hasCrop, forceOriginal }) => { + if (forceOriginal || extension === '.mkv' || !videoInfo?.codec) return null + // A crop is a stream copy into MP4, so it keeps the original's codec. + const container = extension === '.webm' && !hasCrop ? 'video/webm' : 'video/mp4' + return describeMedia( + `${container}; codecs="${videoInfo.codec}"`, + videoInfo.width, + videoInfo.height, + videoInfo.framerate, + videoInfo.bitrate, + ) +} + +// Transcodes are scaled to the target height with the source's aspect ratio and frame +// rate, as ffmpeg's scale=-2: does. +const getTranscodeMedia = (videoInfo, height) => { + if (!videoInfo?.width || !videoInfo?.height) return null + const width = Math.round((videoInfo.width * height) / videoInfo.height / 2) * 2 + return describeMedia(`video/mp4; codecs="${TRANSCODE_CODEC}"`, width, height, videoInfo.framerate) +} + /** * Generates video sources array for Video.js player with quality options - * Defaults to original quality, with 720p and 1080p as alternatives + * Defaults to original quality, with 720p and 1080p as alternatives. Each source + * carries a `media` description so the player can move the default to a transcode + * when this device cannot play the original smoothly. * @param {string} videoId - The video ID * @param {Object} videoInfo - Video info object containing has_720p, has_1080p flags * @param {string} extension - Video file extension (e.g., '.mp4', '.mkv') @@ -248,6 +301,7 @@ export const getVideoSources = (videoId, videoInfo, extension, { forceOriginal = type: 'video/mp4', label: 'Source', selected: true, + media: getSourceMedia(videoInfo, extension, { hasCrop, forceOriginal }), }) if (has1080p) { @@ -255,6 +309,7 @@ export const getVideoSources = (videoId, videoInfo, extension, { forceOriginal = src: getVideoUrl(videoId, '1080p', extension), type: 'video/mp4', label: '1080p', + media: getTranscodeMedia(videoInfo, 1080), }) } @@ -263,6 +318,7 @@ export const getVideoSources = (videoId, videoInfo, extension, { forceOriginal = src: getVideoUrl(videoId, '720p', extension), type: 'video/mp4', label: '720p', + media: getTranscodeMedia(videoInfo, 720), }) } @@ -271,6 +327,7 @@ export const getVideoSources = (videoId, videoInfo, extension, { forceOriginal = src: getVideoUrl(videoId, '480p', extension), type: 'video/mp4', label: '480p', + media: getTranscodeMedia(videoInfo, 480), }) } diff --git a/app/client/src/components/player/VideoJSPlayer.js b/app/client/src/components/player/VideoJSPlayer.js index 1a61b0f4..bd0bd9d9 100644 --- a/app/client/src/components/player/VideoJSPlayer.js +++ b/app/client/src/components/player/VideoJSPlayer.js @@ -4,6 +4,7 @@ import './videoSkinOverrides.css' import { createPlayer, useMedia, Poster } from '@videojs/react' import { Video, videoFeatures } from '@videojs/react/video' import CustomVideoSkin from './CustomVideoSkin' +import usePlayableSources from './usePlayableSources' // Tolerance threshold for checking if player is already at the desired start time (in seconds) const SEEK_TOLERANCE_SECONDS = 0.5 @@ -384,13 +385,7 @@ function FrameStepKeys() { return null } -/** - * VideoJSPlayer — a drop-in replacement powered by Video.js 10. - * - * Accepts the same props as the previous v8 component so that consumers - * (Watch.js, VideoModal.js) do not need to change their usage. - */ -const VideoJSPlayer = ({ +const PlayerWithSources = ({ sources, poster, autoplay = false, @@ -458,4 +453,19 @@ const VideoJSPlayer = ({ ) } +/** + * VideoJSPlayer — a drop-in replacement powered by Video.js 10. + * + * Accepts the same props as the previous v8 component so that consumers + * (Watch.js, VideoModal.js) do not need to change their usage. + * + * Mounts once the starting source is settled, so a source this device cannot + * play smoothly is never loaded just to be switched away from. + */ +const VideoJSPlayer = ({ sources, ...props }) => { + const playable = usePlayableSources(sources) + if (!playable) return null + return +} + export default VideoJSPlayer diff --git a/app/client/src/components/player/usePlayableSources.js b/app/client/src/components/player/usePlayableSources.js new file mode 100644 index 00000000..c1e49948 --- /dev/null +++ b/app/client/src/components/player/usePlayableSources.js @@ -0,0 +1,76 @@ +import { useEffect, useState } from 'react' + +// How long to wait for the browser's answer before starting on the source anyway. +// decodingInfo() normally answers in a few milliseconds; the player is held back +// until it does, so a browser that never answers must not leave it blank. +const DECODING_INFO_TIMEOUT_MS = 1500 + +const decodingInfo = (media) => + Promise.race([ + navigator.mediaCapabilities.decodingInfo({ type: 'file', video: media }), + new Promise((_, reject) => setTimeout(() => reject(new Error('decodingInfo timed out')), DECODING_INFO_TIMEOUT_MS)), + ]) + +/** + * The index of the source to start on: the best quality this device can decode in + * hardware, else the best it can decode smoothly at all, else the one already + * selected. + * + * The source can be something a transcode never is, such as 3440x1440 AV1 at 60 fps. + * A browser without an AV1 decoder in hardware falls back to software, which cannot + * keep up: the audio plays on while the picture freezes, then skips ahead out of sync. + * Nothing buffers, so the stall-based downgrade never sees it. + * + * getVideoSources lists the source first and the transcodes from 1080p down, so the + * first match is the best one. On a device with no hardware decoding at all nothing + * is power efficient, and the source is kept, exactly as before. + */ +const pickStartingIndex = async (sources) => { + const selected = Math.max(0, sources.findIndex((s) => s.selected)) + const results = await Promise.allSettled(sources.map((s) => (s.media ? decodingInfo(s.media) : Promise.reject()))) + const info = results.map((r) => (r.status === 'fulfilled' ? r.value : null)) + + const hardware = info.findIndex((r) => r?.supported && r.smooth && r.powerEfficient) + if (hardware >= 0) return hardware + const smooth = info.findIndex((r) => r?.supported && r.smooth) + if (smooth >= 0) return smooth + return selected +} + +/** + * The player's sources with `selected` moved to the one this device will actually + * play well, or null while the browser is being asked. + * + * Returns the sources unchanged, without waiting, whenever there is nothing to + * decide: one source, no codec known for the selected one, or no Media + * Capabilities API (which browsers only expose to secure contexts, so a plain-http + * instance behaves as it always has). + */ +const usePlayableSources = (sources) => { + const key = sources?.map((s) => s.src).join('\n') || '' + const selectedMedia = sources?.find((s) => s.selected)?.media + const needsCheck = Boolean(sources?.length > 1 && selectedMedia && navigator.mediaCapabilities?.decodingInfo) + const [decision, setDecision] = useState({ key: null, index: null }) + + useEffect(() => { + if (!needsCheck) return + let cancelled = false + pickStartingIndex(sources) + .catch(() => Math.max(0, sources.findIndex((s) => s.selected))) + .then((index) => { + if (!cancelled) setDecision({ key, index }) + }) + return () => { + cancelled = true + } + // `key` stands in for `sources`: the parent builds a new array on every render, + // and asking again for the same URLs would only repeat the same answer. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [key, needsCheck]) + + if (!needsCheck) return sources + if (decision.key !== key) return null + return sources.map((s, i) => ({ ...s, selected: i === decision.index })) +} + +export default usePlayableSources diff --git a/app/server/fireshare/api/upload_tokens.py b/app/server/fireshare/api/upload_tokens.py index 9fc7d8b8..ffb1b666 100644 --- a/app/server/fireshare/api/upload_tokens.py +++ b/app/server/fireshare/api/upload_tokens.py @@ -43,8 +43,8 @@ from .. import permissions as P from ..constants import SUPPORTED_FILE_TYPES from ..ip_whitelist import get_client_ip -from ..models import (FolderRule, GameMetadata, Image, ImageFolderRule, UploadToken, User, - Video) +from ..models import (CustomTag, FolderRule, GameMetadata, Image, ImageFolderRule, ImageInfo, + ImageTagLink, UploadToken, User, Video, VideoInfo, VideoTagLink) from . import api from .decorators import json_body, require_perm from .helpers import sanitize_upload_folder, secure_filename @@ -689,14 +689,56 @@ def token_upload_check(token_user): }) +def _offerable_tags(user): + """The tags an upload from this account may be offered, by name. + + /api/tags cannot serve an upload tool. It does not recognise upload tokens, + so it answers them as it would an anonymous visitor: only tags already on a + public video. A tag created a moment ago is on nothing yet, and it is + exactly the one somebody setting up a folder has come to choose. + + So this keeps /api/tags' rule about who may see what, and changes only what + it gets wrong for an uploader. An account that can view private media sees + every tag, as it would in the browser. Any other account sees the tags that + are on something public, plus the ones on nothing at all — an unused tag + gives away nothing about private media. What stays hidden is only what + /api/tags hides as well: a tag that appears solely on private media. + """ + tags = CustomTag.query.order_by(CustomTag.name).all() + if user.can(P.VIEW_PRIVATE): + return tags + + public_videos = ( + db.session.query(VideoTagLink.tag_id) + .join(Video, Video.video_id == VideoTagLink.video_id) + .join(VideoInfo, VideoInfo.video_id == VideoTagLink.video_id) + .filter(Video.available.is_(True), VideoInfo.private.is_(False)) + ) + public_images = ( + db.session.query(ImageTagLink.tag_id) + .join(Image, Image.image_id == ImageTagLink.image_id) + .join(ImageInfo, ImageInfo.image_id == ImageTagLink.image_id) + .filter(Image.available.is_(True), ImageInfo.private.is_(False)) + ) + on_something_public = {tag_id for (tag_id,) in public_videos.union(public_images)} + on_anything = { + tag_id + for (tag_id,) in db.session.query(VideoTagLink.tag_id).union( + db.session.query(ImageTagLink.tag_id) + ) + } + return [t for t in tags if t.id in on_something_public or t.id not in on_anything] + + @api.route('/api/upload/token/options', methods=['GET']) @upload_token_required def token_upload_options(token_user): - """The folders and games an upload may name, so a tool can offer real choices. + """The folders, games and tags an upload may name, so a tool can offer real choices. Games are listed in full rather than through /api/games, which hides games with nothing linked to them yet: those are exactly the ones an upload might - be the first to use, and `game` name resolution already accepts them. + be the first to use, and `game` name resolution already accepts them. Tags + are listed here for the same reason; see _offerable_tags for which. """ paths = current_app.config['PATHS'] try: @@ -745,6 +787,7 @@ def rules_json(rules): {'id': g.id, 'name': g.name, 'steamgriddb_id': g.steamgriddb_id} for g in games ], + 'tags': [t.json() for t in _offerable_tags(token_user)], }) diff --git a/app/server/fireshare/media_codecs.py b/app/server/fireshare/media_codecs.py new file mode 100644 index 00000000..517a24a7 --- /dev/null +++ b/app/server/fireshare/media_codecs.py @@ -0,0 +1,174 @@ +"""The browser-facing codec string for a video stream. + +A browser can only say whether it will play a file smoothly when it is told +exactly what the file is: `navigator.mediaCapabilities.decodingInfo()` takes an +RFC 6381 codec string such as `avc1.640033` or `av01.0.13M.08`, not a codec +name. The player asks before it picks which quality to start on, so a 1440p60 +AV1 source does not become the default on a device that has to decode it in +software and falls seconds behind its own audio. + +Built from the ffprobe stream entry already stored in `VideoInfo.info`. Anything +that cannot be described with confidence returns None, and the player then +keeps its old behaviour of starting on the source. +""" +import re + +# ffprobe's H.264 profile names, to profile_idc and the constraint byte. +_H264_PROFILES = { + 'baseline': (0x42, 0x00), + 'constrained baseline': (0x42, 0xE0), + 'main': (0x4D, 0x00), + 'extended': (0x58, 0x00), + 'high': (0x64, 0x00), + 'constrained high': (0x64, 0x00), + 'progressive high': (0x64, 0x00), + 'high 10': (0x6E, 0x00), + 'high 10 intra': (0x6E, 0x00), + 'high 4:2:2': (0x7A, 0x00), + 'high 4:2:2 intra': (0x7A, 0x00), + 'high 4:4:4': (0xF4, 0x00), + 'high 4:4:4 predictive': (0xF4, 0x00), + 'high 4:4:4 intra': (0xF4, 0x00), + 'cavlc 4:4:4': (0x2C, 0x00), + 'cavlc 4:4:4 intra': (0x2C, 0x00), +} + +# ffprobe's HEVC profile names, to general_profile_idc and the compatibility +# flags written in reverse bit order, as RFC 6381 wants them. +_HEVC_PROFILES = { + 'main': (1, 0x6), + 'main 10': (2, 0x4), + 'main still picture': (3, 0x2), + 'rext': (4, 0x10), +} + +_AV1_PROFILES = { + 'main': 0, + 'high': 1, + 'professional': 2, +} + +# AV1 levels (seq_level_idx) with their MaxPicSize, MaxHSize, MaxVSize and +# MaxDisplayRate from Annex A of the AV1 spec, smallest first. Used only when +# ffprobe did not report a level. +_AV1_LEVELS = [ + (0, 147456, 2048, 1152, 4423680), + (1, 278784, 2816, 1584, 8363520), + (4, 665856, 4352, 2448, 19975680), + (5, 1065024, 5504, 3096, 31950720), + (8, 2359296, 6144, 3456, 70778880), + (9, 2359296, 6144, 3456, 141557760), + (12, 8912896, 8192, 4352, 267386880), + (13, 8912896, 8192, 4352, 534773760), + (14, 8912896, 8192, 4352, 1069547520), + (15, 8912896, 8192, 4352, 1069547520), + (16, 35651584, 16384, 8704, 1069547520), + (17, 35651584, 16384, 8704, 2139095040), + (18, 35651584, 16384, 8704, 4278190080), + (19, 35651584, 16384, 8704, 4278190080), +] +_AV1_DEFINED_LEVELS = {level for level, *_ in _AV1_LEVELS} + + +def _int(value): + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _framerate(stream): + raw = stream.get('avg_frame_rate') or stream.get('r_frame_rate') or '' + try: + num, den = raw.split('/') + rate = float(num) / float(den) + except (ValueError, ZeroDivisionError): + return None + return rate if rate > 0 else None + + +def _bit_depth(stream): + match = re.search(r'p(\d+)(le|be)?$', stream.get('pix_fmt') or '') + if match: + return int(match.group(1)) + return _int(stream.get('bits_per_raw_sample')) or 8 + + +def _av1_level_for(width, height, fps): + """The smallest AV1 level whose limits hold this picture size and rate.""" + if not width or not height: + return None + pixels = width * height + rate = pixels * (fps or 30) + for level, max_pic, max_h, max_v, max_rate in _AV1_LEVELS: + if pixels <= max_pic and width <= max_h and height <= max_v and rate <= max_rate: + return level + return None + + +def _h264(stream): + profile = _H264_PROFILES.get((stream.get('profile') or '').lower()) + level = _int(stream.get('level')) + if not profile or not level or level <= 0: + return None + profile_idc, constraints = profile + return f"avc1.{profile_idc:02X}{constraints:02X}{level:02X}" + + +def _hevc(stream): + profile = _HEVC_PROFILES.get((stream.get('profile') or '').lower()) + level = _int(stream.get('level')) + if not profile or not level or level <= 0: + return None + tag = stream.get('codec_tag_string') + tag = tag if tag in ('hvc1', 'hev1') else 'hvc1' + profile_idc, compat = profile + # ffprobe does not report the tier. Main tier covers everything a game + # recorder or phone produces. + return f"{tag}.{profile_idc}.{compat:X}.L{level}.B0" + + +def _av1(stream): + profile_name = (stream.get('profile') or '').lower() + if profile_name in _AV1_PROFILES: + profile = _AV1_PROFILES[profile_name] + elif (stream.get('pix_fmt') or '').startswith(('yuv420', 'gray')): + profile = 0 + else: + return None + + level = _int(stream.get('level')) + if level not in _AV1_DEFINED_LEVELS: + level = _av1_level_for(_int(stream.get('width')), _int(stream.get('height')), + _framerate(stream)) + if level is None: + return None + + depth = _bit_depth(stream) + if depth not in (8, 10, 12): + return None + # As with HEVC, the tier is not reported; Main is the one in practice. + return f"av01.{profile}.{level:02d}M.{depth:02d}" + + +_BUILDERS = { + 'h264': _h264, + 'hevc': _hevc, + 'av1': _av1, +} + + +def codec_string(stream): + """The RFC 6381 codec string for an ffprobe video stream, or None.""" + if not stream: + return None + build = _BUILDERS.get((stream.get('codec_name') or '').lower()) + return build(stream) if build else None + + +def stream_bitrate(stream): + """The stream's bitrate in bits per second, when the container records one.""" + if not stream: + return None + rate = _int(stream.get('bit_rate')) + return rate if rate and rate > 0 else None diff --git a/app/server/fireshare/models.py b/app/server/fireshare/models.py index 3363ceaf..1d0921a3 100644 --- a/app/server/fireshare/models.py +++ b/app/server/fireshare/models.py @@ -4,6 +4,7 @@ from flask_login import UserMixin from . import db from . import permissions as perms +from . import media_codecs class User(UserMixin, db.Model): id = db.Column(db.Integer, primary_key=True) @@ -244,6 +245,7 @@ def _cropped_duration(self): return end - start def json(self): + stream = self.vcodec return { "title": self.title, "description": self.description, @@ -252,6 +254,10 @@ def json(self): "height": self.height, "duration": round(self._cropped_duration()) if self.duration else 0, "framerate": self.framerate, + # What the player needs to ask the browser whether the source will + # play smoothly, before choosing it over a transcode. + "codec": media_codecs.codec_string(stream), + "bitrate": media_codecs.stream_bitrate(stream), "has_480p": self.has_480p, "has_720p": self.has_720p, "has_1080p": self.has_1080p, diff --git a/docs/UploadTokens.md b/docs/UploadTokens.md index 4397708e..cd1574a5 100644 --- a/docs/UploadTokens.md +++ b/docs/UploadTokens.md @@ -174,7 +174,7 @@ for c in chunk_*; do done ``` -## Listing folders and games +## Listing folders, games and tags To offer real choices rather than making a user type a folder name from memory: @@ -192,6 +192,9 @@ curl https://fireshare.example.com/api/upload/token/options \ }, "games": [ { "id": 3, "name": "VALORANT", "steamgriddb_id": 12345 } + ], + "tags": [ + { "id": 7, "name": "Clutch", "color": "#FF5733" } ] } ``` @@ -200,6 +203,31 @@ Every game in the library is listed, including ones with nothing linked to them yet — `/api/games` hides those, but they are exactly the games an upload might be the first to use, and the `game` field already accepts them. +### Tags + +`tags` lists the tags an upload may name, sorted by name. Send the ids you want +as the upload's comma-separated `tag_ids`. `color` is the tag's hex colour, or +`null` if it has none. + +Which tags appear follows the rule the tag listings in the browser use: + +* A token whose account can **view private media** sees every tag. +* Any other token sees the tags that are on at least one public item, plus the + tags that are on nothing at all yet. + +The only tags left out are ones that appear solely on private media, which +`/api/tags` hides from the same accounts. Unused tags are included on purpose. +`/api/tags` doesn't recognise upload tokens and would leave them out, but a tag +created a moment ago is exactly the one somebody setting up an upload has come to +choose. + +`tag_ids` is not checked against this list, so an id for a tag that has since +been deleted is accepted. Read the list again before relying on an id you stored +earlier. + +An instance older than this field doesn't send `tags` at all, which is not the +same as having no tags. Treat a missing key as "this Fireshare can't list tags". + ### Folder rules `folder_rules` is the folder-to-game mapping Fireshare uses when scanning: media