Release v1.8.4: smoother playback for heavy sources, and tags for upload tokens

app/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fireshare",
-  "version": "1.8.3",
+  "version": "1.8.4",
   "private": true,
   "dependencies": {
     "@emotion/react": "^11.9.0",

app/client/src/common/utils.js

@@ -215,9 +215,62 @@ export const getImageUrl = (imageId) => {
   return `${baseUrl}/api/image?id=${imageId}`
 }
 
+// Transcodes are H.264 unless an admin has switched the encoder to AV1, and nothing
+// records which one a given transcode used, so they are described as H.264. The
+// description is only used to ask whether the device decodes that kind of file in
+// hardware.
+const TRANSCODE_CODEC = 'avc1.640033'
+
+// Rough bits per pixel per frame, for a file that does not record its bitrate. The
+// browser decides on codec, size and frame rate; the bitrate only has to be plausible.
+const ESTIMATED_BITS_PER_PIXEL = 0.1
+
+const describeMedia = (contentType, width, height, framerate, bitrate) => {
+  if (!width || !height) return null
+  const fps = framerate > 0 ? framerate : 30
+  return {
+    contentType,
+    width,
+    height,
+    framerate: fps,
+    bitrate: bitrate > 0 ? bitrate : Math.round(width * height * fps * ESTIMATED_BITS_PER_PIXEL),
+  }
+}
+
+/**
+ * What the source file is, in the terms navigator.mediaCapabilities asks for, so the
+ * player can check it will play smoothly before starting on it.
+ *
+ * Null when that is unknown or must not matter: the editor always needs the uncut
+ * original, and an .mkv's source is Fireshare's own H.264 conversion rather than the
+ * file the stored codec describes.
+ */
+const getSourceMedia = (videoInfo, extension, { hasCrop, forceOriginal }) => {
+  if (forceOriginal || extension === '.mkv' || !videoInfo?.codec) return null
+  // A crop is a stream copy into MP4, so it keeps the original's codec.
+  const container = extension === '.webm' && !hasCrop ? 'video/webm' : 'video/mp4'
+  return describeMedia(
+    `${container}; codecs="${videoInfo.codec}"`,
+    videoInfo.width,
+    videoInfo.height,
+    videoInfo.framerate,
+    videoInfo.bitrate,
+  )
+}
+
+// Transcodes are scaled to the target height with the source's aspect ratio and frame
+// rate, as ffmpeg's scale=-2:<height> does.
+const getTranscodeMedia = (videoInfo, height) => {
+  if (!videoInfo?.width || !videoInfo?.height) return null
+  const width = Math.round((videoInfo.width * height) / videoInfo.height / 2) * 2
+  return describeMedia(`video/mp4; codecs="${TRANSCODE_CODEC}"`, width, height, videoInfo.framerate)
+}
+
 /**
  * Generates video sources array for Video.js player with quality options
- * Defaults to original quality, with 720p and 1080p as alternatives
+ * Defaults to original quality, with 720p and 1080p as alternatives. Each source
+ * carries a `media` description so the player can move the default to a transcode
+ * when this device cannot play the original smoothly.
  * @param {string} videoId - The video ID
  * @param {Object} videoInfo - Video info object containing has_720p, has_1080p flags
  * @param {string} extension - Video file extension (e.g., '.mp4', '.mkv')
@@ -248,13 +301,15 @@ export const getVideoSources = (videoId, videoInfo, extension, { forceOriginal =
     type: 'video/mp4',
     label: 'Source',
     selected: true,
+    media: getSourceMedia(videoInfo, extension, { hasCrop, forceOriginal }),
   })
 
   if (has1080p) {
     sources.push({
       src: getVideoUrl(videoId, '1080p', extension),
       type: 'video/mp4',
       label: '1080p',
+      media: getTranscodeMedia(videoInfo, 1080),
     })
   }
 
@@ -263,6 +318,7 @@ export const getVideoSources = (videoId, videoInfo, extension, { forceOriginal =
       src: getVideoUrl(videoId, '720p', extension),
       type: 'video/mp4',
       label: '720p',
+      media: getTranscodeMedia(videoInfo, 720),
     })
   }
 
@@ -271,6 +327,7 @@ export const getVideoSources = (videoId, videoInfo, extension, { forceOriginal =
       src: getVideoUrl(videoId, '480p', extension),
       type: 'video/mp4',
       label: '480p',
+      media: getTranscodeMedia(videoInfo, 480),
     })
   }
 

app/client/src/components/player/VideoJSPlayer.js

@@ -4,6 +4,7 @@ import './videoSkinOverrides.css'
 import { createPlayer, useMedia, Poster } from '@videojs/react'
 import { Video, videoFeatures } from '@videojs/react/video'
 import CustomVideoSkin from './CustomVideoSkin'
+import usePlayableSources from './usePlayableSources'
 
 // Tolerance threshold for checking if player is already at the desired start time (in seconds)
 const SEEK_TOLERANCE_SECONDS = 0.5
@@ -384,13 +385,7 @@ function FrameStepKeys() {
   return null
 }
 
-/**
- * VideoJSPlayer — a drop-in replacement powered by Video.js 10.
- *
- * Accepts the same props as the previous v8 component so that consumers
- * (Watch.js, VideoModal.js) do not need to change their usage.
- */
-const VideoJSPlayer = ({
+const PlayerWithSources = ({
   sources,
   poster,
   autoplay = false,
@@ -458,4 +453,19 @@ const VideoJSPlayer = ({
   )
 }
 
+/**
+ * VideoJSPlayer — a drop-in replacement powered by Video.js 10.
+ *
+ * Accepts the same props as the previous v8 component so that consumers
+ * (Watch.js, VideoModal.js) do not need to change their usage.
+ *
+ * Mounts once the starting source is settled, so a source this device cannot
+ * play smoothly is never loaded just to be switched away from.
+ */
+const VideoJSPlayer = ({ sources, ...props }) => {
+  const playable = usePlayableSources(sources)
+  if (!playable) return null
+  return <PlayerWithSources sources={playable} {...props} />
+}
+
 export default VideoJSPlayer

app/client/src/components/player/usePlayableSources.js

@@ -0,0 +1,76 @@
+import { useEffect, useState } from 'react'
+
+// How long to wait for the browser's answer before starting on the source anyway.
+// decodingInfo() normally answers in a few milliseconds; the player is held back
+// until it does, so a browser that never answers must not leave it blank.
+const DECODING_INFO_TIMEOUT_MS = 1500
+
+const decodingInfo = (media) =>
+  Promise.race([
+    navigator.mediaCapabilities.decodingInfo({ type: 'file', video: media }),
+    new Promise((_, reject) => setTimeout(() => reject(new Error('decodingInfo timed out')), DECODING_INFO_TIMEOUT_MS)),
+  ])
+
+/**
+ * The index of the source to start on: the best quality this device can decode in
+ * hardware, else the best it can decode smoothly at all, else the one already
+ * selected.
+ *
+ * The source can be something a transcode never is, such as 3440x1440 AV1 at 60 fps.
+ * A browser without an AV1 decoder in hardware falls back to software, which cannot
+ * keep up: the audio plays on while the picture freezes, then skips ahead out of sync.
+ * Nothing buffers, so the stall-based downgrade never sees it.
+ *
+ * getVideoSources lists the source first and the transcodes from 1080p down, so the
+ * first match is the best one. On a device with no hardware decoding at all nothing
+ * is power efficient, and the source is kept, exactly as before.
+ */
+const pickStartingIndex = async (sources) => {
+  const selected = Math.max(0, sources.findIndex((s) => s.selected))
+  const results = await Promise.allSettled(sources.map((s) => (s.media ? decodingInfo(s.media) : Promise.reject())))
+  const info = results.map((r) => (r.status === 'fulfilled' ? r.value : null))
+
+  const hardware = info.findIndex((r) => r?.supported && r.smooth && r.powerEfficient)
+  if (hardware >= 0) return hardware
+  const smooth = info.findIndex((r) => r?.supported && r.smooth)
+  if (smooth >= 0) return smooth
+  return selected
+}
+
+/**
+ * The player's sources with `selected` moved to the one this device will actually
+ * play well, or null while the browser is being asked.
+ *
+ * Returns the sources unchanged, without waiting, whenever there is nothing to
+ * decide: one source, no codec known for the selected one, or no Media
+ * Capabilities API (which browsers only expose to secure contexts, so a plain-http
+ * instance behaves as it always has).
+ */
+const usePlayableSources = (sources) => {
+  const key = sources?.map((s) => s.src).join('\n') || ''
+  const selectedMedia = sources?.find((s) => s.selected)?.media
+  const needsCheck = Boolean(sources?.length > 1 && selectedMedia && navigator.mediaCapabilities?.decodingInfo)
+  const [decision, setDecision] = useState({ key: null, index: null })
+
+  useEffect(() => {
+    if (!needsCheck) return
+    let cancelled = false
+    pickStartingIndex(sources)
+      .catch(() => Math.max(0, sources.findIndex((s) => s.selected)))
+      .then((index) => {
+        if (!cancelled) setDecision({ key, index })
+      })
+    return () => {
+      cancelled = true
+    }
+    // `key` stands in for `sources`: the parent builds a new array on every render,
+    // and asking again for the same URLs would only repeat the same answer.
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [key, needsCheck])
+
+  if (!needsCheck) return sources
+  if (decision.key !== key) return null
+  return sources.map((s, i) => ({ ...s, selected: i === decision.index }))
+}
+
+export default usePlayableSources

app/server/fireshare/api/upload_tokens.py

@@ -43,8 +43,8 @@
 from .. import permissions as P
 from ..constants import SUPPORTED_FILE_TYPES
 from ..ip_whitelist import get_client_ip
-from ..models import (FolderRule, GameMetadata, Image, ImageFolderRule, UploadToken, User,
-                      Video)
+from ..models import (CustomTag, FolderRule, GameMetadata, Image, ImageFolderRule, ImageInfo,
+                      ImageTagLink, UploadToken, User, Video, VideoInfo, VideoTagLink)
 from . import api
 from .decorators import json_body, require_perm
 from .helpers import sanitize_upload_folder, secure_filename
@@ -689,14 +689,56 @@ def token_upload_check(token_user):
     })
 
 
+def _offerable_tags(user):
+    """The tags an upload from this account may be offered, by name.
+
+    /api/tags cannot serve an upload tool. It does not recognise upload tokens,
+    so it answers them as it would an anonymous visitor: only tags already on a
+    public video. A tag created a moment ago is on nothing yet, and it is
+    exactly the one somebody setting up a folder has come to choose.
+
+    So this keeps /api/tags' rule about who may see what, and changes only what
+    it gets wrong for an uploader. An account that can view private media sees
+    every tag, as it would in the browser. Any other account sees the tags that
+    are on something public, plus the ones on nothing at all — an unused tag
+    gives away nothing about private media. What stays hidden is only what
+    /api/tags hides as well: a tag that appears solely on private media.
+    """
+    tags = CustomTag.query.order_by(CustomTag.name).all()
+    if user.can(P.VIEW_PRIVATE):
+        return tags
+
+    public_videos = (
+        db.session.query(VideoTagLink.tag_id)
+        .join(Video, Video.video_id == VideoTagLink.video_id)
+        .join(VideoInfo, VideoInfo.video_id == VideoTagLink.video_id)
+        .filter(Video.available.is_(True), VideoInfo.private.is_(False))
+    )
+    public_images = (
+        db.session.query(ImageTagLink.tag_id)
+        .join(Image, Image.image_id == ImageTagLink.image_id)
+        .join(ImageInfo, ImageInfo.image_id == ImageTagLink.image_id)
+        .filter(Image.available.is_(True), ImageInfo.private.is_(False))
+    )
+    on_something_public = {tag_id for (tag_id,) in public_videos.union(public_images)}
+    on_anything = {
+        tag_id
+        for (tag_id,) in db.session.query(VideoTagLink.tag_id).union(
+            db.session.query(ImageTagLink.tag_id)
+        )
+    }
+    return [t for t in tags if t.id in on_something_public or t.id not in on_anything]
+
+
 @api.route('/api/upload/token/options', methods=['GET'])
 @upload_token_required
 def token_upload_options(token_user):
-    """The folders and games an upload may name, so a tool can offer real choices.
+    """The folders, games and tags an upload may name, so a tool can offer real choices.
 
     Games are listed in full rather than through /api/games, which hides games
     with nothing linked to them yet: those are exactly the ones an upload might
-    be the first to use, and `game` name resolution already accepts them.
+    be the first to use, and `game` name resolution already accepts them. Tags
+    are listed here for the same reason; see _offerable_tags for which.
     """
     paths = current_app.config['PATHS']
     try:
@@ -745,6 +787,7 @@ def rules_json(rules):
             {'id': g.id, 'name': g.name, 'steamgriddb_id': g.steamgriddb_id}
             for g in games
         ],
+        'tags': [t.json() for t in _offerable_tags(token_user)],
     })