Add voice message functionality

.changeset/add_voice_message_func.md

@@ -0,0 +1,5 @@
+---
+sable: minor
+---
+
+add voice message composing

.gitignore

@@ -18,3 +18,6 @@ devAssets
 *.tfbackend
 !*.tfbackend.example
 crash.log
+
+# the following line was added with the "git ignore" tool by itsrye.dev, version 0.1.0
+.lh

pnpm-workspace.yaml

@@ -1,20 +1,19 @@
-engineStrict: true
-minimumReleaseAge: 1440
-
 allowBuilds:
   '@swc/core': true
   esbuild: true
   sharp: true
   unrs-resolver: true
   workerd: true
+engineStrict: true
+minimumReleaseAge: 1440
 
 overrides:
-  serialize-javascript: '>=7.0.3'
-  rollup: '>=4.59.0'
-  minimatch: '>=10.2.3'
-  lodash: '>=4.17.23'
-  esbuild: '>=0.25.0'
   brace-expansion: '>=1.1.12'
+  esbuild: '>=0.25.0'
+  lodash: '>=4.17.23'
+  minimatch: '>=10.2.3'
+  rollup: '>=4.59.0'
+  serialize-javascript: '>=7.0.3'
 
 peerDependencyRules:
   allowedVersions:

src/app/features/room/AudioMessageRecorder.tsx

@@ -0,0 +1,85 @@
+import { VoiceRecorder } from '$plugins/voice-recorder-kit';
+import FocusTrap from 'focus-trap-react';
+import { Box, Text, color, config } from 'folds';
+import { useRef } from 'react';
+
+type AudioMessageRecorderProps = {
+  onRecordingComplete: (audioBlob: Blob) => void;
+  onRequestClose: () => void;
+  onWaveformUpdate: (waveform: number[]) => void;
+  onAudioLengthUpdate: (length: number) => void;
+};
+
+// We use a react voice recorder library to handle the recording of audio messages, as it provides a simple API and handles the complexities of recording audio in the browser.
+// The component is wrapped in a focus trap to ensure that keyboard users can easily navigate and interact with the recorder without accidentally losing focus or interacting with other parts of the UI.
+// The styling is kept simple and consistent with the rest of the app, using Folds' design tokens for colors, spacing, and typography.
+// we use a modified version of https://www.npmjs.com/package/react-voice-recorder-kit for the recording
+export function AudioMessageRecorder({
+  onRecordingComplete,
+  onRequestClose,
+  onWaveformUpdate,
+  onAudioLengthUpdate,
+}: AudioMessageRecorderProps) {
+  const containerRef = useRef<HTMLDivElement>(null);
+  const isDismissedRef = useRef(false);
+
+  // uses default styling, we use at other places
+  return (
+    <FocusTrap
+      focusTrapOptions={{
+        returnFocusOnDeactivate: false,
+        initialFocus: false,
+        onDeactivate: () => {
+          isDismissedRef.current = true;
+          onRequestClose();
+        },
+        clickOutsideDeactivates: true,
+        allowOutsideClick: true,
+        fallbackFocus: () => containerRef.current!,
+      }}
+    >
+      <div ref={containerRef} tabIndex={-1} style={{ outline: 'none' }}>
+        <Box
+          direction="Column"
+          gap="200"
+          alignItems="Center"
+          style={{
+            backgroundColor: color.Surface.Container,
+            color: color.Surface.OnContainer,
+            border: `${config.borderWidth.B300} solid ${color.Surface.ContainerLine}`,
+            borderRadius: config.radii.R400,
+            boxShadow: config.shadow.E200,
+            padding: config.space.S400,
+            minWidth: 300,
+          }}
+        >
+          <Text size="H4">Audio Message Recorder</Text>
+          <VoiceRecorder
+            autoStart
+            onStop={({
+              audioFile,
+              waveform,
+              audioLength,
+            }: {
+              audioFile: Blob;
+              waveform: number[];
+              audioLength: number;
+            }) => {
+              if (isDismissedRef.current) return;
+              // closes the recorder and sends the audio file back to the parent component to be uploaded and sent as a message
+              onRecordingComplete(audioFile);
+              onWaveformUpdate(waveform);
+              onAudioLengthUpdate(audioLength);
+            }}
+            buttonBackgroundColor={color.SurfaceVariant.Container}
+            buttonHoverBackgroundColor={color.SurfaceVariant.ContainerHover}
+            iconColor={color.Primary.Main}
+            style={{
+              backgroundColor: color.Surface.ContainerActive,
+            }}
+          />
+        </Box>
+      </div>
+    </FocusTrap>
+  );
+}

src/app/features/room/RoomInput.tsx

@@ -157,6 +157,7 @@ import {
   getVideoMsgContent,
 } from './msgContent';
 import { CommandAutocomplete } from './CommandAutocomplete';
+import { AudioMessageRecorder } from './AudioMessageRecorder';
 
 const getReplyContent = (replyDraft: IReplyDraft | undefined): IEventRelation => {
   if (!replyDraft) return {};
@@ -195,6 +196,7 @@ export const RoomInput = forwardRef<HTMLDivElement, RoomInputProps>(
     const [hideActivity] = useSetting(settingsAtom, 'hideActivity');
     const commands = useCommands(mx, room);
     const emojiBtnRef = useRef<HTMLButtonElement>(null);
+    const micBtnRef = useRef<HTMLButtonElement>(null);
     const roomToParents = useAtomValue(roomToParentsAtom);
     const nicknames = useAtomValue(nicknamesAtom);
 
@@ -225,6 +227,9 @@ export const RoomInput = forwardRef<HTMLDivElement, RoomInputProps>(
     const imagePackRooms: Room[] = useImagePackRooms(roomId, roomToParents);
 
     const [toolbar, setToolbar] = useSetting(settingsAtom, 'editorToolbar');
+    const [showAudioRecorder, setShowAudioRecorder] = useState(false);
+    const [audioMsgWaveform, setAudioMsgWaveform] = useState<number[] | undefined>(undefined);
+    const [audioMsgLength, setAudioMsgLength] = useState<number | undefined>(undefined);
     const [autocompleteQuery, setAutocompleteQuery] =
       useState<AutocompleteQuery<AutocompletePrefix>>();
     const [isQuickTextReact, setQuickTextReact] = useState(false);
@@ -406,7 +411,7 @@ export const RoomInput = forwardRef<HTMLDivElement, RoomInputProps>(
           return getVideoMsgContent(mx, fileItem, upload.mxc);
         }
         if (fileItem.file.type.startsWith('audio')) {
-          return getAudioMsgContent(fileItem, upload.mxc);
+          return getAudioMsgContent(fileItem, upload.mxc, audioMsgWaveform, audioMsgLength);
         }
         return getFileMsgContent(fileItem, upload.mxc);
       });
@@ -948,6 +953,47 @@ export const RoomInput = forwardRef<HTMLDivElement, RoomInputProps>(
               >
                 <Icon src={toolbar ? Icons.AlphabetUnderline : Icons.Alphabet} />
               </IconButton>
+              <IconButton
+                ref={micBtnRef}
+                variant="SurfaceVariant"
+                size="300"
+                radii="300"
+                title="record audio message"
+                aria-pressed={showAudioRecorder}
+                onClick={() => setShowAudioRecorder(!showAudioRecorder)}
+              >
+                <Icon src={Icons.Mic} />
+              </IconButton>
+              {showAudioRecorder && (
+                <PopOut
+                  anchor={micBtnRef.current?.getBoundingClientRect() ?? undefined}
+                  offset={8}
+                  position="Top"
+                  align="End"
+                  alignOffset={-44}
+                  content={
+                    <AudioMessageRecorder
+                      onRequestClose={() => {
+                        setShowAudioRecorder(false);
+                      }}
+                      onRecordingComplete={(audioBlob) => {
+                        const file = new File(
+                          [audioBlob],
+                          `sable-audio-message-${Date.now()}.ogg`,
+                          {
+                            type: audioBlob.type,
+                          }
+                        );
+                        handleFiles([file]);
+                        // Close the recorder after handling the file, to give some feedback that the recording was successful
+                        setShowAudioRecorder(false);
+                      }}
+                      onAudioLengthUpdate={(len) => setAudioMsgLength(len)}
+                      onWaveformUpdate={(w) => setAudioMsgWaveform(w)}
+                    />
+                  }
+                />
+              )}
               <UseStateProvider initial={undefined}>
                 {(emojiBoardTab: EmojiBoardTab | undefined, setEmojiBoardTab) => (
                   <PopOut

src/app/features/room/msgContent.ts

@@ -143,7 +143,17 @@ export const getVideoMsgContent = async (
   return content;
 };
 
-export const getAudioMsgContent = (item: TUploadItem, mxc: string): IContent => {
+export type AudioMsgContent = IContent & {
+  waveform?: number[];
+  audioLength?: number;
+};
+
+export const getAudioMsgContent = (
+  item: TUploadItem,
+  mxc: string,
+  waveform?: number[],
+  audioLength?: number
+): AudioMsgContent => {
   const { file, encInfo } = item;
   const content: IContent = {
     msgtype: MsgType.Audio,
@@ -155,6 +165,10 @@ export const getAudioMsgContent = (item: TUploadItem, mxc: string): IContent =>
       mimetype: file.type,
       size: file.size,
     },
+    'org.matrix.msc1767.audio': {
+      waveform: waveform?.map((v) => Math.round(v * 1024)), // scale waveform values to fit in 10 bits (0-1024) for more efficient storage, as per MSC1767 spec
+      duration: item.metadata.markedAsSpoiler || !audioLength ? 0 : audioLength * 1000, // if marked as spoiler, set duration to 0 to hide it in clients that support msc1767
+    },
   };
   if (encInfo) {
     content.file = {