From 86ec6b9e7c6550ba50a3a68376ac1529f3725958 Mon Sep 17 00:00:00 2001
From: albin-karlsson <55614148+albin-karlsson@users.noreply.github.com>
Date: Sat, 20 Apr 2024 11:12:42 +0200
Subject: [PATCH 1/3] Add skip, pause and resume functionality

---
 client/src/components/Output.jsx | 69 ++++++++++++++------------------
 1 file changed, 30 insertions(+), 39 deletions(-)

diff --git a/client/src/components/Output.jsx b/client/src/components/Output.jsx
index bc6e6ef..9ea2801 100644
--- a/client/src/components/Output.jsx
+++ b/client/src/components/Output.jsx
@@ -1,5 +1,6 @@
 import React, { useState, useEffect } from "react";
 import TextOutput from "./TextOutput";
+import AudioOutput from "./AudioOutput";
 import ConversationControls from "./ConversationControls";
 
 function Output({ conversation }) {
@@ -7,58 +8,48 @@ function Output({ conversation }) {
   const [currentSnippetIndex, setCurrentSnippetIndex] = useState(0);
   const [currentMessageTextSnippet, setCurrentMessageTextSnippet] =
     useState("");
-  const [initialize, setInitialize] = useState(true); // Flag to control initialization
   const [isPaused, setIsPaused] = useState(false);
 
-  // Function to calculate the display time based on text length
   const calculateDisplayTime = (text) => Math.max(3, text.length * 0.05);
 
   function handlePauseResume() {
-    if (!isPaused) {
-      console.log("Pausing");
-    } else {
-      console.log("Resuming");
-    }
-
-    setIsPaused(!isPaused);
+    setIsPaused(!isPaused); // Toggle pause state
   }
 
   function handleSkipForward() {
-    // TODO: See if last message before skipping forward
-
-    console.log("Skipping forward");
+    if (conversation.length > currentMessageIndex) {
+      const snippets =
+        conversation[currentMessageIndex].text.split(/(?<=\.\s)/);
+      if (currentSnippetIndex < snippets.length - 1) {
+        // Move to the next snippet within the same message
+        setCurrentSnippetIndex(currentSnippetIndex + 1);
+      } else if (currentMessageIndex < conversation.length - 1) {
+        // Move to the first snippet of the next message
+        setCurrentMessageIndex(currentMessageIndex + 1);
+        setCurrentSnippetIndex(0);
+      }
+    }
   }
 
   useEffect(() => {
-    if (initialize && conversation.length > 0) {
-      setCurrentMessageIndex(0);
-      setCurrentSnippetIndex(0);
-      setInitialize(false); // Reset initialization flag after first setup
-    }
-  }, [conversation, initialize]);
+    if (conversation.length > 0 && !isPaused) {
+      const snippets =
+        conversation[currentMessageIndex].text.split(/(?<=\.\s)/);
+      if (snippets.length > currentSnippetIndex) {
+        setCurrentMessageTextSnippet(snippets[currentSnippetIndex]);
+        const timeout = setTimeout(() => {
+          if (currentSnippetIndex < snippets.length - 1) {
+            setCurrentSnippetIndex(currentSnippetIndex + 1);
+          } else if (currentMessageIndex < conversation.length - 1) {
+            setCurrentMessageIndex(currentMessageIndex + 1);
+            setCurrentSnippetIndex(0);
+          }
+        }, calculateDisplayTime(snippets[currentSnippetIndex]) * 1000);
 
-  useEffect(() => {
-    const processSnippets = () => {
-      if (conversation.length > currentMessageIndex) {
-        const snippets =
-          conversation[currentMessageIndex].text.split(/(?<=\.\s)/);
-        if (snippets.length > currentSnippetIndex) {
-          setCurrentMessageTextSnippet(snippets[currentSnippetIndex]);
-          const timeout = setTimeout(() => {
-            if (currentSnippetIndex < snippets.length - 1) {
-              setCurrentSnippetIndex(currentSnippetIndex + 1);
-            } else if (currentMessageIndex < conversation.length - 1) {
-              setCurrentMessageIndex(currentMessageIndex + 1);
-              setCurrentSnippetIndex(0);
-            }
-          }, calculateDisplayTime(snippets[currentSnippetIndex]) * 1000);
-          return () => clearTimeout(timeout);
-        }
+        return () => clearTimeout(timeout); // Cleanup timeout on unmount or dependency change
       }
-    };
-
-    processSnippets();
-  }, [currentMessageIndex, currentSnippetIndex, conversation]);
+    }
+  }, [currentMessageIndex, currentSnippetIndex, conversation, isPaused]); // Include isPaused in dependencies
 
   return (
     <div>

From 63ef0d7e0ccefe26b2c2e085a35ae5fb4d2e5062 Mon Sep 17 00:00:00 2001
From: albin-karlsson <55614148+albin-karlsson@users.noreply.github.com>
Date: Sat, 20 Apr 2024 12:07:38 +0200
Subject: [PATCH 2/3] Add audio

---
 client/src/components/AudioOutput.jsx | 37 ++++++++++++++++++--
 client/src/components/Contact.jsx     | 15 ++++++--
 client/src/components/Council.jsx     | 22 ++++++++----
 client/src/components/Output.jsx      | 49 +++++++++++++++------------
 4 files changed, 88 insertions(+), 35 deletions(-)

diff --git a/client/src/components/AudioOutput.jsx b/client/src/components/AudioOutput.jsx
index e32964f..50141a9 100644
--- a/client/src/components/AudioOutput.jsx
+++ b/client/src/components/AudioOutput.jsx
@@ -1,7 +1,38 @@
-import React from "react";
+// In the AudioOutput component
+import React, { useEffect, useRef } from "react";
 
-function AudioOutput() {
-  return <></>;
+function AudioOutput({ currentAudioBuffer }) {
+  const audioRef = useRef(null);
+
+  useEffect(() => {
+    if (currentAudioBuffer) {
+      console.log(
+        "Creating blob with ArrayBuffer of size:",
+        currentAudioBuffer.byteLength
+      );
+      const blob = new Blob([currentAudioBuffer], { type: "audio/mp3" });
+      const url = URL.createObjectURL(blob);
+      console.log("Blob URL:", url); // Check the generated URL
+      audioRef.current.src = url;
+      audioRef.current
+        .play()
+        .catch((err) => console.error("Error playing audio:", err));
+
+      return () => {
+        console.log("Revoking URL:", url);
+        URL.revokeObjectURL(url);
+      };
+    }
+  }, [currentAudioBuffer]);
+
+  return (
+    <audio
+      ref={audioRef}
+      controls
+      autoPlay
+      onLoadedData={() => console.log("Audio loaded")}
+    />
+  );
 }
 
 export default AudioOutput;
diff --git a/client/src/components/Contact.jsx b/client/src/components/Contact.jsx
index 3891674..e6e4c71 100644
--- a/client/src/components/Contact.jsx
+++ b/client/src/components/Contact.jsx
@@ -3,7 +3,10 @@ import React from "react";
 function Contact() {
   return (
     <div className="wrapper">
-      <div className="text-container" style={{ justifyContent: "center" }}>
+      <div
+        className="text-container"
+        style={{ justifyContent: "center" }}
+      >
         <h4>
           The project is an initiative by art & design
           <br /> collective Nonhuman Nonsense developed in
@@ -18,11 +21,17 @@ function Contact() {
             @nonhuman-nonsense
           </a>
           <br />
-          <a className="link" href="https://nonhuman-nonsense.com">
+          <a
+            className="link"
+            href="https://nonhuman-nonsense.com"
+          >
             nonhuman-nonsense.com
           </a>
           <br />
-          <a className="link" href="mailto:hello@nonhuman-nonsense.com">
+          <a
+            className="link"
+            href="mailto:hello@nonhuman-nonsense.com"
+          >
             hello@nonhuman-nonsense.com
           </a>
         </h4>
diff --git a/client/src/components/Council.jsx b/client/src/components/Council.jsx
index 796ad5a..91ce0bb 100644
--- a/client/src/components/Council.jsx
+++ b/client/src/components/Council.jsx
@@ -17,6 +17,7 @@ function Council({ options }) {
   const [activeOverlay, setActiveOverlay] = useState("");
   const { width: screenWidth } = useWindowSize();
   const [conversation, setConversation] = useState([]); // State to store conversation updates
+  const [audioBuffers, setAudioBuffers] = useState([]); // To store multiple ArrayBuffers
   const socketRef = useRef(null); // Using useRef to persist socket instance
 
   const foodsContainerStyle = {
@@ -33,26 +34,30 @@ function Council({ options }) {
   useEffect(() => {
     socketRef.current = io();
 
+    // Send initial data to start the conversation
     let promptsAndOptions = {
       options: {
         ...globalOptions,
-        humanName: humanName,
+        humanName,
         raiseHandPrompt: false,
         neverMindPrompt: false,
       },
       name: "New room",
-      topic: topic,
+      topic,
       characters: foods,
     };
-
     socketRef.current.emit("start_conversation", promptsAndOptions);
 
-    // Listen for conversation updates
+    // Listen for conversation text updates
     socketRef.current.on("conversation_update", (message) => {
-      setConversation((prev) => [...prev, message]); // Update conversation state
+      setConversation((prev) => [...prev, message]);
+    });
+
+    // Listen for audio updates
+    socketRef.current.on("audio_update", (audioData) => {
+      setAudioBuffers((prevBuffers) => [...prevBuffers, audioData.audio]);
     });
 
-    // Cleanup on component unmount
     return () => {
       socketRef.current.disconnect();
     };
@@ -108,7 +113,10 @@ function Council({ options }) {
             className="text-container"
             style={{ justifyContent: "end" }}
           >
-            <Output conversation={conversation} />
+            <Output
+              conversation={conversation}
+              audioBuffers={audioBuffers}
+            />
           </div>
         )}
         <div style={foodsContainerStyle}>
diff --git a/client/src/components/Output.jsx b/client/src/components/Output.jsx
index 9ea2801..44114d3 100644
--- a/client/src/components/Output.jsx
+++ b/client/src/components/Output.jsx
@@ -3,33 +3,17 @@ import TextOutput from "./TextOutput";
 import AudioOutput from "./AudioOutput";
 import ConversationControls from "./ConversationControls";
 
-function Output({ conversation }) {
+function Output({ conversation, audioBuffers }) {
   const [currentMessageIndex, setCurrentMessageIndex] = useState(0);
   const [currentSnippetIndex, setCurrentSnippetIndex] = useState(0);
   const [currentMessageTextSnippet, setCurrentMessageTextSnippet] =
     useState("");
   const [isPaused, setIsPaused] = useState(false);
 
-  const calculateDisplayTime = (text) => Math.max(3, text.length * 0.05);
-
-  function handlePauseResume() {
-    setIsPaused(!isPaused); // Toggle pause state
-  }
-
-  function handleSkipForward() {
-    if (conversation.length > currentMessageIndex) {
-      const snippets =
-        conversation[currentMessageIndex].text.split(/(?<=\.\s)/);
-      if (currentSnippetIndex < snippets.length - 1) {
-        // Move to the next snippet within the same message
-        setCurrentSnippetIndex(currentSnippetIndex + 1);
-      } else if (currentMessageIndex < conversation.length - 1) {
-        // Move to the first snippet of the next message
-        setCurrentMessageIndex(currentMessageIndex + 1);
-        setCurrentSnippetIndex(0);
-      }
-    }
-  }
+  useEffect(() => {
+    console.log("Audio buffers length:", audioBuffers.length);
+    console.log("Current audio buffer index:", currentMessageIndex);
+  }, [audioBuffers, currentMessageIndex]);
 
   useEffect(() => {
     if (conversation.length > 0 && !isPaused) {
@@ -51,6 +35,27 @@ function Output({ conversation }) {
     }
   }, [currentMessageIndex, currentSnippetIndex, conversation, isPaused]); // Include isPaused in dependencies
 
+  const calculateDisplayTime = (text) => Math.max(3, text.length * 0.05);
+
+  function handlePauseResume() {
+    setIsPaused(!isPaused); // Toggle pause state
+  }
+
+  function handleSkipForward() {
+    if (conversation.length > currentMessageIndex) {
+      const snippets =
+        conversation[currentMessageIndex].text.split(/(?<=\.\s)/);
+      if (currentSnippetIndex < snippets.length - 1) {
+        // Move to the next snippet within the same message
+        setCurrentSnippetIndex(currentSnippetIndex + 1);
+      } else if (currentMessageIndex < conversation.length - 1) {
+        // Move to the first snippet of the next message
+        setCurrentMessageIndex(currentMessageIndex + 1);
+        setCurrentSnippetIndex(0);
+      }
+    }
+  }
+
   return (
     <div>
       <h2>
@@ -60,7 +65,7 @@ function Output({ conversation }) {
           : ""}
       </h2>
       <TextOutput currentMessageTextSnippet={currentMessageTextSnippet} />
-      <AudioOutput />
+      <AudioOutput currentAudioBuffer={audioBuffers[currentMessageIndex]} />
       <ConversationControls
         isPaused={isPaused}
         onPauseResume={handlePauseResume}

From a3e019edc93c70c409a3bb161cbea5054c5b6218 Mon Sep 17 00:00:00 2001
From: albin-karlsson <55614148+albin-karlsson@users.noreply.github.com>
Date: Sat, 20 Apr 2024 12:34:29 +0200
Subject: [PATCH 3/3] Link audio playback with current message

---
 client/src/components/AudioOutput.jsx | 10 +++++-----
 client/src/components/Council.jsx     | 11 +++++++----
 client/src/components/Output.jsx      | 16 ++++++++++------
 3 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/client/src/components/AudioOutput.jsx b/client/src/components/AudioOutput.jsx
index 50141a9..40c51eb 100644
--- a/client/src/components/AudioOutput.jsx
+++ b/client/src/components/AudioOutput.jsx
@@ -1,16 +1,16 @@
 // In the AudioOutput component
 import React, { useEffect, useRef } from "react";
 
-function AudioOutput({ currentAudioBuffer }) {
+function AudioOutput({ currentAudioMessage }) {
   const audioRef = useRef(null);
 
   useEffect(() => {
-    if (currentAudioBuffer) {
+    if (currentAudioMessage) {
       console.log(
         "Creating blob with ArrayBuffer of size:",
-        currentAudioBuffer.byteLength
+        currentAudioMessage.audio.byteLength
       );
-      const blob = new Blob([currentAudioBuffer], { type: "audio/mp3" });
+      const blob = new Blob([currentAudioMessage.audio], { type: "audio/mp3" });
       const url = URL.createObjectURL(blob);
       console.log("Blob URL:", url); // Check the generated URL
       audioRef.current.src = url;
@@ -23,7 +23,7 @@ function AudioOutput({ currentAudioBuffer }) {
         URL.revokeObjectURL(url);
       };
     }
-  }, [currentAudioBuffer]);
+  }, [currentAudioMessage]);
 
   return (
     <audio
diff --git a/client/src/components/Council.jsx b/client/src/components/Council.jsx
index 91ce0bb..8e3d2e9 100644
--- a/client/src/components/Council.jsx
+++ b/client/src/components/Council.jsx
@@ -17,7 +17,7 @@ function Council({ options }) {
   const [activeOverlay, setActiveOverlay] = useState("");
   const { width: screenWidth } = useWindowSize();
   const [conversation, setConversation] = useState([]); // State to store conversation updates
-  const [audioBuffers, setAudioBuffers] = useState([]); // To store multiple ArrayBuffers
+  const [audioMessages, setAudioMessages] = useState([]); // To store multiple ArrayBuffers
   const socketRef = useRef(null); // Using useRef to persist socket instance
 
   const foodsContainerStyle = {
@@ -54,8 +54,11 @@ function Council({ options }) {
     });
 
     // Listen for audio updates
-    socketRef.current.on("audio_update", (audioData) => {
-      setAudioBuffers((prevBuffers) => [...prevBuffers, audioData.audio]);
+    socketRef.current.on("audio_update", (audioMessage) => {
+      setAudioMessages((prevAudioMessages) => [
+        ...prevAudioMessages,
+        audioMessage,
+      ]);
     });
 
     return () => {
@@ -115,7 +118,7 @@ function Council({ options }) {
           >
             <Output
               conversation={conversation}
-              audioBuffers={audioBuffers}
+              audioMessages={audioMessages}
             />
           </div>
         )}
diff --git a/client/src/components/Output.jsx b/client/src/components/Output.jsx
index 44114d3..5e1b391 100644
--- a/client/src/components/Output.jsx
+++ b/client/src/components/Output.jsx
@@ -3,7 +3,7 @@ import TextOutput from "./TextOutput";
 import AudioOutput from "./AudioOutput";
 import ConversationControls from "./ConversationControls";
 
-function Output({ conversation, audioBuffers }) {
+function Output({ conversation, audioMessages }) {
   const [currentMessageIndex, setCurrentMessageIndex] = useState(0);
   const [currentSnippetIndex, setCurrentSnippetIndex] = useState(0);
   const [currentMessageTextSnippet, setCurrentMessageTextSnippet] =
@@ -11,9 +11,9 @@ function Output({ conversation, audioBuffers }) {
   const [isPaused, setIsPaused] = useState(false);
 
   useEffect(() => {
-    console.log("Audio buffers length:", audioBuffers.length);
+    console.log("Audio buffers length:", audioMessages.length);
     console.log("Current audio buffer index:", currentMessageIndex);
-  }, [audioBuffers, currentMessageIndex]);
+  }, [audioMessages, currentMessageIndex]);
 
   useEffect(() => {
     if (conversation.length > 0 && !isPaused) {
@@ -35,7 +35,7 @@ function Output({ conversation, audioBuffers }) {
     }
   }, [currentMessageIndex, currentSnippetIndex, conversation, isPaused]); // Include isPaused in dependencies
 
-  const calculateDisplayTime = (text) => Math.max(3, text.length * 0.05);
+  const calculateDisplayTime = (text) => Math.max(3, text.length * 0.065);
 
   function handlePauseResume() {
     setIsPaused(!isPaused); // Toggle pause state
@@ -57,7 +57,7 @@ function Output({ conversation, audioBuffers }) {
   }
 
   return (
-    <div>
+    <div style={{ textAlign: "center", width: "75%" }}>
       <h2>
         Speaker:{" "}
         {conversation.length > 0
@@ -65,7 +65,11 @@ function Output({ conversation, audioBuffers }) {
           : ""}
       </h2>
       <TextOutput currentMessageTextSnippet={currentMessageTextSnippet} />
-      <AudioOutput currentAudioBuffer={audioBuffers[currentMessageIndex]} />
+      <AudioOutput
+        currentAudioMessage={audioMessages.find(
+          (a) => a.message_index == currentMessageIndex
+        )}
+      />
       <ConversationControls
         isPaused={isPaused}
         onPauseResume={handlePauseResume}