appendBuffer by chunks using a readable stream

[gspinoza]

Hello,
I'm having problems playing video files from a readable stream. I did a simple test using the Fetch API and it works fine, but in my case I'm dealing with large encrypted files that are decrypted on the fly using rcloneInstance.File.createReadStream(fetchStreamFactory(vid_URL)) from rclone-js , which returns a readable stream. I think the issue might be how I'm handling the appending of the arrayBuffer. Looking at the logs I can see that the file info is successfully extracted and the MIMECodec is also correctly identified, but the video does not play.

My code:

async function playFile(vid_URL) {
  const mp4boxfile = MP4Box.createFile();
  const mediaSource = new MediaSource();
  let mimeCodec;
  let sourceBuffer;

  mp4boxfile.onReady = function(info) {
    console.log(info)
    let videpCoded = info.videoTracks[0].codec
    let trackCodec = info.audioTracks[0].codec
    mimeCodec = `video/mp4; codecs="${trackCodec} , ${videpCoded}"`

    const videoTracks = info.tracks.filter(track => track.type === "video");
    console.log('Video tracks filtered', { videoTracks });
    var trackIndex = 0

    if (videoTracks[trackIndex]) {
      const track = videoTracks[trackIndex];
      // start segmentation
      console.log('Segment options set', { trackId: track.id, nbSamples: 1000 });
      const options = { nbSamples: 1000}
      mp4boxfile.setSegmentOptions(track.id, options)
      const initSegs = mp4boxfile.initializeSegmentation()
      sourceBuffer = mediaSource.addSourceBuffer(mimeCodec);
      sourceBuffer.addEventListener("updateend", onInitAppended);

      initSegs.forEach((initSeg) => {
            console.log('Appended initialization segment', { initSeg }, initSeg);
            sourceBuffer.appendBuffer(initSeg.buffer);
      });
      mp4boxfile.start();
    }; 
  }

  mp4boxfile.onSegment = function (id, user, buffer) {
    console.log("Received segment on track "+id+" for object "+user+" with a length of "+buffer.byteLength);
    console.log('Received segment', { trackId: id, bufferByteLength: buffer.byteLength });
    if (sourceBuffer) {
      if (!sourceBuffer.updating) {
        sourceBuffer.appendBuffer(buffer);
        console.log('Appended media segment to SourceBuffer', { buffer });
      }
    }
  };

  function onSourceClose(e) {
    console.log("MediaSource closed!", "MSE closed");
  }

  function onInitAppended(e) {
    console.log('updateend event after appending init segment', { sourceBuffer });
    console.log(mediaSource.readyState, "open", "MSE opened after init append")
  }

  async function onSourceOpen(e) {
    const opts = {
      start: 0,
      end: undefined, 
      chunkSize: 1024 * 1024 * 4, 
    }
    
    // returns a readable stream
    const decryptedStream = await rcloneInstance.File.createReadStream(fetchStreamFactory(vid_URL), opts)

    const CHUNK_SIZE_THRESHOLD = 1024 * 1024 * 10;
    
    let offset = 0;
    let accumulatedBuffers = [];

    // process chunks
    for await (const chunk of decryptedStream) {
      const arrayBuffer = chunk;
      accumulatedBuffers.push(arrayBuffer);

      // Check if accumulated buffers size exceeds the threshold
      const accumulatedSize = accumulatedBuffers.reduce((acc, buf) => acc + buf.byteLength, 0);
      if (accumulatedSize >= CHUNK_SIZE_THRESHOLD) {
          // Concatenate accumulated buffers
          const concatenatedBuffer = concatenateArrayBuffers(accumulatedBuffers);
          console.log("concatenatedBuffer : " + concatenatedBuffer.byteLength)

          concatenatedBuffer.fileStart = offset;
          offset += concatenatedBuffer.byteLength; // update offset
          mp4boxfile.appendBuffer(concatenatedBuffer);
          console.log('Appended ArrayBuffer to MP4Box file', { offset: offset });
          // Reset accumulated buffers and offset
          accumulatedBuffers = [];
      }
    }

    // Append any remaining buffers
    if (accumulatedBuffers.length > 0) {
      const concatenatedBuffer = concatenateArrayBuffers(accumulatedBuffers);
      concatenatedBuffer.fileStart = offset;
      offset += concatenatedBuffer.byteLength;
      mp4boxfile.appendBuffer(concatenatedBuffer);
    }

  }

  mediaSource.addEventListener("sourceopen", onSourceOpen);
  mediaSource.addEventListener("sourceclose", onSourceClose);
  const blobUrl = URL.createObjectURL(mediaSource);
  setSrc(blobUrl); // sets URL for <video src={src} controls />
};

function concatenateArrayBuffers(chunks) {
  const totalLength = chunks.reduce((acc, chunk) => acc + chunk.length, 0);
  const concatenatedBuffer = new Uint8Array(totalLength);
  let offset = 0;
  for (const chunk of chunks) {
      concatenatedBuffer.set(chunk, offset);
      offset += chunk.length;
  }
  return concatenatedBuffer.buffer;
}