Match framerate in DefaultVideoFrameProcessorPipeline with input stream (#3028)

Co-authored-by: Shi Su <[email protected]>

Author: shi-su

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 ### Fixed
+- Match framerate in `DefaultVideoFrameProcessorPipeline` with input stream
 
 ## [3.26.0] - 2024-10-07
 

docs/classes/defaultvideoframeprocessorpipeline.html

@@ -237,7 +237,7 @@ <h3>processors</h3>
 							<aside class="tsd-sources">
 								<p>Implementation of <a href="../interfaces/videoframeprocessorpipeline.html">VideoFrameProcessorPipeline</a>.<a href="../interfaces/videoframeprocessorpipeline.html#processors">processors</a></p>
 								<ul>
-									<li>Defined in <a href="https://github.com/aws/amazon-chime-sdk-js/blob/main/src/videoframeprocessor/DefaultVideoFrameProcessorPipeline.ts#L193">src/videoframeprocessor/DefaultVideoFrameProcessorPipeline.ts:193</a></li>
+									<li>Defined in <a href="https://github.com/aws/amazon-chime-sdk-js/blob/main/src/videoframeprocessor/DefaultVideoFrameProcessorPipeline.ts#L194">src/videoframeprocessor/DefaultVideoFrameProcessorPipeline.ts:194</a></li>
 								</ul>
 							</aside>
 							<div class="tsd-comment tsd-typography">
@@ -251,7 +251,7 @@ <h4 class="tsd-returns-title">Returns <a href="../interfaces/videoframeprocessor
 							<aside class="tsd-sources">
 								<p>Implementation of <a href="../interfaces/videoframeprocessorpipeline.html">VideoFrameProcessorPipeline</a>.<a href="../interfaces/videoframeprocessorpipeline.html#processors">processors</a></p>
 								<ul>
-									<li>Defined in <a href="https://github.com/aws/amazon-chime-sdk-js/blob/main/src/videoframeprocessor/DefaultVideoFrameProcessorPipeline.ts#L189">src/videoframeprocessor/DefaultVideoFrameProcessorPipeline.ts:189</a></li>
+									<li>Defined in <a href="https://github.com/aws/amazon-chime-sdk-js/blob/main/src/videoframeprocessor/DefaultVideoFrameProcessorPipeline.ts#L190">src/videoframeprocessor/DefaultVideoFrameProcessorPipeline.ts:190</a></li>
 								</ul>
 							</aside>
 							<div class="tsd-comment tsd-typography">
@@ -380,7 +380,7 @@ <h3>process</h3>
 						<li class="tsd-description">
 							<aside class="tsd-sources">
 								<ul>
-									<li>Defined in <a href="https://github.com/aws/amazon-chime-sdk-js/blob/main/src/videoframeprocessor/DefaultVideoFrameProcessorPipeline.ts#L197">src/videoframeprocessor/DefaultVideoFrameProcessorPipeline.ts:197</a></li>
+									<li>Defined in <a href="https://github.com/aws/amazon-chime-sdk-js/blob/main/src/videoframeprocessor/DefaultVideoFrameProcessorPipeline.ts#L198">src/videoframeprocessor/DefaultVideoFrameProcessorPipeline.ts:198</a></li>
 								</ul>
 							</aside>
 							<h4 class="tsd-parameters-title">Parameters</h4>

src/videoframeprocessor/DefaultVideoFrameProcessorPipeline.ts

@@ -144,6 +144,7 @@ export default class DefaultVideoFrameProcessorPipeline implements VideoFramePro
     this.inputVideoStream = inputMediaStream;
     const settings = this.inputVideoStream.getVideoTracks()[0].getSettings();
     this.logger.info(`processing pipeline input stream settings ${JSON.stringify(settings)}`);
+    this.framerate = settings.frameRate;
     this.canvasOutput.width = settings.width;
     this.canvasOutput.height = settings.height;
     this.videoInput.addEventListener('loadedmetadata', this.process);

test/dommock/DOMMockBuilder.ts

@@ -17,6 +17,7 @@ import UserMediaState from './UserMediaState';
 
 export interface StoppableMediaStreamTrack extends MediaStreamTrack {
   streamDeviceID: string | undefined;
+  framerate: number;
 
   // This stops the track _and dispatches 'ended'_.
   // https://stackoverflow.com/a/55960232
@@ -30,6 +31,9 @@ export interface StoppableMediaStreamTrack extends MediaStreamTrack {
 
   // Tracks know their source device ID. This lets us fake it in tests.
   setStreamDeviceID(id: string | undefined): void;
+
+  // Tracks know their framerate as a setting, this lets us fake it in tests.
+  setFramerate(framerate: number): void;
 }
 
 // eslint-disable-next-line
@@ -287,6 +291,7 @@ export default class DOMMockBuilder {
       readyState: MediaStreamTrackState = 'live';
 
       streamDeviceID = mockBehavior.mediaStreamTrackSettings?.deviceId || uuidv1();
+      framerate = 15;
 
       readonly id: string;
       readonly kind: string = '';
@@ -364,6 +369,10 @@ export default class DOMMockBuilder {
         this.streamDeviceID = id;
       }
 
+      setFramerate(framerate: number): void {
+        this.framerate = framerate;
+      }
+
       // This stops the track _and dispatches 'ended'_.
       // https://stackoverflow.com/a/55960232
       externalStop(): void {
@@ -397,6 +406,7 @@ export default class DOMMockBuilder {
           height: mockBehavior.mediaStreamTrackSettings.height,
           facingMode: mockBehavior.mediaStreamTrackSettings.facingMode,
           groupId: mockBehavior.mediaStreamTrackSettings.groupId,
+          frameRate: this.framerate,
         };
       }
 

test/videoframeprocessor/DefaultVideoFrameProcessorPipeline.test.ts

@@ -14,7 +14,7 @@ import VideoFrameProcessor from '../../src/videoframeprocessor/VideoFrameProcess
 import VideoFrameProcessorPipelineObserver from '../../src/videoframeprocessor/VideoFrameProcessorPipelineObserver';
 import VideoFrameProcessorTimer from '../../src/videoframeprocessor/VideoFrameProcessorTimer';
 import DOMMockBehavior from '../dommock/DOMMockBehavior';
-import DOMMockBuilder from '../dommock/DOMMockBuilder';
+import DOMMockBuilder, { StoppableMediaStreamTrack } from '../dommock/DOMMockBuilder';
 
 /**
  * [[MockVideoFrameProcessorTimer]] uses just `setTimeout` to avoid the complexity of
@@ -40,7 +40,7 @@ describe('DefaultVideoFrameProcessorPipeline', () => {
   let domMockBehavior: DOMMockBehavior;
   let domMockBuilder: DOMMockBuilder;
   let mockVideoStream: MediaStream;
-  let mockVideoTrack: MediaStreamTrack;
+  let mockVideoTrack: StoppableMediaStreamTrack;
   let proc: VideoFrameProcessor;
   const mockTimer = new MockVideoFrameProcessorTimer();
 
@@ -73,8 +73,11 @@ describe('DefaultVideoFrameProcessorPipeline', () => {
     mockVideoStream = new MediaStream();
     // @ts-ignore
     mockVideoStream.id = mockStreamId;
-    // @ts-ignore
-    mockVideoTrack = new MediaStreamTrack('attach-media-input-task-video-track-id', 'video');
+    mockVideoTrack = new MediaStreamTrack(
+      // @ts-ignore
+      'attach-media-input-task-video-track-id',
+      'video'
+    ) as StoppableMediaStreamTrack;
     mockVideoStream.addTrack(mockVideoTrack);
     proc = new NoOpVideoFrameProcessor();
     pipe = new DefaultVideoFrameProcessorPipeline(logger, [proc], mockTimer);

test/videoframeprocessor/DefaultVideoTransformDevice.test.ts

@@ -22,7 +22,7 @@ import VideoFxProcessor from '../../src/videofx/VideoFxProcessor';
 import MockEngineWorker from '../../test/videofx/MockEngineWorker';
 import MockFxLib from '../../test/videofx/MockFxLib';
 import DOMMockBehavior from '../dommock/DOMMockBehavior';
-import DOMMockBuilder from '../dommock/DOMMockBuilder';
+import DOMMockBuilder, { StoppableMediaStreamTrack } from '../dommock/DOMMockBuilder';
 
 describe('DefaultVideoTransformDevice', () => {
   const assert: Chai.AssertStatic = chai.assert;
@@ -31,7 +31,7 @@ describe('DefaultVideoTransformDevice', () => {
   let domMockBehavior: DOMMockBehavior;
   let domMockBuilder: DOMMockBuilder;
   let mockVideoStream: MediaStream;
-  let mockVideoTrack: MediaStreamTrack;
+  let mockVideoTrack: StoppableMediaStreamTrack;
 
   class MockObserver implements DefaultVideoTransformDeviceObserver {
     processingDidStart = sinon.stub();
@@ -77,7 +77,7 @@ describe('DefaultVideoTransformDevice', () => {
     // @ts-ignore
     mockVideoStream.id = 'sample';
     // @ts-ignore
-    mockVideoTrack = new MediaStreamTrack('test', 'video');
+    mockVideoTrack = new MediaStreamTrack('test', 'video') as StoppableMediaStreamTrack;
     mockVideoStream.addTrack(mockVideoTrack);
   });