chore: fix android rotation

AgoraIO-Extensions · Dec 17, 2024 · 2ce0107 · 2ce0107
1 parent 0a45342
commit 2ce0107
Show file tree

Hide file tree

Showing 5 changed files with 47 additions and 34 deletions.
diff --git a/ts/Decoder/index.ts b/ts/Decoder/index.ts
@@ -1,11 +1,7 @@
-import {
-  EncodedVideoFrameInfo,
-  VideoCodecType,
-  VideoFrameType,
-} from '../Private/AgoraBase';
+import { EncodedVideoFrameInfo, VideoFrameType } from '../Private/AgoraBase';
 
 import { WebCodecsRenderer } from '../Renderer/WebCodecsRenderer/index';
-import { RendererCacheContext, RendererType } from '../Types';
+import { CodecConfigInfo, RendererCacheContext, RendererType } from '../Types';
 import { AgoraEnv, logDebug, logInfo } from '../Utils';
 
 const frameTypeMapping = {
@@ -19,11 +15,7 @@ export class WebCodecsDecoder {
   private renderers: WebCodecsRenderer[] = [];
   private _cacheContext: RendererCacheContext;
   private pendingFrame: VideoFrame | null = null;
-  private _currentCodecConfig: {
-    codecType: VideoCodecType | undefined;
-    codedWidth: number | undefined;
-    codedHeight: number | undefined;
-  } | null = null;
+  private _currentCodecConfig: CodecConfigInfo | null = null;
 
   private _base_ts = 0;
   private _base_ts_ntp = 1;
@@ -68,7 +60,7 @@ export class WebCodecsDecoder {
       if (renderer.rendererType !== RendererType.WEBCODECSRENDERER) {
         continue;
       }
-      renderer.drawFrame(this.pendingFrame);
+      renderer.drawFrame(this.pendingFrame, this._currentCodecConfig!);
       this.pendingFrame = null;
     }
   }
@@ -91,6 +83,7 @@ export class WebCodecsDecoder {
       codecType: frameInfo.codecType,
       codedWidth: frameInfo.width,
       codedHeight: frameInfo.height,
+      rotation: frameInfo.rotation,
     };
     this._decoder!.configure({
       codec: codec,
@@ -122,7 +115,8 @@ export class WebCodecsDecoder {
     if (
       this._currentCodecConfig?.codecType !== frameInfo.codecType ||
       this._currentCodecConfig?.codedWidth !== frameInfo.width ||
-      this._currentCodecConfig?.codedHeight !== frameInfo.height
+      this._currentCodecConfig?.codedHeight !== frameInfo.height ||
+      this._currentCodecConfig?.rotation !== frameInfo.rotation
     ) {
       logInfo('frameInfo has changed, reconfigure decoder');
       this._decoder.reset();

diff --git a/ts/Renderer/IRenderer.ts b/ts/Renderer/IRenderer.ts
@@ -1,8 +1,6 @@
 import { VideoMirrorModeType } from '../Private/AgoraBase';
 import { RenderModeType, VideoFrame } from '../Private/AgoraMediaBase';
-import { RendererContext, RendererType } from '../Types';
-
-import { frameSize } from './WebCodecsRenderer';
+import { CodecConfigInfo, RendererContext, RendererType } from '../Types';
 
 export abstract class IRenderer {
   parentElement?: HTMLElement;
@@ -13,7 +11,7 @@ export abstract class IRenderer {
   private _frameCount = 0;
   private _startTime: number | null = null;
 
-  public bind(element: HTMLElement, _frameSize?: frameSize) {
+  public bind(element: HTMLElement) {
     this.parentElement = element;
     this.container = document.createElement('div');
     Object.assign(this.container.style, {
@@ -46,7 +44,10 @@ export abstract class IRenderer {
     this.parentElement = undefined;
   }
 
-  public drawFrame(_videoFrame?: VideoFrame): void {
+  public drawFrame(
+    _videoFrame?: VideoFrame,
+    _codecConfig?: CodecConfigInfo
+  ): void {
     if (!this.canvas) return;
     if (this.canvas.style.display !== '') {
       this.canvas.style.display = '';

diff --git a/ts/Renderer/WebCodecsRenderer/index.ts b/ts/Renderer/WebCodecsRenderer/index.ts
@@ -1,12 +1,8 @@
-import { RendererType } from '../../Types';
+import { VideoFrame } from '../../Private/AgoraMediaBase';
+import { CodecConfigInfo, RendererType } from '../../Types';
 import { getContextByCanvas } from '../../Utils';
 import { IRenderer } from '../IRenderer';
 
-export type frameSize = {
-  width: number;
-  height: number;
-};
-
 export class WebCodecsRenderer extends IRenderer {
   gl?: WebGLRenderingContext | WebGL2RenderingContext | null;
   // eslint-disable-next-line auto-import/auto-import
@@ -35,11 +31,9 @@ export class WebCodecsRenderer extends IRenderer {
     }
   `;
 
-  bind(element: HTMLElement, frameSize: frameSize) {
+  bind(element: HTMLElement) {
     super.bind(element);
     if (!this.canvas) return;
-    this.canvas.width = frameSize.width;
-    this.canvas.height = frameSize.height;
     this.offscreenCanvas = this.canvas.transferControlToOffscreen();
     this.gl = getContextByCanvas(this.offscreenCanvas);
     if (!this.gl) return;
@@ -105,11 +99,14 @@ export class WebCodecsRenderer extends IRenderer {
     );
   }
 
-  drawFrame(frame: any) {
+  drawFrame(frame: any, _codecConfig: CodecConfigInfo) {
     if (!this.offscreenCanvas || !frame) return;
-    this.offscreenCanvas.width = frame.displayWidth;
-    this.offscreenCanvas.height = frame.displayHeight;
     this.updateRenderMode();
+    this.rotateCanvas({
+      width: _codecConfig.codedWidth,
+      height: _codecConfig.codedHeight,
+      rotation: _codecConfig.rotation,
+    });
     if (!this.gl) return;
 
     if (this.gl) {
@@ -138,4 +135,21 @@ export class WebCodecsRenderer extends IRenderer {
     super.drawFrame();
     this.getFps();
   }
+
+  protected override rotateCanvas({ width, height, rotation }: VideoFrame) {
+    if (!this.offscreenCanvas || !this.canvas) return;
+
+    if (rotation === 0 || rotation === 180) {
+      this.offscreenCanvas.width = width!;
+      this.offscreenCanvas.height = height!;
+    } else if (rotation === 90 || rotation === 270) {
+      this.offscreenCanvas.height = width!;
+      this.offscreenCanvas.width = height!;
+    } else {
+      throw new Error(
+        `Invalid rotation: ${rotation}, only 0, 90, 180, 270 are supported`
+      );
+    }
+    this.canvas.style.transform += ` rotateZ(${rotation}deg)`;
+  }
 }
diff --git a/ts/Renderer/WebCodecsRendererCache.ts b/ts/Renderer/WebCodecsRendererCache.ts
@@ -49,10 +49,7 @@ export class WebCodecsRendererCache extends IRendererCache {
         if (renderer.rendererType !== RendererType.WEBCODECSRENDERER) {
           continue;
         }
-        renderer.bind(renderer.context.view, {
-          width: _data.videoEncodedFrameInfo.width!,
-          height: _data.videoEncodedFrameInfo.height!,
-        });
+        renderer.bind(renderer.context.view);
       }
 
       try {

diff --git a/ts/Types.ts b/ts/Types.ts
@@ -199,3 +199,10 @@ export const codecMapping: CodecMappingItem[] = [
   { codec: 'vp8', type: VideoCodecType.VideoCodecVp8, profile: 'vp8' },
   { codec: 'vp9', type: VideoCodecType.VideoCodecVp9, profile: 'vp9' },
 ];
+
+export interface CodecConfigInfo {
+  codecType: VideoCodecType | undefined;
+  codedWidth: number | undefined;
+  codedHeight: number | undefined;
+  rotation: number | undefined;
+}