PoC-S33:coding work of data path: UI <----> JNI <----> whisper.cpp <----> kantv-play <----> kantv-core

Author: zhouwg

.gitignore

@@ -57,3 +57,6 @@ build
 
 cdeosplayer/kantv/src/main/assets/audio.wav
 cdeosplayer/kantv/src/main/assets/deepspeech-0.9.3-models.tflite
+
+#generated by source code,so skip it
+cdeosplayer/kantv/src/main/jniLibs/arm64-v8a/libopenblas.so

cdeosplayer/cdeosplayer-exo2/src/main/java/cdeos/media/exo2/CDEOSExo2MediaPlayer.java

@@ -503,6 +503,12 @@ public void setEnableRecord(boolean bEnableRecord) {
         CDEUtils.setTVRecording(bEnableRecord);
     }
 
+    //TODO
+    @Override
+    public void setEnableASR(boolean bEnableASR) {
+        CDEUtils.setTVASR(bEnableASR);
+    }
+
     //FIXME
     @Override
     public void updateRecordingStatus(boolean bEnableRecord) {
@@ -526,6 +532,12 @@ public boolean getEnableRecord() {
         return CDEUtils.getTVRecording();
     }
 
+    //TODO
+    @Override
+    public boolean getEnableASR() {
+        return CDEUtils.getTVASR();
+    }
+
 
     @Override
     public void setDurationDumpES(int durationDumpES) {

cdeosplayer/cdeosplayer-lib/src/main/java/cdeos/media/player/AndroidMediaPlayer.java

@@ -625,6 +625,11 @@ public void setEnableDumpAudioES(boolean bEnable) {
     public void setEnableRecord(boolean bEnableRecord) {
     }
 
+    //TODO
+    @Override
+    public void setEnableASR(boolean bEnableRecord) {
+    }
+
     //TODO
     @Override
     public void updateRecordingStatus(boolean bEnableRecord) {
@@ -636,6 +641,12 @@ public boolean getEnableRecord() {
         return false;
     }
 
+    //TODO
+    @Override
+    public boolean getEnableASR() {
+        return false;
+    }
+
     //TODO
     @Override
     public void setDurationDumpES(int durationDumpES) {

cdeosplayer/cdeosplayer-lib/src/main/java/cdeos/media/player/CDEAssetLoader.java

@@ -42,9 +42,14 @@ public static void copyAssetFile(Context context, String srcFilePath, String des
         FileOutputStream outStream = null;
         try {
             File destFile = new File(destFilePath);
+
+            if (destFile.exists())
+                return;
+
             if (!destFile.exists()) {
                 destFile.createNewFile();
             }
+
             inStream = context.getAssets().open(srcFilePath);
             outStream = new FileOutputStream(destFilePath);
 

cdeosplayer/cdeosplayer-lib/src/main/java/cdeos/media/player/CDEUtils.java

@@ -22,8 +22,10 @@
 
 import android.app.Activity;
 import android.app.ActivityManager;
+import android.app.AlertDialog;
 import android.bluetooth.BluetoothAdapter;
 import android.content.Context;
+import android.content.DialogInterface;
 import android.content.Intent;
 import android.graphics.Bitmap;
 import android.hardware.Sensor;
@@ -232,6 +234,7 @@ public class CDEUtils {
     private static int mRecordSize                  = 20; //default record size is 20 Mbytes
     private static int mRecordMode                  = 0; //0:video, 1:audio, 2:video&audio
     private static boolean mIsTVRecording           = false;
+    private static boolean mIsTVASR                 = false;
     private static String  mRecordingFileName       = "";
     private static boolean mEnableRecordVideoES     = true;
     private static boolean mEnableRecordAudioES     = false;
@@ -3664,6 +3667,14 @@ public static void setTVRecording(boolean bRecording) {
         mIsTVRecording = bRecording;
     }
 
+    public static boolean getTVASR() {
+        return mIsTVASR;
+    }
+
+    public static void setTVASR(boolean bASR) {
+        mIsTVASR = bASR;
+    }
+
 
     public static void setRecordDuration(int recordDuration) {
         mRecordDuration = recordDuration;
@@ -3824,5 +3835,16 @@ public static boolean isDisableUM(){
         return mDisableUM;
     }
 
+    public static void showMsgBox(Context context, String message) {
+        AlertDialog dialog = new AlertDialog.Builder(context).create();
+        dialog.setMessage(message);
+        dialog.setButton(DialogInterface.BUTTON_NEUTRAL, "OK", new DialogInterface.OnClickListener() {
+            public void onClick(DialogInterface dialog, int which) {
+
+            }
+        });
+        dialog.show();
+    }
+
     public static native int kantv_anti_tamper();
 }

cdeosplayer/cdeosplayer-lib/src/main/java/cdeos/media/player/FFmpegMediaPlayer.java

@@ -42,6 +42,8 @@
 import android.view.Surface;
 import android.view.SurfaceHolder;
 
+import org.ggml.whispercpp.whispercpp;
+
 import java.io.FileDescriptor;
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -179,13 +181,14 @@ public final class FFmpegMediaPlayer extends AbstractMediaPlayer {
     private long mTotalPauseTime;
     private boolean bIsPaused;
     private int mDecryptMode = CDEUtils.DECRYPT_SOFT;
-    private boolean mDisableAudioTrack = false; //troubleshooting Huaxia's <<狄仁杰>>
+    private boolean mDisableAudioTrack = false;
     private boolean mDisableVideoTrack = false;
     private boolean mEnableDumpVideoES = false;
     private boolean mEnableDumpAudioES = false;
     private int mDurationDumpES = 10; //default dumped duration is 10 seconds
 
     private boolean mEnableRecord = false;
+    private boolean mEnableASR    = false;
 
     private String mVideoTitle;
 
@@ -218,6 +221,7 @@ public static void loadLibrariesOnce(IjkLibLoader libLoader) {
                 CDELog.j(TAG, "load library kantv-play");
                 libLoader.loadLibrary("kantv-play");
                 CDELog.j(TAG, "after load library kantv-play");
+
                 mIsLibLoaded = true;
             }
         }
@@ -261,6 +265,15 @@ private void initPlayer(IjkLibLoader libLoader) {
         loadLibrariesOnce(libLoader);
         initNativeOnce();
 
+        //TODO
+        try {
+            CDELibraryLoader.load("whispercpp");
+            CDELog.j(TAG, "cpu core counts:" + whispercpp.get_cpu_core_counts());
+        } catch (Exception e) {
+            CDELog.j(TAG, "failed to initialize whispercpp jni");
+            return;
+        }
+
         //should I put it in another better place?
         mBeginTime = System.currentTimeMillis();
 
@@ -856,6 +869,7 @@ public void setEnableRecord(boolean bEnableRecord) {
         _setEnableRecord(mEnableRecord ? 1 : 0);
     }
 
+
     //TODO
     @Override
     public void updateRecordingStatus(boolean bEnableRecord) {
@@ -871,6 +885,23 @@ public boolean getEnableRecord() {
         return mEnableRecord;
     }
 
+    @Override
+    public void setEnableASR(boolean bEnableASR) {
+        mEnableASR = bEnableASR;
+        CDELog.d(TAG, "enable ASR: " + mEnableASR);
+        _setEnableASR(mEnableASR ? 1 : 0);
+    }
+
+    @Override
+    public boolean getEnableASR() {
+        int isASRing= _getEnableASR();
+        if (0 == isASRing)
+            mEnableASR = false;
+        else
+            mEnableASR = true;
+        return mEnableASR;
+    }
+
 
     @Override
     public void setDurationDumpES(int durationDumpES) {
@@ -918,6 +949,10 @@ public String getAudioDecoderName() {
 
     public native int _getEnableRecord();
 
+    public native void _setEnableASR(int bEnableASR);
+
+    public native int _getEnableASR();
+
     @Override
     public native void seekTo(long msec) throws IllegalStateException;
 
@@ -1296,7 +1331,7 @@ public void handleMessage(Message msg) {
                     // No real default action so far.
                     return;
                 case MEDIA_TIMED_TEXT:
-                    CDELog.j(TAG, "subtitle coming");
+                    CDELog.d(TAG, "subtitle coming");
                     if (msg.obj == null) {
                         player.notifyOnTimedText(null);
                     } else {

cdeosplayer/cdeosplayer-lib/src/main/java/cdeos/media/player/IMediaPlayer.java

@@ -141,6 +141,9 @@ void setDataSource(String path)
     void updateRecordingStatus(boolean bEnableRecord);
     boolean getEnableRecord();
 
+    void setEnableASR(boolean bEnableASR);
+    boolean getEnableASR();
+
     void setDurationDumpES(int durationDumpES);//per seconds
     String getVideoInfo();
     String getAudioInfo();

cdeosplayer/cdeosplayer-lib/src/main/java/cdeos/media/player/MediaPlayerProxy.java

@@ -173,24 +173,36 @@ public void setEnableDumpAudioES(boolean bEnable) {
         mBackEndMediaPlayer.setEnableDumpAudioES(bEnable);
     }
 
-    //TODO
+
     @Override
     public void setEnableRecord(boolean bEnableRecord) {
         mBackEndMediaPlayer.setEnableRecord(bEnableRecord);
     }
 
+
+    @Override
+    public void setEnableASR(boolean bEnableASR) {
+        mBackEndMediaPlayer.setEnableASR(bEnableASR);
+    }
+
     @Override
     public void updateRecordingStatus(boolean bEnableRecord) {
         mBackEndMediaPlayer.updateRecordingStatus(bEnableRecord);
     }
 
-    //TODO
+
     @Override
     public boolean getEnableRecord() {
         return false;
     }
 
 
+    @Override
+    public boolean getEnableASR() {
+        return false;
+    }
+
+
     @Override
     public void setDurationDumpES(int durationDumpES) {
         mBackEndMediaPlayer.setDurationDumpES(durationDumpES);

cdeosplayer/cdeosplayer-lib/src/main/java/org/ggml/whispercpp/whispercpp.java

@@ -6,6 +6,13 @@
 public class whispercpp {
   private static final String TAG = whispercpp.class.getName();
 
+  public static final int WHISPER_ASR_MODE_NORMAL       = 0;
+  public static final int WHISPER_ASR_MODE_PRESURETEST  = 1;
+
+  public static native void asr_init(String strModelPath, int nThreadCounts, int nDevMode);
+
+  public static native void asr_finalize();
+
   public static native String get_systeminfo();
 
   public static native int    get_cpu_core_counts();

cdeosplayer/constants.gradle

@@ -39,7 +39,8 @@ project.ext {
     compileSdkVersion   = 34
     buildToolsVersion   = "34.0.0"
     targetSdkVersion    = 34
-    minSdkVersion       = 26 //34 is too strict and the generated apk couldn't run on vivo Android phone
+    //34 is too strict and the generated apk couldn't run on low-end vivo Android phone,target phone should be above Android 8
+    minSdkVersion       = 26
     appTargetSdkVersion = 29
 
     //version and version code of KanTV

cdeosplayer/kantv/src/main/assets/config.json

@@ -1,6 +1,6 @@
 {
   "kantvServer": "www.cde-os.com",
-  "releaseMode": "1",
+  "releaseMode": "0",
   "apkVersion": "1.3.1",
   "apkForTV": "0"
 }

cdeosplayer/kantv/src/main/assets/ggml-tiny-q5_1.bin

@@ -43,6 +43,7 @@
 
 import butterknife.ButterKnife;
 import butterknife.Unbinder;
+import cdeos.media.player.CDEAssetLoader;
 import skin.support.SkinCompatManager;
 import skin.support.observe.SkinObservable;
 import skin.support.observe.SkinObserver;
@@ -367,6 +368,10 @@ public void initApp() {
         CDELog.j(TAG, "screen width x height (dp)   = : " + screenWidth + "x" + screenHeight);
         CDELog.j(TAG, "isRunningOnTV = " + CDEUtils.isRunningOnTV());
 
+        //for realtime subtitle demo with prebuilt android apk
+        String ggmlModelFileName = "ggml-tiny-q5_1.bin"; //31M
+        CDEAssetLoader.copyAssetFile(mContext, ggmlModelFileName, CDEUtils.getDataPath() + ggmlModelFileName);
+
         if (CDEUtils.isRunningOnTV()) {
             setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE);
             CDEUtils.setScreenOrientation(CDEUtils.SCREEN_ORIENTATION_LANDSCAPE);

cdeosplayer/kantv/src/main/java/com/cdeos/kantv/base/BaseAppCompatActivity.java

@@ -38,6 +38,8 @@ public class SettingPlayerView extends LinearLayout implements View.OnClickListe
 
     private Switch tvRecordingSw;
     private Switch landscapeChangeSw;
+    private Switch tvASRSw;
+
     private LinearLayout audioRl;
     private RadioGroup mAspectRatioOptions;
 
@@ -71,6 +73,7 @@ public SettingPlayerView(Context context, AttributeSet attrs) {
 
         landscapeChangeSw = this.findViewById(R.id.landscape_change_sw);
         tvRecordingSw     = this.findViewById(R.id.tv_recording_sw);
+        tvASRSw           = this.findViewById(R.id.tv_asr_sw);
 
         mAspectRatioOptions.setOnCheckedChangeListener((group, checkedId) -> {
             if (checkedId == R.id.aspect_fit_parent) {
@@ -149,7 +152,11 @@ public SettingPlayerView(Context context, AttributeSet attrs) {
         //    tvRecordingSw.setChecked(true);
         //else
          //   tvRecordingSw.setChecked(false);
-
+        tvASRSw.setOnCheckedChangeListener((buttonView, isChecked) -> {
+            CDELog.j(TAG, "tv ASR status:" + isChecked);
+            listener.setTVASRStatus(isChecked);
+        });
+        CDELog.j(TAG, "isASR enabled:" + CDEUtils.getTVASR());
         setPlayerSpeedView(3);
     }
 
@@ -160,6 +167,17 @@ public void updateUIStatus(boolean isRecording) {
             tvRecordingSw.setChecked(true);
         else
             tvRecordingSw.setChecked(false);
+
+    }
+
+    public void updateASRUIStatus(boolean isASR) {
+        CDELog.j(TAG, "isASR:" + isASR);
+        CDELog.j(TAG, "isASR:" + CDEUtils.getTVASR());
+
+        if (CDEUtils.getTVASR())
+            tvASRSw.setChecked(true);
+        else
+            tvASRSw.setChecked(false);
     }
 
     @Override
@@ -287,5 +305,7 @@ public interface SettingVideoListener {
         void setLandscapeStatus(boolean bLandScape);
 
         void setTVRecordingStatus(boolean bRecording);
+
+        void setTVASRStatus(boolean bEnableASR);
     }
 }