From a21ccf51582d988ef514344c248afb8dd8e85a25 Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Thu, 6 Aug 2026 00:41:49 +0200 Subject: [PATCH 1/4] fix gl video timestamp jitter --- .../pedro/library/view/GlStreamInterface.kt | 34 +++++++++++------ .../com/pedro/library/view/GlTimestamp.kt | 38 +++++++++++++++++++ .../java/com/pedro/library/view/OpenGlView.kt | 34 +++++++++++------ 3 files changed, 82 insertions(+), 24 deletions(-) create mode 100644 library/src/main/java/com/pedro/library/view/GlTimestamp.kt diff --git a/library/src/main/java/com/pedro/library/view/GlStreamInterface.kt b/library/src/main/java/com/pedro/library/view/GlStreamInterface.kt index 72d4d03d25..35fb87c038 100644 --- a/library/src/main/java/com/pedro/library/view/GlStreamInterface.kt +++ b/library/src/main/java/com/pedro/library/view/GlStreamInterface.kt @@ -96,6 +96,8 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener, private var previewViewPort: ViewPort? = null private var streamViewPort: ViewPort? = null private var surfaceHandlerThread: HandlerThread? = null + private val sync = Any() + private val glTimestamp = GlTimestamp() private val sensorRotationManager = SensorRotationManager(context, true, true) { orientation, isPortrait -> if (autoHandleOrientation && shouldHandleOrientation) { @@ -185,6 +187,7 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener, } override fun start() { + glTimestamp.reset() threadQueue.clear() executor?.shutdownNow() executor = null @@ -210,11 +213,14 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener, mainRender.getSurfaceTexture().setOnFrameAvailableListener(this) } forceRender.start { - executor?.execute { - try { - draw(true) - } catch (e: RuntimeException) { - renderErrorCallback?.onRenderError(e) ?: throw e + synchronized(sync) { + val timestamp = TimeUtils.getCurrentTimeNano() + executor?.execute { + try { + draw(true, timestamp) + } catch (e: RuntimeException) { + renderErrorCallback?.onRenderError(e) ?: throw e + } } } } @@ -249,7 +255,7 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener, mainRender.release() } - private fun draw(forced: Boolean) { + private fun draw(forced: Boolean, clockTimestamp: Long) { if (!isRunning) return val limitFps = fpsLimiter.limitFPS() if (!forced) forceRender.frameAvailable() @@ -272,7 +278,7 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener, mainRender.drawSource() surfaceManager.swapBuffer() } - val timestamp = TimeUtils.getCurrentTimeNano() + val timestamp = glTimestamp.getTimestamp(surfaceTexture.timestamp, clockTimestamp) val orientation = when (orientationForced) { OrientationForced.PORTRAIT -> true @@ -359,11 +365,14 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener, override fun onFrameAvailable(surfaceTexture: SurfaceTexture?) { if (!isRunning) return - executor?.execute { - try { - draw(false) - } catch (e: RuntimeException) { - renderErrorCallback?.onRenderError(e) ?: throw e + synchronized(sync) { + val timestamp = TimeUtils.getCurrentTimeNano() + executor?.execute { + try { + draw(false, timestamp) + } catch (e: RuntimeException) { + renderErrorCallback?.onRenderError(e) ?: throw e + } } } } @@ -538,6 +547,7 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener, } override fun forceFpsLimit(fps: Int) { + glTimestamp.setFps(fps) fpsLimiter.setFPS(fps) } diff --git a/library/src/main/java/com/pedro/library/view/GlTimestamp.kt b/library/src/main/java/com/pedro/library/view/GlTimestamp.kt new file mode 100644 index 0000000000..9026caf0de --- /dev/null +++ b/library/src/main/java/com/pedro/library/view/GlTimestamp.kt @@ -0,0 +1,38 @@ +package com.pedro.library.view + +class GlTimestamp { + + private var drift = 20_000_000L + private var lastSourceTimestamp = 0L + private var lastClockTimestamp = 0L + private var currentTimestamp = 0L + + fun setFps(fps: Int) { + drift = 3000_000_000L / fps //drift of 3 frames. + } + + fun getTimestamp(sourceTimestamp: Long, clockTimestamp: Long): Long { + if (lastClockTimestamp == 0L) { + lastSourceTimestamp = sourceTimestamp + lastClockTimestamp = clockTimestamp + currentTimestamp = clockTimestamp + return currentTimestamp + } + val delta = if (sourceTimestamp <= lastSourceTimestamp) { + clockTimestamp - lastClockTimestamp + } else sourceTimestamp - lastSourceTimestamp + lastSourceTimestamp = sourceTimestamp + lastClockTimestamp = clockTimestamp + currentTimestamp += delta + + if (currentTimestamp > clockTimestamp + drift) currentTimestamp = clockTimestamp + drift + else if (currentTimestamp < clockTimestamp - drift) currentTimestamp = clockTimestamp - drift + return currentTimestamp + } + + fun reset() { + lastSourceTimestamp = 0 + lastClockTimestamp = 0 + currentTimestamp = 0 + } +} \ No newline at end of file diff --git a/library/src/main/java/com/pedro/library/view/OpenGlView.kt b/library/src/main/java/com/pedro/library/view/OpenGlView.kt index 4824ec216d..e3902be5d4 100644 --- a/library/src/main/java/com/pedro/library/view/OpenGlView.kt +++ b/library/src/main/java/com/pedro/library/view/OpenGlView.kt @@ -78,6 +78,8 @@ open class OpenGlView : SurfaceView, GlInterface, OnFrameAvailableListener, Surf private val forceRenderer = ForceRenderer() private var renderErrorCallback: RenderErrorCallback? = null private var surfaceHandlerThread: HandlerThread? = null + private val sync = Any() + private val glTimestamp = GlTimestamp() constructor(context: Context?) : super(context) { holder.addCallback(this) @@ -141,6 +143,7 @@ open class OpenGlView : SurfaceView, GlInterface, OnFrameAvailableListener, Surf } override fun forceFpsLimit(fps: Int) { + glTimestamp.setFps(fps) fpsLimiter.setFPS(fps) } @@ -212,7 +215,7 @@ open class OpenGlView : SurfaceView, GlInterface, OnFrameAvailableListener, Surf this.takePhotoCallback = takePhotoCallback } - private fun draw(forced: Boolean) { + private fun draw(forced: Boolean, clockTimestamp: Long) { if (!isRunning) return val limitFps = fpsLimiter.limitFPS() if (!forced) forceRenderer.frameAvailable() @@ -242,7 +245,7 @@ open class OpenGlView : SurfaceView, GlInterface, OnFrameAvailableListener, Surf } surfaceManager.swapBuffer() } - val timestamp = TimeUtils.getCurrentTimeNano() + val timestamp = glTimestamp.getTimestamp(surfaceTexture.timestamp, clockTimestamp) if (surfaceManagerEncoder.isReady || surfaceManagerEncoderRecord.isReady || surfaceManagerPhoto.isReady) { mainRender.drawFilters(false) @@ -310,6 +313,7 @@ open class OpenGlView : SurfaceView, GlInterface, OnFrameAvailableListener, Surf } override fun start() { + glTimestamp.reset() threadQueue.clear() executor?.shutdownNow() executor = null @@ -332,11 +336,14 @@ open class OpenGlView : SurfaceView, GlInterface, OnFrameAvailableListener, Surf mainRender.getSurfaceTexture().setOnFrameAvailableListener(this) } forceRenderer.start { - executor?.execute { - try { - draw(true) - } catch (e: RuntimeException) { - renderErrorCallback?.onRenderError(e) ?: throw e + synchronized(sync) { + val timestamp = TimeUtils.getCurrentTimeNano() + executor?.execute { + try { + draw(true, timestamp) + } catch (e: RuntimeException) { + renderErrorCallback?.onRenderError(e) ?: throw e + } } } } @@ -367,11 +374,14 @@ open class OpenGlView : SurfaceView, GlInterface, OnFrameAvailableListener, Surf override fun onFrameAvailable(surfaceTexture: SurfaceTexture) { if (!isRunning) return - executor?.execute { - try { - draw(false) - } catch (e: RuntimeException) { - renderErrorCallback?.onRenderError(e) ?: throw e + synchronized(sync) { + val timestamp = TimeUtils.getCurrentTimeNano() + executor?.execute { + try { + draw(false, timestamp) + } catch (e: RuntimeException) { + renderErrorCallback?.onRenderError(e) ?: throw e + } } } } From f1693b3e47a4bd2bf05acd4a385633b4b4cc0260 Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Thu, 6 Aug 2026 01:22:56 +0200 Subject: [PATCH 2/4] change drift --- .../src/main/java/com/pedro/library/view/GlTimestamp.kt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/library/src/main/java/com/pedro/library/view/GlTimestamp.kt b/library/src/main/java/com/pedro/library/view/GlTimestamp.kt index 9026caf0de..707bd2e747 100644 --- a/library/src/main/java/com/pedro/library/view/GlTimestamp.kt +++ b/library/src/main/java/com/pedro/library/view/GlTimestamp.kt @@ -1,14 +1,19 @@ package com.pedro.library.view +import kotlin.math.min + class GlTimestamp { - private var drift = 20_000_000L + companion object { + private const val DEFAULT_DRIFT = 100_000_000L + } + private var drift = DEFAULT_DRIFT private var lastSourceTimestamp = 0L private var lastClockTimestamp = 0L private var currentTimestamp = 0L fun setFps(fps: Int) { - drift = 3000_000_000L / fps //drift of 3 frames. + drift = if (fps <= 0) DEFAULT_DRIFT else min(DEFAULT_DRIFT, 1000_000_000L / fps) } fun getTimestamp(sourceTimestamp: Long, clockTimestamp: Long): Long { From 6b100ee9750afc2503710b494d134ae5d5e566c6 Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Thu, 6 Aug 2026 02:36:30 +0200 Subject: [PATCH 3/4] fix fps limiter using timestamp --- .../pedro/encoder/input/video/FpsLimiter.java | 30 +++++++------------ .../com/pedro/encoder/video/VideoEncoder.java | 2 +- .../pedro/library/view/GlStreamInterface.kt | 3 +- .../java/com/pedro/library/view/OpenGlView.kt | 12 ++++---- 4 files changed, 19 insertions(+), 28 deletions(-) diff --git a/encoder/src/main/java/com/pedro/encoder/input/video/FpsLimiter.java b/encoder/src/main/java/com/pedro/encoder/input/video/FpsLimiter.java index 519d92a8a7..f4e87a9953 100644 --- a/encoder/src/main/java/com/pedro/encoder/input/video/FpsLimiter.java +++ b/encoder/src/main/java/com/pedro/encoder/input/video/FpsLimiter.java @@ -24,10 +24,8 @@ public class FpsLimiter { - private long startTS = TimeUtils.getCurrentTimeMillis(); - private long ratioF = 1000 / 30; - private long ratio = 1000 / 30; - private long frameStartTS = 0; + private long interval = 1_000_000_000L / 30; + private long time = 0; private boolean configured = false; public void setFPS(int fps) { @@ -37,26 +35,18 @@ public void setFPS(int fps) { } else { configured = true; } - startTS = TimeUtils.getCurrentTimeMillis(); - ratioF = 1000 / fps; - ratio = 1000 / fps; + interval = 1_000_000_000L / fps; } - public boolean limitFPS() { + public boolean limitFPS(long timestamp) { if (!configured) return false; - long lastFrameTimestamp = TimeUtils.getCurrentTimeMillis() - startTS; - if (ratio < lastFrameTimestamp) { - ratio += ratioF; + if (time == 0) { + time = timestamp; return false; } - return true; - } - - public void setFrameStartTs() { - frameStartTS = TimeUtils.getCurrentTimeMillis(); - } - - public long getSleepTime() { - return Math.max(0, ratioF - (TimeUtils.getCurrentTimeMillis() - frameStartTS)); + if (timestamp - time < interval - (interval / 4)) return true; + time += interval; + if (timestamp - time > interval) time = timestamp; + return false; } } diff --git a/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java b/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java index d5d0843e49..a41c24aa98 100644 --- a/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java +++ b/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java @@ -414,7 +414,7 @@ protected MediaCodecInfo chooseEncoder(String mime) { protected Frame getInputFrame() throws InterruptedException { Frame frame = queue.take(); if (frame == null) return null; - if (fpsLimiter.limitFPS()) return getInputFrame(); + if (fpsLimiter.limitFPS(frame.getTimeStamp() * 1000)) return getInputFrame(); byte[] buffer = frame.getBuffer(); boolean isYV12 = frame.getFormat() == ImageFormat.YV12; diff --git a/library/src/main/java/com/pedro/library/view/GlStreamInterface.kt b/library/src/main/java/com/pedro/library/view/GlStreamInterface.kt index 35fb87c038..fd318e2270 100644 --- a/library/src/main/java/com/pedro/library/view/GlStreamInterface.kt +++ b/library/src/main/java/com/pedro/library/view/GlStreamInterface.kt @@ -257,7 +257,6 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener, private fun draw(forced: Boolean, clockTimestamp: Long) { if (!isRunning) return - val limitFps = fpsLimiter.limitFPS() if (!forced) forceRender.frameAvailable() if (!filterQueue.isEmpty() && mainRender.isReady()) { @@ -276,9 +275,9 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener, if (!surfaceManager.makeCurrent()) return mainRender.updateFrame() mainRender.drawSource() - surfaceManager.swapBuffer() } val timestamp = glTimestamp.getTimestamp(surfaceTexture.timestamp, clockTimestamp) + val limitFps = fpsLimiter.limitFPS(timestamp) val orientation = when (orientationForced) { OrientationForced.PORTRAIT -> true diff --git a/library/src/main/java/com/pedro/library/view/OpenGlView.kt b/library/src/main/java/com/pedro/library/view/OpenGlView.kt index e3902be5d4..1235e31e96 100644 --- a/library/src/main/java/com/pedro/library/view/OpenGlView.kt +++ b/library/src/main/java/com/pedro/library/view/OpenGlView.kt @@ -217,7 +217,6 @@ open class OpenGlView : SurfaceView, GlInterface, OnFrameAvailableListener, Surf private fun draw(forced: Boolean, clockTimestamp: Long) { if (!isRunning) return - val limitFps = fpsLimiter.limitFPS() if (!forced) forceRenderer.frameAvailable() if (!filterQueue.isEmpty() && mainRender.isReady()) { @@ -231,21 +230,24 @@ open class OpenGlView : SurfaceView, GlInterface, OnFrameAvailableListener, Surf return } } - if (surfaceManager.isReady && mainRender.isReady()) { if (!surfaceManager.makeCurrent()) return mainRender.updateFrame() mainRender.drawSource() - if (!limitFps) { + } + val timestamp = glTimestamp.getTimestamp(surfaceTexture.timestamp, clockTimestamp) + val limitFps = fpsLimiter.limitFPS(timestamp) + + if (surfaceManager.isReady && mainRender.isReady() && !limitFps) { + if (surfaceManager.makeCurrent()) { mainRender.drawFilters(true) mainRender.drawScreen( previewWidth, previewHeight, aspectRatioMode, 0, isPreviewVerticalFlip, isPreviewHorizontalFlip, null ) + surfaceManager.swapBuffer() } - surfaceManager.swapBuffer() } - val timestamp = glTimestamp.getTimestamp(surfaceTexture.timestamp, clockTimestamp) if (surfaceManagerEncoder.isReady || surfaceManagerEncoderRecord.isReady || surfaceManagerPhoto.isReady) { mainRender.drawFilters(false) From cc2d82e07e6cd73484eb2c328a05ea2c1ab4c6fd Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Thu, 6 Aug 2026 02:54:45 +0200 Subject: [PATCH 4/4] fix camera1 fps --- .../java/com/pedro/encoder/input/video/Camera1ApiManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/encoder/src/main/java/com/pedro/encoder/input/video/Camera1ApiManager.java b/encoder/src/main/java/com/pedro/encoder/input/video/Camera1ApiManager.java index 0b614802a4..7b7f6d2923 100644 --- a/encoder/src/main/java/com/pedro/encoder/input/video/Camera1ApiManager.java +++ b/encoder/src/main/java/com/pedro/encoder/input/video/Camera1ApiManager.java @@ -189,7 +189,7 @@ private void start() { Camera.Parameters parameters = camera.getParameters(); parameters.setPreviewSize(width, height); parameters.setPreviewFormat(imageFormat); - int[] range = FpsUtils.INSTANCE.adaptFpsRange(fps, parameters.getSupportedPreviewFpsRange()); + int[] range = FpsUtils.INSTANCE.adaptFpsRange(fps * 1000, parameters.getSupportedPreviewFpsRange()); Log.i(TAG, "fps: " + range[0] + " - " + range[1]); parameters.setPreviewFpsRange(range[0], range[1]);