From 52b7d145fed6e0603e4cec636be2b53b5f6a6485 Mon Sep 17 00:00:00 2001 From: x270880x Date: Tue, 11 Aug 2026 09:48:54 +0300 Subject: [PATCH 1/3] Rebase surface PTS on the shared start instead of the first video frame MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In SURFACE mode VideoEncoder.checkBuffer rebases every frame on firstTimestamp — the PTS of the first frame this encoder produced. AudioEncoder.calculatePts rebases on presentTimeUs, the origin StreamBase hands to every encoder in startSources(). The two timelines therefore start at different moments: audio at the instant the stream started, video at the instant its first frame came out of the encoder. Whatever sits between them — opening the camera, warming up GL — becomes a constant offset, with the picture running ahead of the sound. Measured on a Samsung SM-A065F (Android 15, camera source, built-in mic, RTMP): video ran 470 ms ahead of audio, 18 clap pairs, spread 427-517 ms. Lips move, speech follows. Both TimestampMode branches perform the same rebase, so switching CLOCK/BUFFER makes no difference — verified by measuring both. Rebasing on presentTimeUs is safe here: GlStreamInterface stamps frames through GlTimestamp, which anchors the first frame to TimeUtils and clamps later frames to within one frame of that clock, and StreamBase reads the same TimeUtils to start the encoders. Both sides are on CLOCK_BOOTTIME microseconds, so the subtraction is meaningful and matches what the audio path already does. Non-surface paths are untouched, and the old behaviour still applies when the encoder was started without a shared origin. With the same test the 490 ms cluster disappears; what remains is inside the resolution of a clap measurement. --- .../com/pedro/encoder/video/VideoEncoder.java | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java b/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java index 880392611..cb1ada579 100644 --- a/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java +++ b/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java @@ -444,6 +444,30 @@ public void formatChanged(@NonNull MediaCodec mediaCodec, @NonNull MediaFormat m spsPpsSetted = sendSPSandPPS(mediaFormat); } + /** + * Surface mode: the EGL timestamp is a huge absolute value that would break RTMP, so it has to be + * rebased to something relative. Rebase it against the start the encoders were given, not against + * this encoder's own first frame. + * + * GlStreamInterface stamps every frame through GlTimestamp, which anchors to TimeUtils — the same + * clock StreamBase reads to start both encoders — so subtracting presentTimeUs is meaningful and + * mirrors what AudioEncoder.calculatePts already does. + * + * Rebasing on the first frame instead put the two timelines on different zeros: audio counted from + * the moment the stream started, video from the moment its first frame came out of the encoder. + * Everything in between — opening the camera, warming up GL — turned into a fixed offset with audio + * behind the picture. On a Samsung SM-A065F that offset measured 470 ms (18 clap pairs, 427-517 ms), + * plainly visible as lips moving before the sound. It is the same in both TimestampMode branches, + * which is why switching modes never helped. + * + * Falls back to the old behaviour when the encoder was started without a shared origin. + */ + private long rebaseSurfacePts(long ptsUs) { + if (presentTimeUs > 0) return Math.max(0, ptsUs - presentTimeUs); + if (firstTimestamp == 0) firstTimestamp = ptsUs; + return Math.max(0, ptsUs - firstTimestamp); + } + @Override protected boolean checkBuffer(@NonNull ByteBuffer byteBuffer, @NonNull MediaCodec.BufferInfo bufferInfo) { if (forceKey && Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) { @@ -510,12 +534,10 @@ protected boolean checkBuffer(@NonNull ByteBuffer byteBuffer, @NonNull MediaCode // Buffer mode: synthesize PTS from wall clock. bufferInfo.presentationTimeUs = TimeUtils.getCurrentTimeMicro() - presentTimeUs; } else { - // Surface mode: EGL timestamp is camera sensor time (nanoseconds from boot ÷ 1000). - // It has clean, jitter-free intervals — but it's a huge absolute value that breaks RTMP. - // Rebase to relative by subtracting the first frame's PTS → clean intervals, starts at 0. - if (firstTimestamp == 0) firstTimestamp = bufferInfo.presentationTimeUs; - bufferInfo.presentationTimeUs -= firstTimestamp; + bufferInfo.presentationTimeUs = rebaseSurfacePts(bufferInfo.presentationTimeUs); } + } else if (formatVideoEncoder == FormatVideoEncoder.SURFACE) { + bufferInfo.presentationTimeUs = rebaseSurfacePts(bufferInfo.presentationTimeUs); } else { if (firstTimestamp == 0) firstTimestamp = bufferInfo.presentationTimeUs; bufferInfo.presentationTimeUs -= firstTimestamp; From 4caaf7dffc4b1dffb277ba492f4c2c46b6c2e8d2 Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Tue, 11 Aug 2026 11:55:25 +0200 Subject: [PATCH 2/3] fix no gl renders pts --- .../com/pedro/encoder/video/VideoEncoder.java | 32 +++++++------------ 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java b/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java index cb1ada579..fab5df3fe 100644 --- a/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java +++ b/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java @@ -445,27 +445,19 @@ public void formatChanged(@NonNull MediaCodec mediaCodec, @NonNull MediaFormat m } /** - * Surface mode: the EGL timestamp is a huge absolute value that would break RTMP, so it has to be - * rebased to something relative. Rebase it against the start the encoders were given, not against - * this encoder's own first frame. - * - * GlStreamInterface stamps every frame through GlTimestamp, which anchors to TimeUtils — the same - * clock StreamBase reads to start both encoders — so subtracting presentTimeUs is meaningful and - * mirrors what AudioEncoder.calculatePts already does. - * - * Rebasing on the first frame instead put the two timelines on different zeros: audio counted from - * the moment the stream started, video from the moment its first frame came out of the encoder. - * Everything in between — opening the camera, warming up GL — turned into a fixed offset with audio - * behind the picture. On a Samsung SM-A065F that offset measured 470 ms (18 clap pairs, 427-517 ms), - * plainly visible as lips moving before the sound. It is the same in both TimestampMode branches, - * which is why switching modes never helped. - * - * Falls back to the old behaviour when the encoder was started without a shared origin. + * Surface mode: rebase the timestamp to something relative. The GlInterface stamps the surface + * with the TimeUtils clock, so we share the origin with the audio encoder. A surface fed directly + * (decoder in FromFileBase, MediaProjection in DisplayBase without opengl) uses another time base + * and must be rebased against its own first frame. + * Using 1 second to difference both ways. */ - private long rebaseSurfacePts(long ptsUs) { - if (presentTimeUs > 0) return Math.max(0, ptsUs - presentTimeUs); - if (firstTimestamp == 0) firstTimestamp = ptsUs; - return Math.max(0, ptsUs - firstTimestamp); + private long rebaseSurfacePts(long pts) { + if (firstTimestamp == 0) { + boolean sharedClock = presentTimeUs > 0 + && Math.abs(pts - TimeUtils.getCurrentTimeMicro()) < 1_000_000; + firstTimestamp = sharedClock ? presentTimeUs : pts; + } + return Math.max(0, pts - firstTimestamp); } @Override From 911fc236660598b20eadfb3aa59d99f8e7eac0b3 Mon Sep 17 00:00:00 2001 From: pedroSG94 Date: Tue, 11 Aug 2026 13:16:15 +0200 Subject: [PATCH 3/3] code tab --- .../src/main/java/com/pedro/encoder/video/VideoEncoder.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java b/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java index fab5df3fe..0a92ea746 100644 --- a/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java +++ b/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java @@ -453,8 +453,7 @@ public void formatChanged(@NonNull MediaCodec mediaCodec, @NonNull MediaFormat m */ private long rebaseSurfacePts(long pts) { if (firstTimestamp == 0) { - boolean sharedClock = presentTimeUs > 0 - && Math.abs(pts - TimeUtils.getCurrentTimeMicro()) < 1_000_000; + boolean sharedClock = presentTimeUs > 0 && Math.abs(pts - TimeUtils.getCurrentTimeMicro()) < 1_000_000; firstTimestamp = sharedClock ? presentTimeUs : pts; } return Math.max(0, pts - firstTimestamp);