pedroSG94
muxer and aac data problem
Hello pedro,
i'm using your library and i would like to change the default muxer (editing code in RecordController, in private void write(int track, ByteBuffer byteBuffer, MediaCodec.BufferInfo info) .
I have tried with jcodec and with ffmpeg, but i have always problem with aac data: the mp4 output file has audio plying too fast than video and i don't understand why. Should it do some kind of editing on aac bytebuffer (except adding adts header that i do)?
Thank you.
Probably your problem is related with timestamp. Make sure that your timestamp is correct. You can get it from info.presentationTimeUs in micro seconds. While you are recording, video and audio should have a similar timestamp to sync it or you will have that result.
How could i create a timestamp? Only with System.currentTimeInMillis()?
Because i have problems using the presentationTimeUs from bufferInfo in RecordController.java.
It looks like that, on your RecordController class, audio data is not sync with video data (in fact the presentationTimeUs are of 6 digits for audio and 8 digits for video ).
I have edited your RecordController like this
import android.content.Context; import android.media.MediaCodec; import android.media.MediaFormat; import android.os.Build; import android.util.Log;
import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; import com.pedro.encoder.utils.CodecUtil; import com.pedro.rtsp.utils.RtpConstants;
import org.jcodec.common.io.SeekableByteChannel;
import java.io.FileDescriptor; import java.io.IOException; import java.nio.ByteBuffer;
/**
* Created by pedro on 08/03/19.
*
* Class to control video recording with MediaMuxer.
*/
public class RecordController {
private static final String TAG = "RecordController";
private Status status = Status.STOPPED;
private JCodecMuxer jCodecMuxer;
private Listener listener;
//Pause/Resume
private long pauseMoment = 0;
private long pauseTime = 0;
private final MediaCodec.BufferInfo videoInfo = new MediaCodec.BufferInfo();
private final MediaCodec.BufferInfo audioInfo = new MediaCodec.BufferInfo();
private String videoMime = CodecUtil.H264_MIME;
private boolean isOnlyAudio = false;
private boolean isOnlyVideo = false;
private long startRecordTime = 0;
public Context context;
public enum Status {
STARTED, STOPPED, RECORDING, PAUSED, RESUMED
}
public interface Listener {
void onStatusChange(Status status);
}
@RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2)
public void startRecord(@NonNull String path, @Nullable Listener listener,int width,int height) throws IOException {
jCodecMuxer = new JCodecMuxer(path,width,height);
jCodecMuxer.context = context;
this.listener = listener;
status = Status.STARTED;
if (listener != null) listener.onStatusChange(status);
init();
startRecordTime = System.currentTimeMillis();
}
@RequiresApi(api = Build.VERSION_CODES.O)
public void startRecord(@NonNull FileDescriptor fd, @Nullable Listener listener) throws IOException {
// mediaMuxer = new MediaMuxer(fd, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
// this.listener = listener;
// status = Status.STARTED;
// if(listener != null) listener.onStatusChange(status);
// if(isOnlyAudio && audioFormat != null) init();
}
@RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2)
public void stopRecord() {
status = Status.STOPPED;
if (jCodecMuxer != null) {
try {
jCodecMuxer.finish();
} catch (Exception ignored) {
}
}
pauseMoment = 0;
pauseTime = 0;
if (listener != null) listener.onStatusChange(status);
}
public void setVideoMime(String videoMime) {
this.videoMime = videoMime;
}
public boolean isRunning() {
return status == Status.STARTED
|| status == Status.RECORDING
|| status == Status.RESUMED
|| status == Status.PAUSED;
}
public boolean isRecording() {
return status == Status.RECORDING;
}
public Status getStatus() {
return status;
}
public void resetFormats() {
}
public void pauseRecord() {
if (status == Status.RECORDING) {
pauseMoment = System.nanoTime() / 1000;
status = Status.PAUSED;
if (listener != null) listener.onStatusChange(status);
}
}
public void resumeRecord() {
if (status == Status.PAUSED) {
pauseTime += System.nanoTime() / 1000 - pauseMoment;
status = Status.RESUMED;
if (listener != null) listener.onStatusChange(status);
}
}
private boolean isKeyFrame(ByteBuffer videoBuffer) {
byte[] header = new byte[5];
videoBuffer.duplicate().get(header, 0, header.length);
if (videoMime.equals(CodecUtil.H264_MIME) && (header[4] & 0x1F) == RtpConstants.IDR) { //h264
return true;
} else { //h265
return videoMime.equals(CodecUtil.H265_MIME)
&& ((header[4] >> 1) & 0x3f) == RtpConstants.IDR_W_DLP
|| ((header[4] >> 1) & 0x3f) == RtpConstants.IDR_N_LP;
}
}
@RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2)
private void write(int track, ByteBuffer byteBuffer, MediaCodec.BufferInfo info) {
try {
if (track == 0){
jCodecMuxer.encodeVideoByteBuffer(byteBuffer,info);
}
else{
jCodecMuxer.encodeAudioByteBuffer(byteBuffer,info);
}
} catch (IllegalStateException | IllegalArgumentException e) {
Log.i(TAG, "Write error", e);
}
}
@RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2)
private void init() {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) {
jCodecMuxer.init();
}
status = Status.RECORDING;
if (listener != null) listener.onStatusChange(status);
}
@RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2)
public void recordVideo(ByteBuffer videoBuffer, MediaCodec.BufferInfo videoInfo) {
if (status == Status.STARTED ) {
if (videoInfo.flags == MediaCodec.BUFFER_FLAG_KEY_FRAME || isKeyFrame(videoBuffer)) {
// videoTrack = mediaMuxer.addTrack(videoFormat);
// init();
}
}else if (status == Status.RESUMED && (videoInfo.flags == MediaCodec.BUFFER_FLAG_KEY_FRAME
|| isKeyFrame(videoBuffer))) {
status = Status.RECORDING;
if (listener != null) listener.onStatusChange(status);
}
if (status == Status.RECORDING) {
updateFormat(this.videoInfo, videoInfo);
write(0, videoBuffer, this.videoInfo);
}
}
@RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2)
public void recordAudio(ByteBuffer audioBuffer, MediaCodec.BufferInfo audioInfo) {
if (status == Status.RECORDING) {
updateFormat(this.audioInfo, audioInfo);
write(1, audioBuffer, this.audioInfo);
}
}
public void setVideoFormat(MediaFormat videoFormat, boolean isOnlyVideo) {
this.isOnlyVideo = isOnlyVideo;
}
public void setAudioFormat(MediaFormat audioFormat, boolean isOnlyAudio) {
this.isOnlyAudio = isOnlyAudio;
if (isOnlyAudio && status == Status.STARTED
&& Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR2) {
init();
}
}
public void setVideoFormat(MediaFormat videoFormat) {
setVideoFormat(videoFormat, false);
}
public void setAudioFormat(MediaFormat audioFormat) {
setAudioFormat(audioFormat, false);
}
//We can't reuse info because could produce stream issues
private void updateFormat(MediaCodec.BufferInfo newInfo, MediaCodec.BufferInfo oldInfo) {
newInfo.flags = oldInfo.flags;
newInfo.offset = oldInfo.offset;
newInfo.size = oldInfo.size;
newInfo.presentationTimeUs = oldInfo.presentationTimeUs - pauseTime;
}
}
And this is one of the muxer i created
public class JCodecMuxer {
private SeekableByteChannel channel;
private MuxerTrack videoTrack;
private MuxerTrack audioTrack;
private int videoframeNo;
private int audioframeNo;
private MP4Muxer muxer;
private Size size;
private long videoPts = 0;
private long audioPts = 0;
public Context context;
public JCodecMuxer(String path, int width, int height) {
try {
this.channel = NIOUtils.writableChannel(new File(path));
} catch (Throwable e) {
e.printStackTrace();
}
this.size = new Size(width,height);
}
public JCodecMuxer(SeekableByteChannel seekableByteChannel, int width, int height) {
this.channel = seekableByteChannel;
this.size = new Size(width,height);
}
@RequiresApi(api = Build.VERSION_CODES.KITKAT)
public void init() {
try {
muxer = MP4Muxer.createMP4Muxer(channel, Brand.MOV);
videoTrack = muxer.addVideoTrack(
Codec.H264, VideoCodecMeta.createSimpleVideoCodecMeta(size, YUV420)
);
audioTrack = muxer.addCompressedAudioTrack(Codec.AAC,AudioFormat.STEREO_44K_S16_LE
);
} catch (Throwable t) {
t.printStackTrace();
}
}
public void encodeVideoByteBuffer(ByteBuffer buffer, MediaCodec.BufferInfo bufferInfo) {
try {
videoPts = 1001 * videoframeNo;
videoTrack.addFrame(
MP4Packet.createMP4Packet(
buffer,
videoPts,
30000,
1001,
videoframeNo,
Packet.FrameType.UNKNOWN,
null,
0,
videoPts,
0
)
);
videoframeNo++;
} catch (Throwable t) {
t.printStackTrace();
}
}
public void encodeAudioByteBuffer(ByteBuffer buffer,MediaCodec.BufferInfo bufferInfo) {
try {
byte[] originalData = new byte[buffer.remaining()];
buffer.get(originalData);
byte[] data = new byte[originalData.length + 7];
addADTStoPacket(data, data.length);
System.arraycopy(originalData,0,data,7,originalData.length);
audioPts = 1024 * audioframeNo;
audioTrack.addFrame(
MP4Packet.createMP4Packet(
ByteBuffer.wrap(data),
audioPts,
44100,
1024,
audioframeNo,
Packet.FrameType.UNKNOWN,
null,
0,
audioPts,
0
)
);
audioframeNo++;
} catch (Throwable t) {
t.printStackTrace();
}
}
/**
* Add ADTS header at the beginning of each and every AAC packet.
* This is needed as MediaCodec encoder generates a packet of raw
* AAC data.
*
* Note the packetLen must count in the ADTS header itself.
**/
private void addADTStoPacket(byte[] packet, int packetLen) {
int profile = 2; //AAC LC
//39=MediaCodecInfo.CodecProfileLevel.AACObjectELD;
int freqIdx = 4; //44.1KHz
int chanCfg = 1; //CPE
// fill in ADTS data
packet[0] = (byte)0xFF;
packet[1] = (byte)0xF9;
packet[2] = (byte)(((profile-1)<<6) + (freqIdx<<2) +(chanCfg>>2));
packet[3] = (byte)(((chanCfg&3)<<6) + (packetLen>>11));
packet[4] = (byte)((packetLen&0x7FF) >> 3);
packet[5] = (byte)(((packetLen&7)<<5) + 0x1F);
packet[6] = (byte)0xFC;
}
public void finish(){
try {
muxer.finish();
audioframeNo = 0;
videoframeNo = 0;
NIOUtils.closeQuietly(channel);
}
catch (Throwable t){
t.printStackTrace();
}
}
}
First of all this code:
audioPts = 1024 * audioframeNo;
Why this timestamp? Anyway, you can re use my timestamp like this:
audioPts = bufferInfo.presentationTimeUs;
This use microseconds so maybe you need change it to milliseconds or other thing depend of your library used.
If you still want to create your own timestamp i'm doing it like this: https://github.com/pedroSG94/rtmp-rtsp-stream-client-java/blob/master/encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java#L454 Where presentTimeUs is System.nanoTime() / 1000 when you start encoders.
using (only for audio)
bufferInfo.presentationTimeUs
improves the situation, but it seems (editing with vlc audio sync) the audio is behind 300 milliseconds. Debugging with android studio, the audio data arrives before the video data, so i had an insight: adding this control in recordAudio
public void recordAudio(ByteBuffer audioBuffer, MediaCodec.BufferInfo audioInfo) {
if (status == Status.RECORDING) {
if (videoInfo.flags == MediaCodec.BUFFER_FLAG_KEY_FRAME)
return;
updateFormat(this.audioInfo, audioInfo);
write(1, audioBuffer, this.audioInfo);
}
}
and now the sync if perfect!
I have written 2 kind of muxers (with Jcodec and with Xuggler), it could be perfect to add them to your library and leaving user the option to choose on them --> obsviously with these 2 muxers the user has the ability to use java API to edit the mp4 data while it's written (for example, crypting)
This could be interesting if I can handle to do it without add it directly to the main project. The reason is that I don't want add a 3rd party library that could be abandoned in anytime and I can't solve issues if I have any problem. If you can create a PR or share me code used I can try create a new project that could be used like an extra gradle that allow you use it similar to this: https://github.com/pedroSG94/RTSP-Server Basically a project that extend from this one and add other feature.
ok, but i think it should be a mix of what you said:
- PR to edit the RecordController class, adding two classes (interface Muxer.java and AndroidMediaMuxer with the default MediaMuxer ) and editing the methods of "startRecord" in CameraBase classes
- create a new project with different implementation for Muxer.java (as Jcodec and Xuggler i have created)