Switch to AMF3 Decode Error (BufferUnderflowException)

[short6ft5]

BufferUnderflowException occurres when attempting to decode RTMP message which uses a marker signaling a switch from AMF0 to AMF3.

Have tested against the following versions:

• 1.0.8-M13
• 1.0.9-RELEASE
• 1.0.10-M6

Stack Trace

java.nio.BufferUnderflowException
	at java.nio.Buffer.nextGetIndex(Buffer.java:500)
	at java.nio.HeapByteBuffer.get(HeapByteBuffer.java:135)
	at org.apache.mina.core.buffer.AbstractIoBuffer.get(AbstractIoBuffer.java:501)
	at org.red5.io.amf.Input.readBoolean(Input.java:167)
	at org.red5.io.object.Deserializer.deserialize(Deserializer.java:93)
	at org.red5.server.net.rtmp.codec.RTMPProtocolDecoder.handleParameters(RTMPProtocolDecoder.java:1136)
	at org.red5.server.net.rtmp.codec.RTMPProtocolDecoder.decodeAction(RTMPProtocolDecoder.java:779)
	at org.red5.server.net.rtmp.codec.RTMPProtocolDecoder.decodeMessage(RTMPProtocolDecoder.java:524)
	at solers.sockpuppet.Red5ClientIT.testDecodeMessage(Red5ClientIT.java:176)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
	at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
	at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
	at org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:86)
	at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38)
	at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:538)
	at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:760)
	at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:460)
	at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:206)

Log

14:09:43.139 [main] DEBUG org.red5.io.object.Deserializer - Datatype: String
14:09:43.152 [main] DEBUG org.red5.io.amf.Input - Length: 7 limit: 22
14:09:43.153 [main] DEBUG org.red5.io.amf.Input - String: _result
14:09:44.658 [main] DEBUG org.red5.io.object.Deserializer - Datatype: Number
14:09:44.658 [main] DEBUG org.red5.io.amf.Input - readNumber from 11 bytes
14:09:44.659 [main] DEBUG org.red5.io.amf.Input - Number: 2.0
14:09:48.126 [main] DEBUG org.red5.io.object.Deserializer - Datatype: null
14:09:49.456 [main] DEBUG org.red5.io.amf.Input - Switch to AMF3
14:09:49.995 [main] DEBUG org.red5.io.object.Deserializer - Datatype: XML[-113]
14:09:52.667 [main] DEBUG org.red5.io.object.Deserializer - Datatype: Boolean

Unit Test

import java.nio.ByteBuffer;

import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Hex;
import org.apache.mina.core.buffer.IoBuffer;
import org.apache.mina.core.buffer.SimpleBufferAllocator;
import org.easymock.EasyMock;
import org.junit.Test;
import org.red5.server.api.IConnection;
import org.red5.server.net.rtmp.RTMPConnection;
import org.red5.server.net.rtmp.codec.RTMPProtocolDecoder;
import org.red5.server.net.rtmp.message.Constants;
import org.red5.server.net.rtmp.message.Header;

public class Red5ClientTest {

    @Test
    public void testDecodeMessage() throws DecoderException {
        
        //Mocking out the connection
        RTMPConnection conn = EasyMock.createNiceMock(RTMPConnection.class);
        EasyMock.expect(conn.getEncoding()).andStubReturn(IConnection.Encoding.AMF0);
        EasyMock.replay(conn);
        
        //Setting up the header
        Header header = new Header();
        header.setDataType(Constants.TYPE_INVOKE);
        
        //Defining the RMTP AMF0 body that is causing the issue
        byte[] _resultPacket = Hex.decodeHex("0200075F726573756C74004000000000000000051101".toCharArray());
        ByteBuffer byteBuffer = ByteBuffer.wrap(_resultPacket);
        IoBuffer ioBuffer = new SimpleBufferAllocator().wrap(byteBuffer);
        
        //Attempting to decode the body, but will result in stack trace
        RTMPProtocolDecoder decoder = new RTMPProtocolDecoder();
        decoder.decodeMessage(conn, header, ioBuffer);
    }
}

Wireshark Packet Capture ScreenShot