diff --git a/.gitignore b/.gitignore index 28f8f297..3777608d 100644 --- a/.gitignore +++ b/.gitignore @@ -119,4 +119,6 @@ thirdparty/zlib-1.2.11/ .env docker/__pycache__ -docker-compose.override.yml \ No newline at end of file +docker-compose.override.yml + +!/tests/TestBitStreams/*.bin diff --git a/dCommon/AMFDeserialize.cpp b/dCommon/AMFDeserialize.cpp new file mode 100644 index 00000000..7f3f6d95 --- /dev/null +++ b/dCommon/AMFDeserialize.cpp @@ -0,0 +1,158 @@ +#include "AMFDeserialize.h" + +#include "AMFFormat.h" + +/** + * AMF3 Reference document https://rtmp.veriskope.com/pdf/amf3-file-format-spec.pdf + * AMF3 Deserializer written by EmosewaMC + */ + +AMFValue* AMFDeserialize::Read(RakNet::BitStream* inStream) { + if (!inStream) return nullptr; + AMFValue* returnValue = nullptr; + // Read in the value type from the bitStream + int8_t marker; + inStream->Read(marker); + // Based on the typing, create the value associated with that and return the base value class + switch (marker) { + case AMFValueType::AMFUndefined: { + returnValue = new AMFUndefinedValue(); + break; + } + + case AMFValueType::AMFNull: { + returnValue = new AMFNullValue(); + break; + } + + case AMFValueType::AMFFalse: { + returnValue = new AMFFalseValue(); + break; + } + + case AMFValueType::AMFTrue: { + returnValue = new AMFTrueValue(); + break; + } + + case AMFValueType::AMFInteger: { + returnValue = ReadAmfInteger(inStream); + break; + } + + case AMFValueType::AMFDouble: { + returnValue = ReadAmfDouble(inStream); + break; + } + + case AMFValueType::AMFString: { + returnValue = ReadAmfString(inStream); + break; + } + + case AMFValueType::AMFArray: { + returnValue = ReadAmfArray(inStream); + break; + } + + // TODO We do not need these values, but if someone wants to implement them + // then please do so and add the corresponding unit tests. + case AMFValueType::AMFXMLDoc: + case AMFValueType::AMFDate: + case AMFValueType::AMFObject: + case AMFValueType::AMFXML: + case AMFValueType::AMFByteArray: + case AMFValueType::AMFVectorInt: + case AMFValueType::AMFVectorUInt: + case AMFValueType::AMFVectorDouble: + case AMFValueType::AMFVectorObject: + case AMFValueType::AMFDictionary: { + throw static_cast(marker); + break; + } + default: + throw static_cast(marker); + break; + } + return returnValue; +} + +uint32_t AMFDeserialize::ReadU29(RakNet::BitStream* inStream) { + bool byteFlag = true; + uint32_t actualNumber{}; + uint8_t numberOfBytesRead{}; + while (byteFlag && numberOfBytesRead < 4) { + uint8_t byte{}; + inStream->Read(byte); + // Parse the byte + if (numberOfBytesRead < 3) { + byteFlag = byte & static_cast(1 << 7); + byte = byte << 1UL; + } + // Combine the read byte with our current read in number + actualNumber <<= 8UL; + actualNumber |= static_cast(byte); + // If we are not done reading in bytes, shift right 1 bit + if (numberOfBytesRead < 3) actualNumber = actualNumber >> 1UL; + numberOfBytesRead++; + } + return actualNumber; +} + +std::string AMFDeserialize::ReadString(RakNet::BitStream* inStream) { + auto length = ReadU29(inStream); + // Check if this is a reference + bool isReference = length % 2 == 1; + // Right shift by 1 bit to get index if reference or size of next string if value + length = length >> 1; + if (isReference) { + std::string value(length, 0); + inStream->Read(&value[0], length); + // Empty strings are never sent by reference + if (!value.empty()) accessedElements.push_back(value); + return value; + } else { + // Length is a reference to a previous index - use that as the read in value + return accessedElements[length]; + } +} + +AMFValue* AMFDeserialize::ReadAmfDouble(RakNet::BitStream* inStream) { + auto doubleValue = new AMFDoubleValue(); + double value; + inStream->Read(value); + doubleValue->SetDoubleValue(value); + return doubleValue; +} + +AMFValue* AMFDeserialize::ReadAmfArray(RakNet::BitStream* inStream) { + auto arrayValue = new AMFArrayValue(); + auto sizeOfDenseArray = (ReadU29(inStream) >> 1); + if (sizeOfDenseArray >= 1) { + char valueType; + inStream->Read(valueType); // Unused + for (uint32_t i = 0; i < sizeOfDenseArray; i++) { + arrayValue->PushBackValue(Read(inStream)); + } + } else { + while (true) { + auto key = ReadString(inStream); + // No more values when we encounter an empty string + if (key.size() == 0) break; + arrayValue->InsertValue(key, Read(inStream)); + } + } + return arrayValue; +} + +AMFValue* AMFDeserialize::ReadAmfString(RakNet::BitStream* inStream) { + auto stringValue = new AMFStringValue(); + stringValue->SetStringValue(ReadString(inStream)); + return stringValue; +} + +AMFValue* AMFDeserialize::ReadAmfInteger(RakNet::BitStream* inStream) { + auto integerValue = new AMFIntegerValue(); + integerValue->SetIntegerValue(ReadU29(inStream)); + return integerValue; +} diff --git a/dCommon/AMFDeserialize.h b/dCommon/AMFDeserialize.h new file mode 100644 index 00000000..d03c150f --- /dev/null +++ b/dCommon/AMFDeserialize.h @@ -0,0 +1,71 @@ +#pragma once + +#include "BitStream.h" + +#include +#include + +class AMFValue; +class AMFDeserialize { + public: + /** + * Read an AMF3 value from a bitstream. + * + * @param inStream inStream to read value from. + * @return Returns an AMFValue with all the information from the bitStream in it. + */ + AMFValue* Read(RakNet::BitStream* inStream); + private: + /** + * @brief Private method to read a U29 integer from a bitstream + * + * @param inStream bitstream to read data from + * @return The number as an unsigned 29 bit integer + */ + uint32_t ReadU29(RakNet::BitStream* inStream); + + /** + * @brief Reads a string from a bitstream + * + * @param inStream bitStream to read data from + * @return The read string + */ + std::string ReadString(RakNet::BitStream* inStream); + + /** + * @brief Read an AMFDouble value from a bitStream + * + * @param inStream bitStream to read data from + * @return Double value represented as an AMFValue + */ + AMFValue* ReadAmfDouble(RakNet::BitStream* inStream); + + /** + * @brief Read an AMFArray from a bitStream + * + * @param inStream bitStream to read data from + * @return Array value represented as an AMFValue + */ + AMFValue* ReadAmfArray(RakNet::BitStream* inStream); + + /** + * @brief Read an AMFString from a bitStream + * + * @param inStream bitStream to read data from + * @return String value represented as an AMFValue + */ + AMFValue* ReadAmfString(RakNet::BitStream* inStream); + + /** + * @brief Read an AMFInteger from a bitStream + * + * @param inStream bitStream to read data from + * @return Integer value represented as an AMFValue + */ + AMFValue* ReadAmfInteger(RakNet::BitStream* inStream); + + /** + * List of strings read so far saved to be read by reference. + */ + std::vector accessedElements; +}; diff --git a/dCommon/AMFFormat.h b/dCommon/AMFFormat.h index 95863594..c3d0936c 100644 --- a/dCommon/AMFFormat.h +++ b/dCommon/AMFFormat.h @@ -308,6 +308,18 @@ public: \return Where the iterator ends */ _AMFArrayList_::iterator GetDenseIteratorEnd(); + + //! Returns the associative map + /*! + \return The associative map + */ + _AMFArrayMap_ GetAssociativeMap() { return this->associative; }; + + //! Returns the dense array + /*! + \return The dense array + */ + _AMFArrayList_ GetDenseArray() { return this->dense; }; }; //! The anonymous object value AMF type diff --git a/dCommon/CMakeLists.txt b/dCommon/CMakeLists.txt index 7e047a9c..7262bfef 100644 --- a/dCommon/CMakeLists.txt +++ b/dCommon/CMakeLists.txt @@ -1,4 +1,5 @@ set(DCOMMON_SOURCES "AMFFormat.cpp" + "AMFDeserialize.cpp" "AMFFormat_BitStream.cpp" "BinaryIO.cpp" "dConfig.cpp" diff --git a/tests/AMFDeserializeTests.cpp b/tests/AMFDeserializeTests.cpp new file mode 100644 index 00000000..88405803 --- /dev/null +++ b/tests/AMFDeserializeTests.cpp @@ -0,0 +1,404 @@ +#include +#include +#include +#include + +#include "AMFDeserialize.h" +#include "AMFFormat.h" +#include "CommonCxxTests.h" + +std::unique_ptr ReadFromBitStream(RakNet::BitStream* bitStream) { + AMFDeserialize deserializer; + std::unique_ptr returnValue(deserializer.Read(bitStream)); + return returnValue; +} + +int ReadAMFUndefinedFromBitStream() { + CBITSTREAM + bitStream.Write(0x00); + std::unique_ptr res(ReadFromBitStream(&bitStream)); + ASSERT_EQ(res->GetValueType(), AMFValueType::AMFUndefined); + return 0; +} + +int ReadAMFNullFromBitStream() { + CBITSTREAM + bitStream.Write(0x01); + std::unique_ptr res(ReadFromBitStream(&bitStream)); + ASSERT_EQ(res->GetValueType(), AMFValueType::AMFNull); + return 0; +} + +int ReadAMFFalseFromBitStream() { + CBITSTREAM + bitStream.Write(0x02); + std::unique_ptr res(ReadFromBitStream(&bitStream)); + ASSERT_EQ(res->GetValueType(), AMFValueType::AMFFalse); + return 0; +} + +int ReadAMFTrueFromBitStream() { + CBITSTREAM + bitStream.Write(0x03); + std::unique_ptr res(ReadFromBitStream(&bitStream)); + ASSERT_EQ(res->GetValueType(), AMFValueType::AMFTrue); + return 0; +} + +int ReadAMFIntegerFromBitStream() { + CBITSTREAM + { + bitStream.Write(0x04); + // 127 == 01111111 + bitStream.Write(127); + std::unique_ptr res(ReadFromBitStream(&bitStream)); + ASSERT_EQ(res->GetValueType(), AMFValueType::AMFInteger); + // Check that the max value of a byte can be read correctly + ASSERT_EQ(static_cast(res.get())->GetIntegerValue(), 127); + } + bitStream.Reset(); + { + bitStream.Write(0x04); + bitStream.Write(UINT32_MAX); + std::unique_ptr res(ReadFromBitStream(&bitStream)); + ASSERT_EQ(res->GetValueType(), AMFValueType::AMFInteger); + // Check that we can read the maximum value correctly + ASSERT_EQ(static_cast(res.get())->GetIntegerValue(), 536870911); + } + bitStream.Reset(); + { + bitStream.Write(0x04); + // 131 == 10000011 + bitStream.Write(131); + // 255 == 11111111 + bitStream.Write(255); + // 127 == 01111111 + bitStream.Write(127); + std::unique_ptr res(ReadFromBitStream(&bitStream)); + ASSERT_EQ(res->GetValueType(), AMFValueType::AMFInteger); + // Check that short max can be read correctly + ASSERT_EQ(static_cast(res.get())->GetIntegerValue(), UINT16_MAX); + } + bitStream.Reset(); + { + bitStream.Write(0x04); + // 255 == 11111111 + bitStream.Write(255); + // 127 == 01111111 + bitStream.Write(127); + std::unique_ptr res(ReadFromBitStream(&bitStream)); + ASSERT_EQ(res->GetValueType(), AMFValueType::AMFInteger); + // Check that 2 byte max can be read correctly + ASSERT_EQ(static_cast(res.get())->GetIntegerValue(), 16383); + } + return 0; +} + +int ReadAMFDoubleFromBitStream() { + CBITSTREAM + bitStream.Write(0x05); + bitStream.Write(25346.4f); + std::unique_ptr res(ReadFromBitStream(&bitStream)); + ASSERT_EQ(res->GetValueType(), AMFValueType::AMFDouble); + ASSERT_EQ(static_cast(res.get())->GetDoubleValue(), 25346.4f); + return 0; +} + +int ReadAMFStringFromBitStream() { + CBITSTREAM + bitStream.Write(0x06); + bitStream.Write(0x0F); + std::string toWrite = "stateID"; + for (auto e : toWrite) bitStream.Write(e); + std::unique_ptr res(ReadFromBitStream(&bitStream)); + ASSERT_EQ(res->GetValueType(), AMFValueType::AMFString); + ASSERT_EQ(static_cast(res.get())->GetStringValue(), "stateID"); + return 0; +} + +int ReadAMFArrayFromBitStream() { + CBITSTREAM + // Test empty AMFArray + bitStream.Write(0x09); + bitStream.Write(0x01); + bitStream.Write(0x01); + { + std::unique_ptr res(ReadFromBitStream(&bitStream)); + ASSERT_EQ(res->GetValueType(), AMFValueType::AMFArray); + ASSERT_EQ(static_cast(res.get())->GetAssociativeMap().size(), 0); + ASSERT_EQ(static_cast(res.get())->GetDenseArray().size(), 0); + } + bitStream.Reset(); + // Test a key'd value + bitStream.Write(0x09); + bitStream.Write(0x01); + bitStream.Write(0x15); + for (auto e : "BehaviorID") if (e != '\0') bitStream.Write(e); + bitStream.Write(0x06); + bitStream.Write(0x0B); + for (auto e : "10447") if (e != '\0') bitStream.Write(e); + bitStream.Write(0x01); + { + std::unique_ptr res(ReadFromBitStream(&bitStream)); + ASSERT_EQ(res->GetValueType(), AMFValueType::AMFArray); + ASSERT_EQ(static_cast(res.get())->GetAssociativeMap().size(), 1); + ASSERT_EQ(static_cast(static_cast(res.get())->FindValue("BehaviorID"))->GetStringValue(), "10447"); + } + // Test a dense array + return 0; +} + +/** + * This test checks that if we recieve an unimplemented AMFValueType + * we correctly throw an error and can actch it. + */ +int TestUnimplementedAMFValues() { + std::vector unimplementedValues = { + AMFValueType::AMFXMLDoc, + AMFValueType::AMFDate, + AMFValueType::AMFObject, + AMFValueType::AMFXML, + AMFValueType::AMFByteArray, + AMFValueType::AMFVectorInt, + AMFValueType::AMFVectorUInt, + AMFValueType::AMFVectorDouble, + AMFValueType::AMFVectorObject, + AMFValueType::AMFDictionary + }; + // Run unimplemented tests to check that errors are thrown if + // unimplemented AMF values are attempted to be parsed. + std::ifstream fileStream; + fileStream.open("AMFBitStreamUnimplementedTest.bin", std::ios::binary); + + // Read a test BitStream from a file + std::vector baseBitStream; + char byte = 0; + while (fileStream.get(byte)) { + baseBitStream.push_back(byte); + } + + fileStream.close(); + + for (auto amfValueType : unimplementedValues) { + RakNet::BitStream testBitStream; + for (auto element : baseBitStream) { + testBitStream.Write(element); + } + testBitStream.Write(amfValueType); + bool caughtException = false; + try { + ReadFromBitStream(&testBitStream); + } catch (AMFValueType unimplementedValueType) { + caughtException = true; + } + std::cout << "Testing unimplemented value " << amfValueType << " Did we catch an exception: " << (caughtException ? "YES" : "NO") << std::endl; + ASSERT_EQ(caughtException, true); + } + return 0; +} + +int TestLiveCapture() { + std::ifstream testFileStream; + testFileStream.open("AMFBitStreamTest.bin", std::ios::binary); + + // Read a test BitStream from a file + RakNet::BitStream testBitStream; + char byte = 0; + while (testFileStream.get(byte)) { + testBitStream.Write(byte); + } + + testFileStream.close(); + + auto resultFromFn = ReadFromBitStream(&testBitStream); + auto result = static_cast(resultFromFn.get()); + // Test the outermost array + + ASSERT_EQ(dynamic_cast(result->FindValue("BehaviorID"))->GetStringValue(), "10447"); + ASSERT_EQ(dynamic_cast(result->FindValue("objectID"))->GetStringValue(), "288300744895913279") + + // Test the execution state array + auto executionState = dynamic_cast(result->FindValue("executionState")); + ASSERT_NE(executionState, nullptr); + + auto strips = dynamic_cast(executionState->FindValue("strips"))->GetDenseArray(); + + ASSERT_EQ(strips.size(), 1); + + auto stripsPosition0 = dynamic_cast(strips[0]); + + auto actionIndex = dynamic_cast(stripsPosition0->FindValue("actionIndex")); + + ASSERT_EQ(actionIndex->GetDoubleValue(), 0.0f); + + auto stripIDExecution = dynamic_cast(stripsPosition0->FindValue("id")); + + ASSERT_EQ(stripIDExecution->GetDoubleValue(), 0.0f); + + auto stateIDExecution = dynamic_cast(executionState->FindValue("stateID")); + + ASSERT_EQ(stateIDExecution->GetDoubleValue(), 0.0f); + + auto states = dynamic_cast(result->FindValue("states"))->GetDenseArray(); + + ASSERT_EQ(states.size(), 1); + + auto firstState = dynamic_cast(states[0]); + + auto stateID = dynamic_cast(firstState->FindValue("id")); + + ASSERT_EQ(stateID->GetDoubleValue(), 0.0f); + + auto stripsInState = dynamic_cast(firstState->FindValue("strips"))->GetDenseArray(); + + ASSERT_EQ(stripsInState.size(), 1); + + auto firstStrip = dynamic_cast(stripsInState[0]); + + auto actionsInFirstStrip = dynamic_cast(firstStrip->FindValue("actions"))->GetDenseArray(); + + ASSERT_EQ(actionsInFirstStrip.size(), 3); + + auto actionID = dynamic_cast(firstStrip->FindValue("id")); + + ASSERT_EQ(actionID->GetDoubleValue(), 0.0f) + + auto uiArray = dynamic_cast(firstStrip->FindValue("ui")); + + auto xPos = dynamic_cast(uiArray->FindValue("x")); + auto yPos = dynamic_cast(uiArray->FindValue("y")); + + ASSERT_EQ(xPos->GetDoubleValue(), 103.0f); + ASSERT_EQ(yPos->GetDoubleValue(), 82.0f); + + auto stripID = dynamic_cast(firstStrip->FindValue("id")); + + ASSERT_EQ(stripID->GetDoubleValue(), 0.0f) + + auto firstAction = dynamic_cast(actionsInFirstStrip[0]); + + auto firstType = dynamic_cast(firstAction->FindValue("Type")); + + ASSERT_EQ(firstType->GetStringValue(), "OnInteract"); + + auto firstCallback = dynamic_cast(firstAction->FindValue("__callbackID__")); + + ASSERT_EQ(firstCallback->GetStringValue(), ""); + + auto secondAction = dynamic_cast(actionsInFirstStrip[1]); + + auto secondType = dynamic_cast(secondAction->FindValue("Type")); + + ASSERT_EQ(secondType->GetStringValue(), "FlyUp"); + + auto secondCallback = dynamic_cast(secondAction->FindValue("__callbackID__")); + + ASSERT_EQ(secondCallback->GetStringValue(), ""); + + auto secondDistance = dynamic_cast(secondAction->FindValue("Distance")); + + ASSERT_EQ(secondDistance->GetDoubleValue(), 25.0f); + + auto thirdAction = dynamic_cast(actionsInFirstStrip[2]); + + auto thirdType = dynamic_cast(thirdAction->FindValue("Type")); + + ASSERT_EQ(thirdType->GetStringValue(), "FlyDown"); + + auto thirdCallback = dynamic_cast(thirdAction->FindValue("__callbackID__")); + + ASSERT_EQ(thirdCallback->GetStringValue(), ""); + + auto thirdDistance = dynamic_cast(thirdAction->FindValue("Distance")); + + ASSERT_EQ(thirdDistance->GetDoubleValue(), 25.0f); + + return 0; +} + +int TestNullStream() { + auto result = ReadFromBitStream(nullptr); + ASSERT_EQ(result.get(), nullptr); + return 0; +} + +int AMFDeserializeTests(int argc, char** const argv) { + std::cout << "Checking that using a null bitstream doesnt cause exception" << std::endl; + if (TestNullStream()) return 1; + std::cout << "passed nullptr test, checking basic tests" << std::endl; + if (ReadAMFUndefinedFromBitStream() != 0) return 1; + if (ReadAMFNullFromBitStream() != 0) return 1; + if (ReadAMFFalseFromBitStream() != 0) return 1; + if (ReadAMFTrueFromBitStream() != 0) return 1; + if (ReadAMFIntegerFromBitStream() != 0) return 1; + if (ReadAMFDoubleFromBitStream() != 0) return 1; + if (ReadAMFStringFromBitStream() != 0) return 1; + if (ReadAMFArrayFromBitStream() != 0) return 1; + std::cout << "Passed basic test, checking live capture" << std::endl; + if (TestLiveCapture() != 0) return 1; + std::cout << "Passed live capture, checking unimplemented amf values" << std::endl; + if (TestUnimplementedAMFValues() != 0) return 1; + std::cout << "Passed all tests." << std::endl; + return 0; +} + +/** + * Below is the AMF that is in the AMFBitStreamTest.bin file that we are reading in + * from a bitstream to test. +args: amf3! +{ + "objectID": "288300744895913279", + "BehaviorID": "10447", + "executionState": amf3! + { + "strips": amf3! + [ + amf3! + { + "actionIndex": 0.0, + "id": 0.0, + }, + ], + "stateID": 0.0, + }, + "states": amf3! + [ + amf3! + { + "id": 0.0, + "strips": amf3! + [ + amf3! + { + "actions": amf3! + [ + amf3! + { + "Type": "OnInteract", + "__callbackID__": "", + }, + amf3! + { + "Distance": 25.0, + "Type": "FlyUp", + "__callbackID__": "", + }, + amf3! + { + "Distance": 25.0, + "Type": "FlyDown", + "__callbackID__": "", + }, + ], + "id": 0.0, + "ui": amf3! + { + "x": 103.0, + "y": 82.0, + }, + }, + ], + }, + ], +} + */ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b0e2c28d..9c6e57d3 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,6 +1,7 @@ # create the testing file and list of tests create_test_sourcelist (Tests CommonCxxTests.cpp + AMFDeserializeTests.cpp TestNiPoint3.cpp TestLDFFormat.cpp ) @@ -13,6 +14,17 @@ target_link_libraries(CommonCxxTests ${COMMON_LIBRARIES}) set (TestsToRun ${Tests}) remove (TestsToRun CommonCxxTests.cpp) +# Copy test files to testing directory +configure_file( + ${CMAKE_SOURCE_DIR}/tests/TestBitStreams/AMFBitStreamTest.bin ${PROJECT_BINARY_DIR}/tests/AMFBitStreamTest.bin + COPYONLY +) + +configure_file( + ${CMAKE_SOURCE_DIR}/tests/TestBitStreams/AMFBitStreamUnimplementedTest.bin ${PROJECT_BINARY_DIR}/tests/AMFBitStreamUnimplementedTest.bin + COPYONLY +) + # Add all the ADD_TEST for each test foreach (test ${TestsToRun}) get_filename_component (TName ${test} NAME_WE) diff --git a/tests/TestBitStreams/AMFBitStreamTest.bin b/tests/TestBitStreams/AMFBitStreamTest.bin new file mode 100644 index 00000000..05241f35 Binary files /dev/null and b/tests/TestBitStreams/AMFBitStreamTest.bin differ diff --git a/tests/TestBitStreams/AMFBitStreamUnimplementedTest.bin b/tests/TestBitStreams/AMFBitStreamUnimplementedTest.bin new file mode 100644 index 00000000..624bc5c3 --- /dev/null +++ b/tests/TestBitStreams/AMFBitStreamUnimplementedTest.bin @@ -0,0 +1 @@ + BehaviorID \ No newline at end of file