Added support for custom types to structparser and rewrote captureviewer and the struct definitions to use them.

captureviewer.pyw

@@ -8,18 +8,34 @@ import tkinter.filedialog as filedialog
 import tkinter.messagebox as messagebox
 import xml.etree.ElementTree as ET
 import zipfile
+import zlib
 from collections import OrderedDict
 from tkinter import BooleanVar, END, Menu
 
 import amf3
 import structparser
 import viewer
-from pyraknet.bitstream import BitStream, c_bit, c_float, c_int, c_int64, c_ubyte, c_uint, c_uint64, c_ushort
+import ldf
+from pyraknet.bitstream import BitStream, c_bit, c_bool, c_float, c_int, c_int64, c_ubyte, c_uint, c_uint64, c_ushort
+
+def compressed_ldf_handler(stream):
+	size = stream.read(c_uint)
+	is_compressed = stream.read(c_bool)
+	if is_compressed:
+		uncompressed_size = stream.read(c_uint)
+		uncompressed = zlib.decompress(stream.read(bytes, length=stream.read(c_uint)))
+		assert len(uncompressed) == uncompressed_size
+	else:
+		uncompressed = stream.read(bytes, length=size)
+	return ldf.from_ldf(BitStream(uncompressed))
+
+type_handlers = {}
+type_handlers["compressed_ldf"] = compressed_ldf_handler
 
 with open("packetdefinitions/replica/creation_header.structs", encoding="utf-8") as file:
-	creation_header_parser = structparser.StructParser(file.read())
+	creation_header_parser = structparser.StructParser(file.read(), type_handlers)
 with open("packetdefinitions/replica/serialization_header.structs", encoding="utf-8") as file:
-	serialization_header_parser = structparser.StructParser(file.read())
+	serialization_header_parser = structparser.StructParser(file.read(), type_handlers)
 
 component_name = OrderedDict()
 component_name[108] = "Component 108",
@@ -69,13 +85,13 @@ for comp_id, indices in component_name.items():
 		comp_parser[comp_id] = []
 		for index in indices:
 			with open("packetdefinitions/replica/components/"+index+".structs") as file:
-				comp_parser[comp_id].append(structparser.StructParser(file.read()))
+				comp_parser[comp_id].append(structparser.StructParser(file.read(), type_handlers))
 
 norm_parser = {}
 for rootdir, _, files in os.walk("packetdefinitions"):
 	for filename in files:
 		with open(rootdir+"/"+filename) as file:
-			norm_parser[filename[:filename.rindex(".")]] = structparser.StructParser(file.read())
+			norm_parser[filename[:filename.rindex(".")]] = structparser.StructParser(file.read(), type_handlers)
 	break
 
 class ParserOutput:
@@ -99,7 +115,7 @@ class ParserOutput:
 				self.tags.append("error")
 				import traceback
 				traceback.print_tb(tb)
-			self.text = exc_name+" "+str(exc_value)+"\n"+self.text
+			self.text = exc_name+" "+str(exc_type.__name__)+": "+str(exc_value)+"\n"+self.text
 			return True
 
 	def append(self, structs):
@@ -256,7 +272,7 @@ class CaptureViewer(viewer.Viewer):
 
 		obj = CaptureObject(network_id=network_id, object_id=object_id, lot=lot)
 		self.objects.append(obj)
-		obj.entry = self.tree.insert("", END, text=packet_name, values=(id_, parser_output.text), tags=parser_output.tags)
+		obj.entry = self.tree.insert("", END, text=packet_name, values=(id_, parser_output.text.replace("{", "<crlbrktopen>").replace("}", "<crlbrktclose>").replace("\\", "<backslash>")), tags=parser_output.tags)
 
 	@staticmethod
 	def parse_serialization(packet, parser_output, parsers, is_creation=False):
@@ -292,7 +308,7 @@ class CaptureViewer(viewer.Viewer):
 			parser_output.tags.append("error")
 		else:
 			error = ""
-		self.tree.insert(obj.entry, END, text=packet_name, values=(error, parser_output.text), tags=parser_output.tags)
+		self.tree.insert(obj.entry, END, text=packet_name, values=(error, parser_output.text.replace("{", "<crlbrktopen>").replace("}", "<crlbrktclose>").replace("\\", "<backslash>")), tags=parser_output.tags)
 
 	def parse_game_message(self, packet_name, packet):
 		object_id = packet.read(c_int64)

ldf.py

@@ -0,0 +1,28 @@
+from pyraknet.bitstream import BitStream, c_bool, c_int, c_int64, c_ubyte, c_uint
+
+def from_ldf(ldf):
+	ldf_dict = {}
+	if isinstance(ldf, BitStream):
+		for _ in range(ldf.read(c_uint)):
+			encoded_key = ldf.read(bytes, length=ldf.read(c_ubyte))
+			key = encoded_key.decode("utf-16-le")
+			data_type_id = ldf.read(c_ubyte)
+			if data_type_id == 0:
+				value = ldf.read(str, length_type=c_uint)
+			elif data_type_id == 1:
+				value = ldf.read(c_int)
+			elif data_type_id == 5:
+				value = ldf.read(c_uint)
+			elif data_type_id == 7:
+				value = ldf.read(c_bool)
+			elif data_type_id in (8, 9):
+				value = ldf.read(c_int64)
+			elif data_type_id == 13:
+				value = ldf.read(bytes, length=ldf.read(c_uint))
+			else:
+				raise NotImplementedError(key, data_type_id)
+			ldf_dict[key] = value
+	else:
+		raise NotImplementedError
+
+	return ldf_dict

packetdefinitions/replica/components/Character.structs

@@ -68,8 +68,7 @@ if creation:
 	[u64] - Number of 1st Place Race Finishes
 	[bit] - ???
 	[bit] - ???
-		[u16] - count of characters
-			[u16] - some LDF string?
+		[u16-wstring] - some LDF string?
 [bit] - flag
 	[bit] - flag?, assert == False
 	[bit] - ???, assert == False

packetdefinitions/replica/components/Inventory.structs

@@ -13,8 +13,7 @@ Index 1 ($+952860):
 		[bit] - flag
 			[u32] - ???, expect == 4
 		[bit] - flag
-			[u32] - length of following structure
-				[u8] - compressed data
+			[compressed_ldf] - extra data
 		[bit] - ??? (perhaps a flag that specifies if the item gets loaded or if data needs to be retrieved from the cdclient database?), expect == True
 [bit] - flag
 	[u32] - ???, assert == 0

packetdefinitions/replica/components/ModuleAssembly.structs

@@ -5,5 +5,4 @@ if creation:
 		[bit] - flag
 			[s64] - ???
 		[bit] - ???
-		[u16] - length
-			[u16] - wchar
+		[u16-wstring] - ???

packetdefinitions/replica/components/MovingPlatform.structs

@@ -2,8 +2,7 @@ Component 25 - Moving Platform
 flag=[bit] - flag
 [bit] - flag
 	[bit] - ???
-		[u16] - path name length
-			[u16] - path name
+		[u16-wstring] - path name
 		[u32] - ???
 		[bit] - ???
 if flag:

packetdefinitions/replica/components/Pet.structs

@@ -9,7 +9,5 @@ Index 1 ($+8D1270):
 		[s64] - ???
 	[bit] - flag, expect == False
 		[u32] - ???
-		[u8] - length
-			[u16] - ???
-		[u8] - length
-			[u16] - ???
+		[u8-wstring] - pet name
+		[u8-wstring] - pet owner name

packetdefinitions/replica/components/RacingControl.structs

@@ -38,8 +38,7 @@ end of ScriptedActivity
 end of something
 [bit] - flag
 	[u16] - remaining laps?
-	[u16] - length
-		[u16] - path name
+	[u16-wstring] - path name
 [bit] - flag
 	[bit] - flag
 		[s64] - ???

packetdefinitions/replica/components/Rebuild.structs

@@ -1,7 +1,7 @@
 Second index ($+90AE10):
 start of ScriptedActivity
 [bit] - flag
-	[u32] - length
+	[u32] - count
 		[u64] - player object id
 		constant size 10 loop
 		[float] - ???

packetdefinitions/replica/components/Render.structs

@@ -1,11 +1,9 @@
 Component 2 - Render (tested using LOT 1)
 Index 1 ($+840310):
 if creation:
-	[u32] - number of BehaviorEffects? (see BehaviorEffect table in cdclient), if this is -1 the client logs "Bad FX Unserialize", expect != -1
-		[u8] - length
-			[u8] - effectID string
-		[u32] - effectID
-		[u8] - length
-			[u16] - effectType
+	[s32] - number of BehaviorEffects? (see BehaviorEffect table in cdclient), if this is -1 the client logs "Bad FX Unserialize", expect != -1
+		[u8-string] - effectID string
+		[s32] - effectID
+		[u8-wstring] - effectType
 		[float] - ???
 		[s64] - ???

packetdefinitions/replica/components/Script.structs

@@ -2,5 +2,4 @@ Component 5 - Script (tested using LOT 3495)
 Index 1 ($+87CDF0):
 if creation:
 	[bit] - flag
-		[u32] - size of following struct
-			[u8] - compressed data, x bytes according to prev struct
+		[compressed_ldf] - script variables

packetdefinitions/replica/creation_header.structs

@@ -3,12 +3,10 @@
 	[u16] - NetworkID
 objectID=[s64] - objectID
 [s32] - LOT
-[u8] - length
-	[u16] - name
+[u8-wstring] - name
 [u32] - time_since_created_on_server?
 [bit] - flag, expect == False
-	[u32] - size of following struct
-		[u8] - compressed data
+	[compressed_ldf] - config variables?
 [bit] - trigger_id, expect == False
 [bit] - flag
 	[s64] - spawner object ID

structparser.py

@@ -8,8 +8,6 @@ from collections import namedtuple
 from pyraknet.bitstream import BitStream, c_bit, c_float, c_double, c_int8, c_uint8, c_int16, c_uint16, c_int32, c_uint32, c_int64, c_uint64
 
 VAR_CHARS = r"[^ \t\[\]]+"
-BITSTREAM_TYPES = {"bit": c_bit, "float": c_float, "double": c_double, "s8": c_int8, "u8": c_uint8, "s16": c_int16, "u16": c_uint16, "s32": c_int32, "u32": c_uint32, "s64": c_int64, "u64": c_uint64}
-TYPES_RE = "("+"|".join(BITSTREAM_TYPES.keys())+")"
 
 DEFINITION_SYNTAX = re.compile(r"""
 	^(?P<indent>\t*)                         # Indentation
@@ -21,7 +19,7 @@ DEFINITION_SYNTAX = re.compile(r"""
 	|
 	((?P<var_assign>"""+VAR_CHARS+r""")=)?   # Assign this struct a variable so the value can be back-referenced later
 	\[
-	(?P<type>"""+TYPES_RE+r""")              # Struct type
+	(?P<type>.*)                             # Struct type
 	\]
 	\ -\ (?P<description>.*?)                # Description for the struct
 	(,\ expect\ (?P<expect>(.+?)))?          # Expect the value to be like this expression. Struct attribute 'unexpected' will be None if no expects, True if any expects are False, or False if all expects are True.
@@ -37,11 +35,12 @@ StructDefinition = namedtuple("struct_token", ("var_assign", "type", "descriptio
 Structure = namedtuple("Structure", ("level", "description", "value", "unexpected"))
 
 class StructParser:
-	def __init__(self, struct_defs):
+	def __init__(self, struct_defs, type_handlers={}):
 		"""
 		Set up the parser with the structure definitions.
 		Arguments:
 			struct_defs: A string of structure definitions in my custom format (currently unnamed), see the documentation of that for details.
+			type_handlers: Parsing handlers for custom types, provided as {"type": handler_func}.
 		"""
 		self._variables = {}
 		struct_defs = struct_defs.splitlines()
@@ -49,6 +48,26 @@ class StructParser:
 
 		self.defs = self._to_tree(iter(struct_defs))[0]
 
+		self._type_handlers = {}
+		self._type_handlers["bit"] = lambda stream: stream.read(c_bit)
+		self._type_handlers["float"] = lambda stream: stream.read(c_float)
+		self._type_handlers["double"] = lambda stream: stream.read(c_double)
+		self._type_handlers["s8"] = lambda stream: stream.read(c_int8)
+		self._type_handlers["u8"] = lambda stream: stream.read(c_uint8)
+		self._type_handlers["s16"] = lambda stream: stream.read(c_int16)
+		self._type_handlers["u16"] = lambda stream: stream.read(c_uint16)
+		self._type_handlers["s32"] = lambda stream: stream.read(c_int32)
+		self._type_handlers["u32"] = lambda stream: stream.read(c_uint32)
+		self._type_handlers["s64"] = lambda stream: stream.read(c_int64)
+		self._type_handlers["u64"] = lambda stream: stream.read(c_uint64)
+		# string types
+		self._type_handlers["u8-string"] = lambda stream: stream.read(str, char_size=1, length_type=c_uint8)
+		self._type_handlers["u16-string"] = lambda stream: stream.read(str, char_size=1, length_type=c_uint16)
+
+		self._type_handlers["u8-wstring"] = lambda stream: stream.read(str, char_size=2, length_type=c_uint8)
+		self._type_handlers["u16-wstring"] = lambda stream: stream.read(str, char_size=2, length_type=c_uint16)
+		self._type_handlers.update(type_handlers)
+
 	def parse(self, data, variables=None):
 		"""
 		Parse the binary data, yielding structure objects.
@@ -112,7 +131,7 @@ class StructParser:
 		if def_["break"] is not None:
 			return BreakStatement()
 
-		type_ = BITSTREAM_TYPES[def_["type"]]
+		type_ = def_["type"]
 
 		if def_["expect"] is not None:
 			expects = [compile("value "+i, "<expect>", "eval") for i in def_["expect"].split(" and ")]
@@ -142,7 +161,7 @@ class StructParser:
 				elif isinstance(def_, BreakStatement):
 					return True
 				else:
-					value = stream.read(def_.type)
+					value = self._type_handlers[def_.type](stream)
 
 					if def_.expects:
 						for expression in def_.expects: