Improved support for components and game messages, along with further minor updates.

captureviewer.pyw

@@ -1,4 +1,5 @@
 import configparser
+import math
 import os
 import sqlite3
 import tkinter.filedialog as filedialog
@@ -9,7 +10,7 @@ from tkinter import BooleanVar, END, Menu
 
 import viewer
 import structparser
-from pyraknet.bitstream import BitStream, c_bit, c_float, c_int, c_int64, c_ubyte, c_uint, c_ushort
+from pyraknet.bitstream import BitStream, c_bit, c_float, c_int, c_int64, c_ubyte, c_uint, c_uint64, c_ushort
 
 with open("packetdefinitions/replica/creation_header.structs", encoding="utf-8") as file:
 	creation_header_parser = structparser.StructParser(file.read())
@@ -17,8 +18,12 @@ with open("packetdefinitions/replica/serialization_header.structs", encoding="ut
 	serialization_header_parser = structparser.StructParser(file.read())
 
 component_name = OrderedDict()
+component_name[108] = "Component 108"
+component_name[61] = "ModuleAssembly"
 component_name[1] = "ControllablePhysics"
 component_name[3] = "SimplePhysics"
+component_name[20] = "RigidBodyPhantomPhysics"
+component_name[30] = "VehiclePhysics 30"
 component_name[40] = "PhantomPhysics"
 component_name[7] = "Destructible"
 component_name[49] = "Switch"
@@ -31,14 +36,21 @@ component_name[60] = "BaseCombatAI"
 component_name[16] = "Vendor"
 component_name[6] = "Bouncer"
 component_name[39] = "ScriptedActivity"
-component_name[12] = None
+component_name[71] = "RacingControl"
 component_name[2] = "Render"
-component_name[107] = "Index36"
+component_name[107] = "Component 107"
+component_name[12] = None
 component_name[31] = None
 component_name[35] = None
+component_name[36] = None
+component_name[45] = None
+component_name[55] = None
 component_name[56] = None
 component_name[64] = None
+component_name[65] = None
+component_name[68] = None
 component_name[73] = None
+component_name[113] = None
 component_name[114] = None
 comp_ids = list(component_name.keys())
 
@@ -234,7 +246,7 @@ class CaptureViewer(viewer.Viewer):
 			parser_output.text += "\n"+name+"\n\n"
 			parser_output.append(parser.parse(packet, {"creation":is_creation}))
 		if not packet.all_read():
-			raise IndexError("Not completely read")
+			raise IndexError("Not completely read, %i bytes unread" % (len(packet) - math.ceil(packet._read_offset / 8)))
 
 	def parse_serialization_packet(self, packet_name, packet):
 		network_id = packet.read(c_ushort)
@@ -285,37 +297,60 @@ class CaptureViewer(viewer.Viewer):
 			msg_id -= 1
 		elif msg_id <= 0x208:
 			msg_id -= 5
-		elif msg_id <= 0x231:
+		elif msg_id <= 0x223:
+			msg_id -= 6
+		elif msg_id <= 0x240:
 			msg_id -= 8
+		elif msg_id <= 0x2a3:
+			msg_id -= 10
+		elif msg_id <= 0x2b5:
+			msg_id -= 12
+		elif msg_id <= 0x2d5:
+			msg_id -= 7
 		elif msg_id <= 0x30d:
 			msg_id -= 10
 		elif msg_id <= 0x353:
 			msg_id -= 9
-		elif msg_id <= 0x37a:
+		elif msg_id <= 0x37b:
 			msg_id -= 10
-		elif msg_id <= 0x3a6:
+		elif msg_id <= 0x3bd:
 			msg_id -= 9
-		elif msg_id <= 0x430:
+		elif msg_id <= 0x3d4:
+			msg_id -= 30
+		elif msg_id <= 0x3ec:
+			msg_id -= 32
+		elif msg_id <= 0x433:
 			msg_id -= 33
-		elif msg_id <= 0x4c7:
+		elif msg_id <= 0x4cf:
 			msg_id -= 34
-		elif msg_id <= 0x510:
+		elif msg_id <= 0x51d:
 			msg_id -= 31
 		elif msg_id <= 0x58b:
 			msg_id -= 30
 		elif msg_id <= 0x5e7:
 			msg_id -= 29
+		elif msg_id <= 0x637:
+			msg_id -= 28
+		elif msg_id <= 0x670:
+			msg_id -= 27
+		elif msg_id <= 0x6e7:
+			msg_id -= 26
 
 		try:
 			message = self.gamemsgs[msg_id]
 			msg_name = message.get("name")
 			network = message.get("network")
+			attr_values = OrderedDict()
 			if network is None or ((("[53-05-00-0c]" in packet_name and "client" not in network) or ("[53-04-00-05]" in packet_name and "server" not in network)) and network != "duplicated"):
 				raise ValueError
 
 			attrs = message.findall("attr")
+
+			if msg_name == "Teleport":
+				attrs = [attr for attr in attrs if attr.get("name") != "NoGravTeleport"]
+
 			attrs.sort(key=lambda x: x.get("name"))
-			attr_values = OrderedDict()
+
 			if message.find("freeze") is not None or message.find("thaw") is not None:
 				# Custom serializations
 				if msg_name == "NotifyMissionTask":
@@ -335,11 +370,94 @@ class CaptureViewer(viewer.Viewer):
 					attr_values["playerID"] = packet.read(c_int64)
 					attr_values["missionID"] = packet.read(c_int)
 					attr_values["bMissionOffered"] = packet.read(c_bit)
+				elif msg_name == "FetchModelMetadataResponse":
+					attr_values["ugID"] = packet.read(c_int64)
+					attr_values["objectID"] = packet.read(c_int64)
+					attr_values["requestorID"] = packet.read(c_int64)
+					attr_values["context"] = packet.read(c_int)
+					attr_values["bHasUGData"] = packet.read(c_bit)
+					attr_values["bHasBPData"] = packet.read(c_bit)
+					if attr_values["bHasUGData"]:
+						attr_values["UGM_unknown1"] = packet.read(c_int64)
+						attr_values["UGM_unknown2"] = packet.read(c_int64)
+						attr_values["UGM_unknown_str_1"] = packet.read(str, length_type=c_uint)
+						attr_values["UGM_unknown_str_2"] = packet.read(str, length_type=c_uint)
+						attr_values["UGM_unknown3"] = packet.read(c_int64)
+						attr_values["UGM_unknown4"] = packet.read(c_int64)
+						attr_values["UGM_unknown_str_3"] = packet.read(str, length_type=c_uint)
+						unknown_list = []
+						for _ in range(packet.read(c_ubyte)):
+							unknown_list.append(packet.read(c_int64))
+						attr_values["UGM_unknown_list"] = unknown_list
+
+					if attr_values["bHasBPData"]:
+						attr_values["BPM_unknown1"] = packet.read(c_int64)
+						attr_values["BPM_some_timestamp"] = packet.read(c_uint64)
+						attr_values["BPM_unknown2"] = packet.read(c_uint)
+						attr_values["BPM_unknown3"] = packet.read(c_float), packet.read(c_float), packet.read(c_float)
+						attr_values["BPM_unknown4"] = packet.read(c_float), packet.read(c_float), packet.read(c_float)
+						attr_values["BPM_unknown_bool_1"] = packet.read(c_bit)
+						attr_values["BPM_unknown_bool_2"] = packet.read(c_bit)
+						attr_values["BPM_unknown_str_1"] = packet.read(str, length_type=c_uint)
+						attr_values["BPM_unknown_bool_3"] = packet.read(c_bit)
+						attr_values["BPM_unknown5"] = packet.read(c_uint)
+
+				elif msg_name == "NotifyPetTamingPuzzleSelected":
+					bricks = []
+					for _ in range(packet.read(c_uint)):
+						bricks.append((packet.read(c_uint), packet.read(c_uint)))
+					attr_values["randBrickIDList"] = bricks
+				elif msg_name == "DownloadPropertyData":
+					attr_values["object_id"] = packet.read(c_int64)
+					attr_values["component_id"] = packet.read(c_int)
+					attr_values["mapID"] = packet.read(c_ushort)
+					attr_values["vendorMapID"] = packet.read(c_ushort)
+					attr_values["unknown1"] = packet.read(c_uint)
+					attr_values["property_name"] = packet.read(str, length_type=c_uint)
+					attr_values["property_description"] = packet.read(str, length_type=c_uint)
+					attr_values["owner_name"] = packet.read(str, length_type=c_uint)
+					attr_values["owner_object_id"] = packet.read(c_int64)
+					attr_values["type"] = packet.read(c_uint)
+					attr_values["sizecode"] = packet.read(c_uint)
+					attr_values["minimumPrice"] = packet.read(c_uint)
+					attr_values["rentDuration"] = packet.read(c_uint)
+					attr_values["timestamp1"] = packet.read(c_uint64)
+					attr_values["unknown2"] = packet.read(c_uint)
+					attr_values["unknown3"] = packet.read(c_uint64)
+					attr_values["spawnName"] = packet.read(str, length_type=c_uint)
+					attr_values["unknown_str_1"] = packet.read(str, length_type=c_uint)
+					attr_values["unknown_str_2"] = packet.read(str, length_type=c_uint)
+					attr_values["durationType"] = packet.read(c_uint)
+					attr_values["unknown4"] = packet.read(c_uint)
+					attr_values["unknown5"] = packet.read(c_uint)
+					attr_values["unknown6"] = packet.read(c_ubyte)
+					attr_values["unknown7"] = packet.read(c_uint64)
+					attr_values["unknown8"] = packet.read(c_uint)
+					attr_values["unknown_str_3"] = packet.read(str, length_type=c_uint)
+					attr_values["unknown9"] = packet.read(c_uint64)
+					attr_values["unknown10"] = packet.read(c_uint)
+					attr_values["unknown11"] = packet.read(c_uint)
+					attr_values["zoneX"] = packet.read(c_float)
+					attr_values["zoneY"] = packet.read(c_float)
+					attr_values["zoneZ"] = packet.read(c_float)
+					attr_values["maxBuildHeight"] = packet.read(c_float)
+					attr_values["timestamp2"] = packet.read(c_uint64)
+					attr_values["unknown12"] = packet.read(c_ubyte)
+					path = []
+					for _ in range(packet.read(c_uint)):
+						path.append((packet.read(c_float), packet.read(c_float), packet.read(c_float)))
+					attr_values["path"] = path
 				elif msg_name == "ModularBuildFinish":
 					lots = []
 					for _ in range(packet.read(c_ubyte)):
 						lots.append(packet.read(c_int))
 					attr_values["moduleTemplateIDs"] = lots
+				elif msg_name == "PetTamingTryBuild":
+					selections = []
+					for _ in range(packet.read(c_uint)):
+						selections.append((packet.read(c_uint), packet.read(c_uint)))
+					attr_values["currentSelections"] = selections
+					attr_values["clientFailed"] = packet.read(c_bit)
 				else:
 					raise NotImplementedError("Custom serialization")
 				values = "\n".join(["%s = %s" % (a, b) for a, b in attr_values.items()])
@@ -351,6 +469,9 @@ class CaptureViewer(viewer.Viewer):
 
 				for attr in attrs:
 					if attr.get("returnValue") is not None:
+						if attr.get("returnValue") == "false":
+							continue
+						else:
 							raise NotImplementedError(attr.get("name"), "returnValue")
 					type_ = attr.get("type")
 					default = attr.get("default")
@@ -368,7 +489,7 @@ class CaptureViewer(viewer.Viewer):
 						value = packet.read(c_ushort)
 					elif type_ in ("int", "LOT"):
 						value = packet.read(c_int)
-					elif type_ in ("unsigned int", "TSkillID"):
+					elif type_ in ("unsigned int", "LWOCLONEID", "TSkillID"):
 						value = packet.read(c_uint)
 					elif type_ == "__int64":
 						value = packet.read(c_int64)
@@ -383,6 +504,9 @@ class CaptureViewer(viewer.Viewer):
 									break
 					elif type_ == "float":
 						value = packet.read(c_float)
+					elif type_ == "BinaryBuffer":
+						length = packet.read(c_uint)
+						value = packet.read(bytes, length=length)
 					elif type_ == "std::string":
 						length = packet.read(c_uint)
 						if length > 255: # in case this isn't the right message after all and we read a way too high value
@@ -415,15 +539,12 @@ class CaptureViewer(viewer.Viewer):
 		except NotImplementedError as e:
 			values = (msg_name, str(e)+"\nlen: "+str(len(packet)-10)+"\n"+"\n".join(["%s = %s" % (a, b) for a, b in attr_values.items()]))
 			tags = ["error"]
-		except (IndexError, UnicodeDecodeError) as e:
+		except Exception as e:
 			print(packet_name, msg_name)
 			import traceback
 			traceback.print_exc()
 			values = ("likely not "+msg_name, "Error while parsing, likely not this message!\n"+str(e)+"\nlen: "+str(len(packet)-10)+"\n"+"\n".join(["%s = %s" % (a, b) for a, b in attr_values.items()]))
 			tags = ["error"]
-		except Exception as e:
-			values = ("likely not "+msg_name, "Error while parsing, likely not this message!\n"+str(e)+"\nlen: "+str(len(packet)-10))
-			tags = ["error"]
 		else:
 			values = (msg_name, "\n".join(["%s = %s" % (a, b) for a, b in attr_values.items()]))
 			tags = []
@@ -439,10 +560,11 @@ class CaptureViewer(viewer.Viewer):
 		else:
 			packet.skip_read(1)
 		parser_output = ParserOutput()
+		with parser_output:
 			parser_output.append(norm_parser[id_].parse(packet))
 		self.tree.insert("", END, text=packet_name, values=(id_, parser_output.text), tags=parser_output.tags)
 
-	def on_item_select(self, event):
+	def on_item_select(self, _):
 		item = self.tree.selection()[0]
 		self.item_inspector.delete(1.0, END)
 		self.item_inspector.insert(END, self.tree.item(item, "values")[1])

luzviewer.pyw

@@ -99,7 +99,7 @@ class LUZViewer(viewer.Viewer):
 					rotation = stream.read(c_float), stream.read(c_float), stream.read(c_float), stream.read(c_float)
 					scale = stream.read(c_float)
 					config_data = stream.read(str, length_type=c_uint)
-					config_data = config_data.replace("{", "<crlbrktstart>").replace("}", "<crlbrktend>").replace("\\", "<backslash>") # for some reason these characters aren't properly escaped when sent to Tk
+					config_data = config_data.replace("{", "<crlbrktopen>").replace("}", "<crlbrktclose>").replace("\\", "<backslash>") # for some reason these characters aren't properly escaped when sent to Tk
 					assert stream.read(c_uint) == 0
 					lot_name = ""
 					if lot == 176:
@@ -115,7 +115,7 @@ class LUZViewer(viewer.Viewer):
 				pass
 			stream._read_offset = (start_pos + chunk_length) * 8 # go to the next CHNK
 
-	def on_item_select(self, event):
+	def on_item_select(self, _):
 		item = self.tree.selection()[0]
 		item_type = self.tree.item(item, "text")
 		if item_type == "Zone":

packetdefinitions/replica/components/BaseCombatAI.structs

@@ -1,5 +1,5 @@
 Component 60 - BaseCombatAI (tested using LOT 6366)
 Index 1 ($+824290):
-[bit] - flag
+[bit] - flag, expect == False
 	[u32] - ???
 	[s64] - ???

packetdefinitions/replica/components/Component 108.structs

@@ -0,0 +1,8 @@
+Component 108 - Something vehicle related (tested using LOT 7305)
+Index 1 ($+7DC1F0)
+[bit] - flag
+	[bit] - flag
+		[s64] - driver object id
+	[bit] - flag
+		[u32] - ???
+	[bit] - ???

packetdefinitions/replica/components/ModuleAssembly.structs

@@ -0,0 +1,9 @@
+Component 61 - ModuleAssembly (tested using LOT 8092)
+Index 1 ($+913F30):
+if creation:
+	[bit] - flag
+		[bit] - flag
+			[s64] - ???
+		[bit] - ???
+		[u16] - length
+			[u16] - wchar

packetdefinitions/replica/components/Pet.structs

@@ -3,11 +3,11 @@ Index 1 ($+8D1270):
 [bit] - flag
 	[u32] - ???
 	[u32] - ???
-	[bit] - flag
+	[bit] - flag, expect == False
 		[s64] - ???
-	[bit] - flag
+	[bit] - flag, expect == False
 		[s64] - ???
-	[bit] - flag
+	[bit] - flag, expect == False
 		[u32] - ???
 		[u8] - length
 			[u16] - ???

packetdefinitions/replica/components/RacingControl.structs

@@ -0,0 +1,47 @@
+Component 71 - RacingControl (tested using LOT 11296)
+Index 1 ($+949620):
+start of something
+start of ScriptedActivity
+[bit] - flag
+	[u32] - length
+		[u64] - player object id
+		constant size 10 loop
+		[float] - ???
+		[float] - ???
+		[float] - ???
+		[float] - ???
+		[float] - ???
+		[float] - ???
+		[float] - ???
+		[float] - ???
+		[float] - ???
+		[float] - ???
+end of ScriptedActivity
+[bit] - flag
+	[u16] - ???
+[bit] - flag
+	while True:
+		not_break=[bit] - flag
+		if not not_break:
+			break
+		[s64] - player object id
+		[s64] - car object id
+		[u32] - ???
+		[bit] - ???
+[bit] - flag
+	while True:
+		not_break=[bit] - flag
+		if not not_break:
+			break
+		[s64] - player object id
+		[u32] - ???
+end of something
+[bit] - flag
+	[u16] - remaining laps?
+	[u16] - length
+		[u16] - path name
+[bit] - flag
+	[bit] - flag
+		[s64] - ???
+		[float] - ???
+		[float] - ???

packetdefinitions/replica/components/RigidBodyPhantomPhysics.structs

@@ -0,0 +1,10 @@
+Component 20 - RigidBodyPhantomPhysics (tested using LOT 7680)
+Index 1 ($+7D90C0):
+[bit] - flag
+	[float] - position x
+	[float] - position y
+	[float] - position z
+	[float] - rotation x
+	[float] - rotation y
+	[float] - rotation z
+	[float] - rotation w

packetdefinitions/replica/components/ScriptedActivity.structs

@@ -2,7 +2,7 @@ Component 39 - ScriptedActivity (tested using LOT 11165)
 Index 1 ($+9002B0):
 [bit] - flag
 	[u32] - length
-		[u64] - ???
+		[s64] - player object id
 		constant size 10 loop
 		[float] - ???
 		[float] - ???

packetdefinitions/replica/components/VehiclePhysics 30.structs

@@ -0,0 +1,36 @@
+Component 30 - VehiclePhysics (tested using LOT 7305)
+Index 1 ($+7FD4D0)
+[bit] - flag
+	[float] - position x
+	[float] - position y
+	[float] - position z
+	[float] - rotation x
+	[float] - rotation y
+	[float] - rotation z
+	[float] - rotation w
+	[bit] - is on ground
+	[bit] - ???
+	[bit] - flag
+		[float] - velocity x
+		[float] - velocity y
+		[float] - velocity z
+	[bit] - flag
+		[float] - angular velocity x
+		[float] - angular velocity y
+		[float] - angular velocity z
+	[bit] - flag
+		[s64] - ???
+		[float] - ???
+		[float] - ???
+		[float] - ???
+		[bit] - flag
+			[float] - ???
+			[float] - ???
+			[float] - ???
+	if not creation:
+		[bit] - flag?
+if creation:
+	[u8] - ???
+	[bit] - ???
+[bit] - flag
+	[bit] - ???

packetdefinitions/replica/creation_header.structs

@@ -8,12 +8,12 @@ objectID=[s64] - objectID
 [u32] - time_since_created_on_server?
 [bit] - flag, expect == False
 	[u32] - size of following struct
-		[u8] - compressed data, x bytes according to prev struct
+		[u8] - compressed data
 [bit] - trigger_id, expect == False
 [bit] - flag
 	[s64] - spawner object ID
 [bit] - flag
-	[u32] - spawner_node_id, expect == 0
+	[u32] - spawner_node_id
 [bit] - flag
 	[float] - ???
 [bit] - flag, assert == False

viewer.py

@@ -17,7 +17,7 @@ class Viewer(Frame):
 	def create_widgets(self):
 		find_entry = Entry(textvariable=self.find_input)
 		find_entry.pack(fill=X)
-		find_entry.bind("<Return>", lambda event: self.find())
+		find_entry.bind("<Return>", self.find)
 
 		pane = PanedWindow(orient=HORIZONTAL)
 		pane.pack(fill=BOTH, expand=True)
@@ -45,15 +45,15 @@ class Viewer(Frame):
 		scrollbar.configure(command=self.item_inspector.yview)
 		pane.add(frame)
 
-	def find(self):
+	def find(self, _):
 		query = self.find_input.get().lower()
 		for item in self.tree.tag_has("match"):
 			tags = list(self.tree.item(item, "tags"))
 			tags.remove("match")
 			self.tree.item(item, tags=tags)
 		for parent, detached_children in self.detached_items.items():
-			for i in detached_children:
+			for index, item in detached_children:
-				self.tree.reattach(i, parent, END)
+				self.tree.reattach(item, parent, index)
 		if query:
 			self.filter_items(query)
 
@@ -68,9 +68,9 @@ class Viewer(Frame):
 				self.tree.see(item)
 			if self.filter_items(query, item) and item in detached_children:
 				detached_children.remove(item) # don't detach if a child matches
-		self.detached_items[parent] = detached_children
+		self.detached_items[parent] = [(self.tree.index(item), item) for item in detached_children]
-		for i in detached_children:
+		for item in detached_children:
-			self.tree.detach(i)
+			self.tree.detach(item)
 		return len(detached_children) != len(all_children) # return true if any children match
 
 	def sort_column(self, col, reverse, parent=""):