Implemented support for shared subcomponents and updated structures.

README.md

@@ -2,4 +2,6 @@ Various utilities.
 
 Requires Python 3, versions below 3.4 might work but are not tested.
 
+Some programs use the bitstream module from my python RakNet implementation for reading binary files. It's available at https://bitbucket.org/lcdr/pyraknet/src. Make sure to place it where Python can find it.
+
 ## License: GPL v3

captureviewer.pyw

@@ -21,27 +21,29 @@ with open("packetdefinitions/replica/serialization_header.structs", encoding="ut
 	serialization_header_parser = structparser.StructParser(file.read())
 
 component_name = OrderedDict()
-component_name[108] = "Component 108"
+component_name[108] = "Component 108",
-component_name[61] = "ModuleAssembly"
+component_name[61] = "ModuleAssembly",
-component_name[1] = "ControllablePhysics"
+component_name[1] = "ControllablePhysics",
-component_name[3] = "SimplePhysics"
+component_name[3] = "SimplePhysics",
-component_name[20] = "RigidBodyPhantomPhysics"
+component_name[20] = "RigidBodyPhantomPhysics",
-component_name[30] = "VehiclePhysics 30"
+component_name[30] = "VehiclePhysics 30",
-component_name[40] = "PhantomPhysics"
+component_name[40] = "PhantomPhysics",
-component_name[7] = "Destructible"
+component_name[7] = "Destructible", "Stats"
-component_name[49] = "Switch"
+component_name[23] = "Stats", "Collectible"
-component_name[26] = "Pet"
+component_name[26] = "Pet",
-component_name[4] = "Character"
+component_name[4] = "Character",
-component_name[17] = "Inventory"
+component_name[17] = "Inventory",
-component_name[5] = "Script"
+component_name[5] = "Script",
-component_name[9] = "Skill"
+component_name[9] = "Skill",
-component_name[60] = "BaseCombatAI"
+component_name[60] = "BaseCombatAI",
-component_name[16] = "Vendor"
+component_name[48] = "Stats", "Rebuild"
-component_name[6] = "Bouncer"
+component_name[49] = "Switch",
-component_name[39] = "ScriptedActivity"
+component_name[16] = "Vendor",
-component_name[71] = "RacingControl"
+component_name[6] = "Bouncer",
-component_name[2] = "Render"
+component_name[39] = "ScriptedActivity",
-component_name[107] = "Component 107"
+component_name[71] = "RacingControl",
+component_name[2] = "Render",
+component_name[107] = "Component 107",
 component_name[12] = None
 component_name[31] = None
 component_name[35] = None
@@ -53,15 +55,18 @@ component_name[64] = None
 component_name[65] = None
 component_name[68] = None
 component_name[73] = None
+component_name[104] = None
 component_name[113] = None
 component_name[114] = None
 comp_ids = list(component_name.keys())
 
 comp_parser = {}
-for key, value in component_name.items():
+for comp_id, indices in component_name.items():
-	if value is not None:
+	if indices is not None:
-		with open("packetdefinitions/replica/components/"+value+".structs") as file:
+		comp_parser[comp_id] = []
-			comp_parser[key] = structparser.StructParser(file.read())
+		for index in indices:
+			with open("packetdefinitions/replica/components/"+index+".structs") as file:
+				comp_parser[comp_id].append(structparser.StructParser(file.read()))
 
 norm_parser = {}
 for rootdir, _, files in os.walk("packetdefinitions"):
@@ -167,10 +172,11 @@ class CaptureViewer(viewer.Viewer):
 
 	def load_captures(self, captures):
 		self.tree.set_children("")
+		self.detached_items.clear()
 		self.objects = []
 		print("Loading captures, this might take a while")
-		for capture in captures:
+		for i, capture in enumerate(captures):
-			print("Loading", capture)
+			print("Loading", capture, "[%i/%i]" % (i+1, len(captures)))
 			with zipfile.ZipFile(capture) as capture:
 				files = [i for i in capture.namelist() if "of" not in i]
 
@@ -219,12 +225,14 @@ class CaptureViewer(viewer.Viewer):
 				print("Name for lot", lot, "not found")
 				lot_name = str(lot)
 			component_types = [i[0] for i in self.db.execute("select component_type from ComponentsRegistry where id == "+str(lot)).fetchall()]
-			parsers = []
+			parsers = OrderedDict()
 			try:
 				component_types.sort(key=comp_ids.index)
 				for comp_type in component_types:
 					if component_name[comp_type] is not None:
-						parsers.append((component_name[comp_type], comp_parser[comp_type]))
+						for name, parser in zip(component_name[comp_type], comp_parser[comp_type]):
+							if name not in parsers:
+								parsers[name] = parser
 			except ValueError as e:
 				error = "ERROR: Unknown component "+str(e.args[0].split()[0])+" "+str(component_types)
 			else:
@@ -250,7 +258,7 @@ class CaptureViewer(viewer.Viewer):
 	@staticmethod
 	def parse_serialization(packet, parser_output, parsers, is_creation=False):
 		parser_output.append(serialization_header_parser.parse(packet))
-		for name, parser in parsers:
+		for name, parser in parsers.items():
 			parser_output.text += "\n"+name+"\n\n"
 			parser_output.append(parser.parse(packet, {"creation":is_creation}))
 		if not packet.all_read():
@@ -259,9 +267,9 @@ class CaptureViewer(viewer.Viewer):
 	def parse_serialization_packet(self, packet_name, packet):
 		network_id = packet.read(c_ushort)
 		obj = None
-		for j in self.objects:
+		for i in self.objects:
-			if j.network_id == network_id:
+			if i.network_id == network_id:
-				obj = j
+				obj = i
 				break
 		if obj is None:
 			obj = CaptureObject(network_id=network_id)
@@ -269,7 +277,7 @@ class CaptureViewer(viewer.Viewer):
 			obj.entry = self.tree.insert("", END, text="Unknown", values=("network_id="+str(network_id), ""))
 
 		if obj.lot is None:
-			parsers = []
+			parsers = {}
 			error = "Unknown object"
 		else:
 			_, parsers, error = self.lot_data[obj.lot]

packetdefinitions/replica/components/BaseCombatAI.structs

@@ -1,5 +1,5 @@
 Component 60 - BaseCombatAI (tested using LOT 6366)
 Index 1 ($+824290):
-[bit] - flag, expect == False
+[bit] - flag
 	[u32] - ???
 	[s64] - ???

packetdefinitions/replica/components/Character.structs

@@ -82,4 +82,4 @@ if creation:
 	[s64] - ???
 	[u8] - ??? (count for next struct?)
 	[bit] - ???
-	[u32] - ???
+	[s32] - ???

packetdefinitions/replica/components/Collectible.structs

@@ -0,0 +1,2 @@
+Second index ($+845EC0):
+[u16] - ???

packetdefinitions/replica/components/Destructible.structs

@@ -37,42 +37,3 @@ if creation:
 			if trigger:
 				[s64] - ???
 			[u32] - ???
-
-Index 2 ($+92BBD0):
-if creation:
-	[bit] - flag
-		[u32] - ???, assert == 0
-		[u32] - ???, assert == 0
-		[u32] - ???
-		[u32] - ???
-		[u32] - ???, assert == 0
-		[u32] - ???, assert == 0
-		[u32] - ???, assert == 0
-		[u32] - ???, assert == 0
-		[u32] - ???
-[bit] - flag
-	[u32] - current health
-	maxhealth=[float] - ???, has same number as max health struct, assert % 1 == 0
-	[u32] - current armor
-	maxarmor=[float] - has same number as max armor struct, assert % 1 == 0
-	[u32] - current imagination
-	maximag=[float] - ???, has same number as max imagination, assert % 1 == 0
-	[u32] - ???, assert == 0
-	[bit] - ???
-	[bit] - ???, assert == False
-	[bit] - ???, assert == False
-	[float] - max health, assert == maxhealth
-	[float] - max armor, assert == maxarmor
-	[float] - max imagination, assert == maximag
-	[u32] - count
-		[s32] - faction id
-	trigger=[bit] - flag
-	if creation:
-		[bit] - flag, assert == False
-		[bit] - flag, assert == False
-		if trigger:
-			[bit] - ???, assert == False
-			[bit] - flag
-				[u32] - ???
-[bit] - flag
-	[bit] - ???

packetdefinitions/replica/components/PhantomPhysics.structs

@@ -8,7 +8,7 @@ Index 1 ($+834DB0):
 	[float] - rotation y
 	[float] - rotation z
 	[float] - rotation w
-[bit] - flag, assert == True
+[bit] - flag
 	[bit] - flag, expect == False
 		[u32] - ???
 		[float] - ???

packetdefinitions/replica/components/Rebuild.structs

@@ -0,0 +1,30 @@
+Second index ($+90AE10):
+start of ScriptedActivity
+[bit] - flag
+	[u32] - length
+		[u64] - player object id
+		constant size 10 loop
+		[float] - ???
+		[float] - ???
+		[float] - ???
+		[float] - ???
+		[float] - ???
+		[float] - ???
+		[float] - ???
+		[float] - ???
+		[float] - ???
+		[float] - ???
+end of ScriptedActivity
+[bit] - flag
+	[u32] - rebuild state
+	[bit] - ???
+	[bit] - ???
+	[float] - ???
+	[u32] - ???
+	if creation:
+		[bit] - ???
+			[u32] - ???
+		[float] - ???
+		[float] - ???
+		[float] - ???
+		[bit] - ???

packetdefinitions/replica/components/SimplePhysics.structs

@@ -11,7 +11,7 @@ if creation:
 	[float] - ???
 	[float] - ???
 [bit] - flag
-	[u32] - ???, assert == 5
+	[u32] - ???, expect == 5
 [bit] - flag
 	[float] - position x
 	[float] - position y

packetdefinitions/replica/components/Skill.structs

@@ -16,6 +16,6 @@ if creation:
 				[s64] - ???
 				[s64] - ???, expect == 0
 				[bit] - ???, assert == False
-				[u32] - ???, expect == 0
+				[float] - ???, expect == 0
 				[u32] - ???, expect == 0
 				[u32] - ???, expect == 0

packetdefinitions/replica/components/Stats.structs

@@ -0,0 +1,38 @@
+Index shared by Collectible (first index), Destructible (second index), and Rebuild (first index) ($+92BBD0):
+if creation:
+	[bit] - flag
+		[u32] - ???
+		[u32] - is boss?, expect == 0
+		[u32] - ???
+		[u32] - ???
+		[u32] - ???, assert == 0
+		[u32] - ???, assert == 0
+		[u32] - ???, assert == 0
+		[u32] - ???
+		[u32] - ???
+[bit] - flag
+	[u32] - current health
+	maxhealth=[float] - ???, has same number as max health struct, assert % 1 == 0
+	[u32] - current armor
+	maxarmor=[float] - has same number as max armor struct, assert % 1 == 0
+	[u32] - current imagination
+	maximag=[float] - ???, has same number as max imagination, assert % 1 == 0
+	[u32] - ???, assert == 0
+	[bit] - ???
+	[bit] - ???, assert == False
+	[bit] - ???, assert == False
+	[float] - max health, assert == maxhealth
+	[float] - max armor, assert == maxarmor
+	[float] - max imagination, assert == maximag
+	[u32] - count
+		[s32] - faction id
+	trigger=[bit] - flag
+	if creation:
+		[bit] - flag
+		[bit] - flag, assert == False
+		if trigger:
+			[bit] - ???, assert == False
+			[bit] - flag
+				[u32] - ???
+[bit] - flag
+	[bit] - ???

packetdefinitions/replica/serialization_header.structs

@@ -1,7 +1,7 @@
 [bit] - flag
 	[bit] - flag, expect == False
-		[s64] - ???
+		[s64] - parent object id
 		[bit] - ???
 	[bit] - flag, expect == False
-		[u16] - ??? (if > 0 read next block n times?)
+		[u16] - count
-			[u64] - ???
+			[u64] - child object id

viewer.py

@@ -54,6 +54,7 @@ class Viewer(Frame):
 		for parent, detached_children in self.detached_items.items():
 			for index, item in detached_children:
 				self.tree.reattach(item, parent, index)
+		self.detached_items.clear()
 		if query:
 			self.filter_items(query)
 
@@ -83,4 +84,4 @@ class Viewer(Frame):
 			self.sort_column(col, reverse, child)
 		if parent == "":
 			# reverse sort next time
-			self.tree.heading(col, command=lambda: self.sort_column(col, not reverse))
+			self.tree.heading(col, command=lambda: self.sort_column(col, not reverse))