Removed data-offset field from PackInfo as it is not needed in most cases. Instead, pack_at_offset returns the data-offset, slightly improving performance, and reducing memory demands

Author: Byron

base.py

@@ -64,8 +64,8 @@ class OPackInfo(tuple):
 	location in the pack at which that actual data stream can be found."""
 	__slots__ = tuple()
 
-	def __new__(cls, packoffset, dataoffset, type, size):
-		return tuple.__new__(cls, (packoffset, dataoffset, type, size))
+	def __new__(cls, packoffset, type, size):
+		return tuple.__new__(cls, (packoffset,type, size))
 
 	def __init__(self, *args):
 		tuple.__init__(self)
@@ -76,21 +76,17 @@ def __init__(self, *args):
 	def pack_offset(self):
 		return self[0]
 
-	@property
-	def data_offset(self):
-		return self[1]
-		
 	@property
 	def type(self):
-		return type_id_to_type_map[self[2]]
+		return type_id_to_type_map[self[1]]
 
 	@property
 	def type_id(self):
-		return self[2]
+		return self[1]
 
 	@property
 	def size(self):
-		return self[3]
+		return self[2]
 
 	#} END interface
 
@@ -102,13 +98,13 @@ class ODeltaPackInfo(OPackInfo):
 	the pack offset of the base object"""
 	__slots__ = tuple()
 
-	def __new__(cls, packoffset, dataoffset, type, size, delta_info):
-		return tuple.__new__(cls, (packoffset, dataoffset, type, size, delta_info))
+	def __new__(cls, packoffset, type, size, delta_info):
+		return tuple.__new__(cls, (packoffset, type, size, delta_info))
 
 	#{ Interface 
 	@property
 	def delta_info(self):
-		return self[4]
+		return self[3]
 	#} END interface 
 
 
@@ -142,35 +138,35 @@ class OPackStream(OPackInfo):
 	is provided"""
 	__slots__ = tuple()
 
-	def __new__(cls, packoffset, dataoffset, type, size, stream, *args):
+	def __new__(cls, packoffset, type, size, stream, *args):
 		"""Helps with the initialization of subclasses"""
-		return tuple.__new__(cls, (packoffset, dataoffset, type, size, stream))
+		return tuple.__new__(cls, (packoffset, type, size, stream))
 
 	#{ Stream Reader Interface 
 	def read(self, size=-1):
-		return self[4].read(size)
+		return self[3].read(size)
 
 	@property
 	def stream(self):
-		return self[4]
+		return self[3]
 	#} END stream reader interface
 
 
 class ODeltaPackStream(ODeltaPackInfo):
 	"""Provides a stream outputting the uncompressed offset delta information"""
 	__slots__ = tuple()
 
-	def __new__(cls, packoffset, dataoffset, type, size, delta_info, stream):
-		return tuple.__new__(cls, (packoffset, dataoffset, type, size, delta_info, stream))
+	def __new__(cls, packoffset, type, size, delta_info, stream):
+		return tuple.__new__(cls, (packoffset, type, size, delta_info, stream))
 
 
 	#{ Stream Reader Interface 
 	def read(self, size=-1):
-		return self[5].read(size)
+		return self[4].read(size)
 
 	@property
 	def stream(self):
-		return self[5]
+		return self[4]
 	#} END stream reader interface
 
 

pack.py

@@ -55,7 +55,7 @@
 
 def pack_object_at(data, offset, as_stream):
 	"""
-	:return: PackInfo|PackStream
+	:return: Tuple(abs_data_offset, PackInfo|PackStream)
 		an object of the correct type according to the type_id  of the object.
 		If as_stream is True, the object will contain a stream, allowing  the
 		data to be read decompressed.
@@ -97,14 +97,14 @@ def pack_object_at(data, offset, as_stream):
 	if as_stream:
 		stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size)
 		if delta_info is None:
-			return OPackStream(offset, abs_data_offset, type_id, uncomp_size, stream)
+			return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream)
 		else:
-			return ODeltaPackStream(offset, abs_data_offset, type_id, uncomp_size, delta_info, stream)
+			return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream)
 	else:
 		if delta_info is None:
-			return OPackInfo(offset, abs_data_offset, type_id, uncomp_size)
+			return abs_data_offset, OPackInfo(offset, type_id, uncomp_size)
 		else:
-			return ODeltaPackInfo(offset, abs_data_offset, type_id, uncomp_size, delta_info)
+			return abs_data_offset, ODeltaPackInfo(offset, type_id, uncomp_size, delta_info)
 		# END handle info
 	# END handle stream
 
@@ -278,6 +278,7 @@ def sha_to_index(self, sha):
 			if the sha was not found in this pack index
 		:param sha: 20 byte sha to lookup"""
 		first_byte = ord(sha[0])
+		get_sha = self.sha
 		lo = 0					# lower index, the left bound of the bisection
 		if first_byte != 0:
 			lo = self._fanout_table[first_byte-1]
@@ -286,7 +287,7 @@ def sha_to_index(self, sha):
 		# bisect until we have the sha
 		while lo < hi:
 			mid = (lo + hi) / 2
-			c = cmp(sha, self.sha(mid))
+			c = cmp(sha, get_sha(mid))
 			if c < 0:
 				hi = mid
 			elif not c:
@@ -346,12 +347,12 @@ def _iter_objects(self, start_offset, as_stream=True):
 
 		null = NullStream()
 		while cur_offset < content_size:
-			ostream = pack_object_at(data, cur_offset, True)
+			data_offset, ostream = pack_object_at(data, cur_offset, True)
 			# scrub the stream to the end - this decompresses the object, but yields
 			# the amount of compressed bytes we need to get to the next offset
 
 			stream_copy(ostream.read, null.write, ostream.size, chunk_size)
-			cur_offset += (ostream.data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read()
+			cur_offset += (data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read()
 
 
 			# if a stream is requested, reset it beforehand
@@ -399,7 +400,7 @@ def collect_streams(self, offset):
 		:param offset: specifies the first byte of the object within this pack"""
 		out = list()
 		while True:
-			ostream = pack_object_at(self._data, offset, True)
+			ostream = pack_object_at(self._data, offset, True)[1]
 			out.append(ostream)
 			if ostream.type_id == OFS_DELTA:
 				offset = ostream.pack_offset - ostream.delta_info
@@ -420,13 +421,13 @@ def info(self, offset):
 		"""Retrieve information about the object at the given file-absolute offset
 		:param offset: byte offset
 		:return: OPackInfo instance, the actual type differs depending on the type_id attribute"""
-		return pack_object_at(self._data, offset or self.first_object_offset, False)
+		return pack_object_at(self._data, offset or self.first_object_offset, False)[1]
 
 	def stream(self, offset):
 		"""Retrieve an object at the given file-relative offset as stream along with its information
 		:param offset: byte offset
 		:return: OPackStream instance, the actual type differs depending on the type_id attribute"""
-		return pack_object_at(self._data, offset or self.first_object_offset, True)
+		return pack_object_at(self._data, offset or self.first_object_offset, True)[1]
 
 	def stream_iter(self, start_offset=0):
 		""":return: iterator yielding OPackStream compatible instances, allowing 

test/performance/test_db.py

@@ -14,7 +14,6 @@ class TestGitDBPerformance(TestBigRepoR):
 
 	def test_pack_random_access(self):
 		pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack"))
-		assert len(pdb._entities) > 1
 
 		# sha lookup
 		st = time()
@@ -72,3 +71,4 @@ def test_pack_random_access(self):
 		elapsed = time() - st
 		total_kib = total_size / 1000
 		print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed)
+		

test/test_base.py

@@ -35,14 +35,12 @@ def test_streams(self):
 		assert pinfo.type == str_blob_type
 		assert pinfo.type_id == blob_id
 		assert pinfo.pack_offset == 0
-		assert pinfo.data_offset == 1
 
 		dpinfo = ODeltaPackInfo(0, 1, blob_id, s, sha)
 		assert dpinfo.type == str_blob_type
 		assert dpinfo.type_id == blob_id
 		assert dpinfo.delta_info == sha
 		assert dpinfo.pack_offset == 0
-		assert dpinfo.data_offset == 1
 
 
 		# test ostream

test/test_pack.py

@@ -72,7 +72,6 @@ def _assert_pack_file(self, pack, version, size):
 			stream = pack.stream(obj.pack_offset)
 
 			assert info.pack_offset == stream.pack_offset
-			assert info.data_offset == stream.data_offset
 			assert info.type_id == stream.type_id
 			assert hasattr(stream, 'read')