Implemented main PackEntity object retrieval method and moved constructor for delta_streams out of the PackFile, into the stream itself where it belongs.

All this is still to be tested

Author: Byron

fun.py

@@ -14,6 +14,8 @@
 # INVARIANTS
 OFS_DELTA = 6
 REF_DELTA = 7
+delta_types = (OFS_DELTA, REF_DELTA)
+
 type_id_to_type_map = 	{
 							0 : "",				# EXT 1
 							1 : "commit",

pack.py

@@ -5,19 +5,24 @@
 from util import (
 					LockedFD,
 					LazyMixin,
-					file_contents_ro, 
-					unpack_from
+					unpack_from,
+					file_contents_ro,
 					)
 
 from fun import (
 					pack_object_header_info,
+					type_id_to_type_map,
 					stream_copy, 
 					chunk_size,
+					delta_types,
 					OFS_DELTA, 
-					REF_DELTA
+					REF_DELTA,
+					msb_size
 				)
 
-from base import (
+from base import (		# Amazing !
+						OInfo,
+						OStream,
 						OPackInfo,
 						OPackStream,
 						ODeltaPackInfo,
@@ -35,7 +40,7 @@
 
 __all__ = ('PackIndexFile', 'PackFile')
 
-_delta_types = (OFS_DELTA, REF_DELTA) 
+ 
 
 
 #{ Utilities 
@@ -95,8 +100,6 @@ def pack_object_at(data, offset, as_stream):
 		# END handle info
 	# END handle stream
 
-			
-
 #} END utilities
 
 
@@ -355,6 +358,7 @@ def collect_streams(self, offset):
 			the last one is either a full object, or a REF_Delta stream. The latter
 			type needs its reference object to be locked up in an ODB to form a valid
 			delta chain.
+			If the object at offset is no delta, the size of the list is 1.
 		:param offset: specifies the first byte of the object within this pack"""
 		out = list()
 		while True:
@@ -370,31 +374,7 @@ def collect_streams(self, offset):
 			# END handle type
 		# END while chaining streams
 		return out
-	
-	def to_delta_stream(self, stream_list):
-		"""Convert the given list of streams into a stream which resolves deltas
-		(if availble) when reading from it.
-		:param stream_list: one or more stream objects. If the first stream is a Delta, 
-			there must be at least two streams in the list. The list's last stream
-			must be a non-delta stream.
-		:return: Non-Delta OPackStream object whose stream can be used to obtain 
-			the decompressed resolved data
-		:raise ValueError: if the stream list cannot be handled due to a missing base object"""
-		if len(stream_list) == 1:
-			if stream_list[0].type_id in _delta_types:
-				raise ValueError("Cannot resolve deltas if only one stream is given", stream_list[0].type)
-			# its an object, no need to resolve anything
-			return stream_list[0]
-		# END single object special handling
-		
-		if stream_list[-1].type_id in _delta_types:
-			raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type)
-		# END check stream
-		
-		# just create the respective stream wrapper
-		return DeltaApplyReader(stream_list)
-		
-	
+
 	#} END pack specific
 
 	#{ Read-Database like Interface
@@ -437,8 +417,58 @@ def __init__(self, basename):
 		self._pack = self.PackFileCls("%s.pack" % basename)			# corresponding PackFile instance
 
 
+	def _sha_to_index(self, sha):
+		""":return: index for the given sha, or raise"""
+		index = self._index.sha_to_index(sha)
+		if index is None:
+			raise BadObject(sha)
+		return index
+	
 	def _iter_objects(self, as_stream):
-		raise NotImplementedError
+		"""Iterate over all objects in our index and yield their OInfo or OStream instences"""
+		raise NotImplementedError()
+	
+	def _object(self, sha, as_stream):
+		""":return: OInfo or OStream object providing information about the given sha"""
+		# its a little bit redundant here, but it needs to be efficient
+		offset = self._index.offset(self._sha_to_index(sha))
+		type_id, uncomp_size, data_rela_offset = pack_object_header_info(buffer(self._pack._data, offset))
+		if as_stream:
+			if type_id not in delta_types:
+				packstream = self._pack.stream(offset)
+				return OStream(sha, packstream.type, packstream.size, packstream.stream)
+			# END handle non-deltas
+			
+			# produce a delta stream containing all info
+			# To prevent it from applying the deltas when querying the size, 
+			# we extract it from the delta stream ourselves
+			streams = self.collect_streams_at_offset(offset)
+			buf = streams[0].read(512)
+			offset, src_size = msb_size(buf)
+			offset, target_size = msb_size(buf, offset)
+			
+			streams[0].seek(0)				# assure it can be read by the delta reader
+			dstream = DeltaApplyReader.new(streams)
+			
+			return OStream(sha, dstream.type, target_size, dstream) 
+		else:
+			if type_id not in delta_types:
+				return OInfo(sha, type_id_to_type_map[type_id], uncomp_size)
+			# END handle non-deltas
+			
+			# deltas are a little tougher - unpack the first bytes to obtain
+			# the actual target size, as opposed to the size of the delta data
+			streams = self.collect_streams_at_offset(offset)
+			buf = streams[0].read(512)
+			offset, src_size = msb_size(buf)
+			offset, target_size = msb_size(buf, offset)
+			
+			# collect the streams to obtain the actual object type
+			if streams[-1].type_id in delta_types:
+				raise BadObject(sha, "Could not resolve delta object")
+				
+			return OInfo(sha, streams[-1].type, target_size) 
+		# END handle stream
 
 	#{ Read-Database like Interface
 
@@ -447,14 +477,14 @@ def info(self, sha):
 		:param sha: 20 byte sha1
 		:raise BadObject:
 		:return: OInfo instance"""
-		raise NotImplementedError()
+		return self._object(sha, as_stream=False)
 
 	def stream(self, sha):
 		"""Retrieve an object stream along with its information as identified by the given sha
 		:param sha: 20 byte sha1
 		:raise BadObject: 
 		:return: OStream instance"""
-		raise NotImplementedError()
+		return self._object(sha, as_stream=True)
 
 	#} END Read-Database like Interface
 
@@ -470,4 +500,47 @@ def stream_iter(self):
 		OStream instances"""
 		return self._iter_objects(as_stream=True)
 
-	#} Interface
+	def collect_streams_at_offset(self, offset):
+		"""As the version in the PackFile, but can resolve REF deltas within this pack
+		For more info, see ``collect_streams``
+		:param offset: offset into the pack file at which the object can be found"""
+		streams = self._pack.collect_streams(offset)
+		
+		# try to resolve the last one if needed. It is assumed to be either
+		# a REF delta, or a base object, as OFFSET deltas are resolved by the pack
+		if streams[-1].type_id == REF_DELTA:
+			stream = streams[-1]
+			while stream.type_id in delta_types:
+				if stream.type_id == REF_DELTA:
+					sindex = self._index.sha_to_index(stream.delta_info)
+					if sindex is None:
+						break
+					stream = self._pack.stream(self._index.offset(sindex))
+					streams.append(stream)
+				else:
+					# must be another OFS DELTA - this could happen if a REF 
+					# delta we resolve previously points to an OFS delta. Who 
+					# would do that ;) ? We can handle it though
+					stream = self._pack.stream(stream.delta_info)
+					streams.append(stream)
+				# END handle ref delta
+			# END resolve ref streams
+		# END resolve streams
+		
+		return streams
+		
+	def collect_streams(self, sha):
+		"""As ``PackFile.collect_streams``, but takes a sha instead of an offset.
+		Additionally, ref_delta streams will be resolved within this pack.
+		If this is not possible, the stream will be left alone, hence it is adivsed
+		to check for unresolved ref-deltas and resolve them before attempting to 
+		construct a delta stream.
+		:param sha: 20 byte sha1 specifying the object whose related streams you want to collect
+		:return: list of streams, first being the actual object delta, the last being 
+			a possibly unresolved base object.
+		:raise BadObject:"""
+		return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha)))
+		
+		
+		
+	#} END interface

stream.py

@@ -7,7 +7,8 @@
 from fun import (
 					msb_size,
 					stream_copy, 
-					apply_delta_data
+					apply_delta_data, 
+					delta_types
 				)
 
 from util import (
@@ -19,7 +20,7 @@
 		zlib
 	)
 
-__all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer')
+__all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader')
 
 
 #{ RO Streams
@@ -418,6 +419,50 @@ def seek(self, offset, whence=os.SEEK_SET):
 			raise ValueError("Can only seek to position 0")
 		# END handle offset
 		self._size
+		
+	#{ Interface 
+	
+	@classmethod
+	def new(cls, stream_list):
+		"""Convert the given list of streams into a stream which resolves deltas
+		when reading from it.
+		:param stream_list: two or more stream objects, first stream is a Delta
+			to the object that you want to resolve, followed by N additional delta
+			streams. The list's last stream must be a non-delta stream.
+		:return: Non-Delta OPackStream object whose stream can be used to obtain 
+			the decompressed resolved data
+		:raise ValueError: if the stream list cannot be handled"""
+		if len(stream_list) < 2:
+			raise ValueError("Need at least two streams")
+		# END single object special handling
+		
+		if stream_list[-1].type_id in delta_types:
+			raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type)
+		# END check stream
+		
+		return cls(stream_list)
+		
+	#} END interface
+	
+	
+	#{ OInfo like Interface
+	
+	@property
+	def type(self):
+		return self._bstream.type
+		
+	@property
+	def type_id(self):
+		return self._bstream.type_id
+		
+	@property
+	def size(self):
+		""":return: number of uncompressed bytes in the stream"""
+		return self._size
+		
+	#} END oinfo like interface 
+	
+	
 #} END RO streams
 
 

test/test_pack.py

@@ -5,6 +5,10 @@
 					with_packs_rw,
 					fixture_path
 				)
+from gitdb.stream import (
+							DeltaApplyReader
+						)
+
 from gitdb.pack import (
 							PackIndexFile,
 							PackFile
@@ -71,15 +75,16 @@ def _assert_pack_file(self, pack, version, size):
 
 			# read the stream
 			try:
-				dstream = pack.to_delta_stream(streams)
+				dstream = DeltaApplyReader.new(streams)
 			except ValueError:
 				# ignore these, old git versions use only ref deltas, 
 				# which we havent resolved ( as we are without an index )
+				# Also ignore non-delta streams
 				continue
 			# END get deltastream
 
 			# read all
-			assert len(dstream.read())
+			assert len(dstream.read()) 
 
 			# read chunks
 			# NOTE: the current implementation is safe, it basically transfers