index reading from V2 index files implemeneted and tested.

Added LazyMixin type from git-python

Author: Byron

fun.py

@@ -113,3 +113,4 @@ def stream_copy(read, write, size, chunk_size):
 
 
 #} END routines
+

pack.py

@@ -1 +1,191 @@
 """Contains PackIndex and PackFile implementations"""
+from util import (
+					LockedFD,
+					LazyMixin,
+					file_contents_ro, 
+					unpack_from
+					)
+
+from struct import (
+						pack,
+					)
+
+__all__ = ('PackIndex', 'Pack')
+
+
+class PackIndex(LazyMixin):
+	"""A pack index provides offsets into the corresponding pack, allowing to find
+	locations for offsets faster."""
+	
+	# Dont use slots as we dynamically bind functions for each version, need a dict for this
+	# The slots you see here are just to keep track of our instance variables
+	# __slots__ = ('_indexpath', '_fanout_table', '_data', '_version', 
+	#				'_sha_list_offset', '_crc_list_offset', '_pack_offset', '_pack_64_offset')
+
+	# used in v2 indices
+	_sha_list_offset = 8 + 1024
+
+	def __init__(self, indexpath):
+		super(PackIndex, self).__init__()
+		self._indexpath = indexpath
+	
+	def _set_cache_(self, attr):
+		if attr == "_packfile_checksum":
+			self._packfile_checksum = self._data[-40:-20]
+		elif attr == "_packfile_checksum":
+			self._packfile_checksum = self._data[-20:]
+		elif attr == "_data":
+			lfd = LockedFD(self._indexpath)
+			fd = lfd.open()
+			self._data = file_contents_ro(fd)
+			lfd.rollback()
+		else:
+			# now its time to initialize everything - if we are here, someone wants
+			# to access the fanout table or related properties
+			
+			# CHECK VERSION
+			self._version = (self._data[:4] == '\377tOc' and 2) or 1
+			if self._version == 2:
+				version_id = unpack_from(">L", self._data, 4)[0] 
+				assert version_id == self._version, "Unsupported index version: %i" % version_id
+			# END assert version
+			
+			# SETUP FUNCTIONS
+			# setup our functions according to the actual version
+			for fname in ('entry', 'offset', 'sha', 'crc'):
+				setattr(self, fname, getattr(self, "_%s_v%i" % (fname, self._version)))
+			# END for each function to initialize
+			
+			
+			# INITIALIZE DATA
+			# byte offset is 8 if version is 2, 0 otherwise
+			self._initialize()
+		# END handle attributes
+		
+
+	#{ Access V1
+	
+	def _entry_v1(self, i):
+		""":return: tuple(offset, binsha)"""
+		return unpack_from(">L20s", self._data, 1024 + i*24)[0] 
+	
+	def _offset_v1(self, i):
+		"""see ``_offset_v2``"""
+		return unpack_from(">L", self._data, 1024 + i*24)[0]
+	
+	def _sha_v1(self, i):
+		"""see ``_sha_v2``"""
+		base = 1024 + i*24
+		return self._data[base:base+20]
+		
+	def _crc_v1(self, i):
+		"""unsupported"""
+		return 0
+		
+	#} END access V1
+	
+	#{ Access V2
+	def _entry_v2(self, i):
+		""":return: tuple(offset, binsha, crc)"""
+		return (self._offset_v2(i), self._sha_v2(i), self._crc_v2(i))
+	
+	def _offset_v2(self, i):
+		""":return: 32 or 64 byte offset into pack files. 64 byte offsets will only 
+			be returned if the pack is larger than 4 GiB, or 2^32"""
+		offset = unpack_from(">L", self._data, self._pack_offset + i * 4)[0]
+		
+		# if the high-bit is set, this indicates that we have to lookup the offset
+		# in the 64 bit region of the file. The current offset ( lower 31 bits )
+		# are the index into it
+		if offset & 0x80000000:
+			offset = unpack_from(">Q", self._data, self._pack_64_offset + (self.offset & ~0x80000000) * 8)[0]
+		# END handle 64 bit offset
+		
+		return offset
+		
+	def _sha_v2(self, i):
+		""":return: sha at the given index of this file index instance"""
+		base = self._sha_list_offset + i * 20
+		return self._data[base:base+20]
+		
+	def _crc_v2(self, i):
+		""":return: 4 bytes crc for the object at index i"""
+		return unpack_from(">L", self._data, self._crc_list_offset + i * 4)[0] 
+		
+	#} END access V2
+	
+	#{ Initialization
+	
+	def _initialize(self):
+		"""initialize base data"""
+		self._fanout_table = self._read_fanout((self._version == 2) * 8)
+		
+		if self._version == 2:
+			self._crc_list_offset = self._sha_list_offset + self.size * 20
+			self._pack_offset = self._crc_list_offset + self.size * 4
+			self._pack_64_offset = self._pack_offset + self.size * 4
+		# END setup base
+		
+	def _read_fanout(self, byte_offset):
+		"""Generate a fanout table from our data"""
+		d = self._data
+		out = list()
+		append = out.append
+		for i in range(256):
+			append(unpack_from('>L', d, byte_offset + i*4)[0])
+		# END for each entry
+		return out
+	
+	#} END initialization
+		
+	#{ Properties
+	@property
+	def version(self):
+		return self._version
+		
+	@property
+	def size(self):
+		""":return: amount of objects referred to by this index"""
+		return self._fanout_table[255]
+		
+	@property
+	def packfile_checksum(self):
+		""":return: 20 byte sha representing the sha1 hash of the pack file"""
+		return self._data[-40:-20]
+		
+	@property
+	def indexfile_checksum(self):
+		""":return: 20 byte sha representing the sha1 hash of this index file"""
+		return self._data[-20:]
+		
+	def sha_to_index(self, sha):
+		"""
+		:return: index usable with the ``offset`` or ``entry`` method, or None
+			if the sha was not found in this pack index
+		:param sha: 20 byte sha to lookup"""
+		first_byte = ord(sha[0])
+		lo = 0					# lower index, the left bound of the bisection
+		if first_byte != 0:
+			lo = self._fanout_table[first_byte-1]
+		hi = self._fanout_table[first_byte]		# the upper, right bound of the bisection
+		
+		# bisect until we have the sha
+		while lo < hi:
+			mid = (lo + hi) / 2
+			c = cmp(sha, self.sha(mid))
+			if c < 0:
+				hi = mid
+			elif not c:
+				return mid
+			else:
+				lo = mid
+			# END handle midpoint
+		# END bisect
+		return None
+	
+	#} END properties
+	
+	
+class Pack(LazyMixin):
+	"""A pack is a file written according to the Version 2 for git packs"""
+	

stream.py

@@ -361,7 +361,9 @@ def read(self, size=-1):
 		if win_size < 8:
 			self._cwe = self._cws + 8
 		# END adjust winsize
-		indata = self._m[self._cws:self._cwe]		# another copy ... :(
+		
+		# takes a slice, but doesn't copy the data, it says ... 
+		indata = buffer(self._m, self._cws, self._cwe - self._cws)
 
 		# get the actual window end to be sure we don't use it for computations
 		self._cwe = self._cws + len(indata)

test/db/lib.py

@@ -1,7 +1,7 @@
 """Base classes for object db testing"""
 from gitdb.test.lib import (
 	with_rw_directory,
-	with_packs,
+	with_packs_rw,
 	ZippedStoreShaWriter,
 	TestBase
 	)
@@ -20,7 +20,7 @@
 from cStringIO import StringIO
 
 
-__all__ = ('TestDBBase', 'with_rw_directory', 'with_packs' )
+__all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw' )
 
 class TestDBBase(TestBase):
 	"""Base class providing testing routines on databases"""

test/db/test_pack.py

@@ -4,7 +4,7 @@
 class TestPackDB(TestDBBase):
 
 	@with_rw_directory
-	@with_packs
+	@with_packs_rw
 	def test_writing(self, path):
 		ldb = PackedDB(path)
 		# TODO

test/lib.py

@@ -47,7 +47,7 @@ def wrapper(self):
 	return wrapper
 
 
-def with_packs(func):
+def with_packs_rw(func):
 	"""Function that provides a path into which the packs for testing should be 
 	copied. Will pass on the path to the actual function afterwards"""
 	def wrapper(self, path):

test/test_pack.py

@@ -1,16 +1,38 @@
 """Test everything about packs reading and writing"""
-
 from lib import (
 					TestBase,
 					with_rw_directory, 
-					with_packs
+					with_packs_rw,
+					fixture_path
 				)
-					
+from gitdb.pack import (
+							PackIndex
+						)
+import os
+
 
 class TestPack(TestBase):
 
-	@with_rw_directory
-	@with_packs
-	def test_reading(self, pack_dir):
-		# initialze a pack file for reading
-		pass
+	def test_pack_index(self):
+		# read v2 index information
+		index_file = fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx')
+		index = PackIndex(index_file)
+		
+		assert index.packfile_checksum != index.indexfile_checksum
+		assert index.version == 2
+		assert index.size == 30
+		
+		# get all data of all objects
+		for oidx in xrange(index.size):
+			sha = index.sha(oidx)
+			assert oidx == index.sha_to_index(sha)
+			
+			entry = index.entry(oidx)
+			assert len(entry) == 3
+			
+			assert entry[0] == index.offset(oidx)
+			assert entry[1] == sha
+			assert entry[2] == index.crc(oidx)
+		# END for each object index in indexfile
+		
+		

util.py

@@ -1,7 +1,9 @@
 import binascii
 import os
+import mmap
 import sys
 import errno
+import cStringIO
 
 try:
 	import async.mod.zlib as zlib
@@ -16,6 +18,22 @@
 except ImportError:
     import sha
 
+try:
+	from struct import unpack_from
+except ImportError:
+	from struct import unpack, calcsize
+	__calcsize_cache = dict()
+	def unpack_from(fmt, data, offset=0):
+		try:
+			size = __calcsize_cache[fmt]
+		except KeyError:
+			size = calcsize(fmt)
+			__calcsize_cache[fmt] = size
+		# END exception handling
+		return unpack(fmt, data[offset : offset + size])
+	# END own unpack_from implementation
+
+
 #{ Globals
 
 # A pool distributing tasks, initially with zero threads, hence everything 
@@ -76,6 +94,28 @@ def stream_copy(source, destination, chunk_size=512*1024):
 	# END reading output stream
 	return br
 
+def file_contents_ro(fd, stream=False, allow_mmap=True):
+	""":return: read-only contents of the file represented by the file descriptor fd
+	:param fd: file descriptor opened for reading
+	:param stream: if False, random access is provided, otherwise the stream interface
+		is provided.
+	:param allow_mmap: if True, its allowed to map the contents into memory, which 
+		allows large files to be handled and accessed efficiently. The file-descriptor
+		will change its position if this is False"""
+	try:
+		if allow_mmap:
+			# supports stream and random access
+			return mmap.mmap(fd, 0, access=mmap.ACCESS_READ)
+	except OSError:
+		pass
+	# END exception handling
+	
+	# read manully
+	contents = os.read(fd, os.fstat(fd).st_size)
+	if stream:
+		return cStringIO.StringIO(contents)
+	return contents
+		
 def to_hex_sha(sha):
 	""":return: hexified version  of sha"""
 	if len(sha) == 40:
@@ -93,6 +133,35 @@ def to_bin_sha(sha):
 
 #{ Utilities
 
+class LazyMixin(object):
+	"""
+	Base class providing an interface to lazily retrieve attribute values upon 
+	first access. If slots are used, memory will only be reserved once the attribute
+	is actually accessed and retrieved the first time. All future accesses will 
+	return the cached value as stored in the Instance's dict or slot.
+	"""
+	__slots__ = tuple()
+	
+	def __getattr__(self, attr):
+		"""
+		Whenever an attribute is requested that we do not know, we allow it 
+		to be created and set. Next time the same attribute is reqeusted, it is simply
+		returned from our dict/slots.
+		"""
+		self._set_cache_(attr)
+		# will raise in case the cache was not created
+		return object.__getattribute__(self, attr)
+
+	def _set_cache_(self, attr):
+		""" This method should be overridden in the derived class. 
+		It should check whether the attribute named by attr can be created
+		and cached. Do nothing if you do not know the attribute or call your subclass
+		
+		The derived class may create as many additional attributes as it deems 
+		necessary in case a git command returns more information than represented 
+		in the single attribute."""
+		pass
+
 
 class FDStreamWrapper(object):
 	"""A simple wrapper providing the most basic functions on a file descriptor