Implemented basic info and stream retrieval as well as pack file handling of PackedDB - its now operational. Next up is a performance test

Author: Byron

db/pack.py

@@ -4,29 +4,95 @@
 						ObjectDBR
 				)
 
+from gitdb.util import (
+							to_bin_sha, 
+							LazyMixin
+						)
+
 from gitdb.exc import (
+							BadObject,
 							UnsupportedOperation,
 						)
 
+from gitdb.pack import PackEntity
+
+import os
+import glob
 __all__ = ('PackedDB', )
 
-class PackedDB(FileDBBase, ObjectDBR):
+
+#{ Utilities
+
+
+class PackedDB(FileDBBase, ObjectDBR, LazyMixin):
 	"""A database operating on a set of object packs"""
 
+	# sort the priority list every N queries
+	_sort_interval = 15
+	
 	def __init__(self, root_path):
 		super(PackedDB, self).__init__(root_path)
+		# list of lists with three items:
+		# * hits - number of times the pack was hit with a request
+		# * entity - Pack entity instance
+		# * sha_to_index - PackIndexFile.sha_to_index method for direct cache query
+		# self._entities = list()		# lazy loaded list
+		self._hit_count	= 0				# amount of hits
+		self._st_mtime = 0				# last modification data of our root path
+		
+	def _set_cache_(self, attr):
+		# currently it can only be our _entities attribute
+		self._entities = list()
+		self.update_pack_entity_cache()
+		
+	def _sort_entities(self):
+		self._entities.sort(key=lambda l: l[0], reverse=True)
+		
+	def _pack_info(self, sha):
+		""":return: tuple(entity, index) for an item at the given sha
+		:param sha: 20 or 40 byte sha
+		:raise BadObject:
+		:note: This method is not thread-safe, but may be hit in multi-threaded
+			operation. The worst thing that can happen though is a counter that 
+			was not incremented, or the list being in wrong order. So we safe
+			the time for locking here, lets see how that goes"""
+		# presort ?
+		if self._hit_count % self._sort_interval == 0:
+			self._sort_entities()
+		# END update sorting
+		
+		sha = to_bin_sha(sha)
+		for item in self._entities:
+			index = item[2](sha)
+			if index is not None:
+				item[0] += 1			# one hit for you
+				self._hit_count += 1	# general hit count
+				return (item[1], index)
+			# END index found in pack
+		# END for each item
 
+		# no hit, see whether we have to update packs
+		# NOTE: considering packs don't change very often, we safe this call
+		# and leave it to the super-caller to trigger that
+		raise BadObject(sha)
 
 	#{ Object DB Read 
 
 	def has_object(self, sha):
-		raise NotImplementedError()
+		try:
+			self._pack_info(sha)
+			return True
+		except BadObject:
+			return False
+		# END exception handling
 
 	def info(self, sha):
-		raise NotImplementedError()
+		entity, index = self._pack_info(sha)
+		return entity.info_at_index(index)
 
 	def stream(self, sha):
-		raise NotImplementedError()
+		entity, index = self._pack_info(sha)
+		return entity.stream_at_index(index)
 
 	#} END object db read
 
@@ -39,6 +105,55 @@ def store(self, istream):
 		raise UnsupportedOperation()
 
 	def store_async(self, reader):
+		# TODO: add ObjectDBRW before implementing this
 		raise NotImplementedError()
 
 	#} END object db write
+	
+	
+	#{ Interface 
+	
+	def update_pack_entity_cache(self, force=False):
+		"""Update our cache with the acutally existing packs on disk. Add new ones, 
+		and remove deleted ones. We keep the unchanged ones
+		:param force: If True, the cache will be updated even though the directory
+			does not appear to have changed according to its modification timestamp.
+		:return: True if the packs have been updated so there is new information, 
+			False if there was no change to the pack database"""
+		stat = os.stat(self.root_path())
+		if not force and stat.st_mtime <= self._st_mtime:
+			return False
+		# END abort early on no change
+		self._st_mtime = stat.st_mtime
+		
+		# packs are supposed to be prefixed with pack- by git-convention
+		# get all pack files, figure out what changed
+		pack_files = set(glob.glob(os.path.join(self.root_path(), "pack-*.pack")))
+		our_pack_files = set(item[1].pack().path() for item in self._entities)
+		
+		# new packs
+		for pack_file in (pack_files - our_pack_files):
+			# init the hit-counter/priority with the size, a good measure for hit-
+			# probability. Its implemented so that only 12 bytes will be read
+			entity = PackEntity(pack_file)
+			self._entities.append([entity.pack().size(), entity, entity.index().sha_to_index])
+		# END for each new packfile
+		
+		# removed packs
+		for pack_file in (our_pack_files - pack_files):
+			del_index = -1
+			for i, item in enumerate(self._entities):
+				if item[1].pack().path() == pack_file:
+					del_index = i
+					break
+				# END found index
+			# END for each entity
+			assert del_index != -1
+			del(self._entities[del_index])
+		# END for each removed pack
+		
+		# reinitialize prioritiess
+		self._sort_entities()
+		return True
+	
+	#} END interface

pack.py

@@ -40,6 +40,7 @@
 
 from struct import (
 						pack,
+						unpack,
 					)
 
 from itertools import izip
@@ -196,7 +197,7 @@ def _offset_v2(self, i):
 		# in the 64 bit region of the file. The current offset ( lower 31 bits )
 		# are the index into it
 		if offset & 0x80000000:
-			offset = unpack_from(">Q", self._data, self._pack_64_offset + (self.offset & ~0x80000000) * 8)[0]
+			offset = unpack_from(">Q", self._data, self._pack_64_offset + (offset & ~0x80000000) * 8)[0]
 		# END handle 64 bit offset
 
 		return offset
@@ -291,7 +292,7 @@ def sha_to_index(self, sha):
 			elif not c:
 				return mid
 			else:
-				lo = mid
+				lo = mid + 1
 			# END handle midpoint
 		# END bisect
 		return None
@@ -326,13 +327,15 @@ def _set_cache_(self, attr):
 			fd = ldb.open()
 			self._data = file_contents_ro(fd)
 			ldb.rollback()
-			# TODO: figure out whether we should better keep the lock, or maybe
-			# add a .keep file instead ?
-		else:
+			
 			# read the header information
 			type_id, self._version, self._size = unpack_from(">4sLL", self._data, 0)
-			assert type_id == "PACK", "Pack file format is invalid: %r" % type_id
-			assert self._version in (2, 3), "Cannot handle pack format version %i" % self._version
+			
+			# TODO: figure out whether we should better keep the lock, or maybe
+			# add a .keep file instead ?
+		else: # must be '_size' or '_version'
+			# read header info - we do that just with a file stream
+			type_id, self._version, self._size = unpack(">4sLL", open(self._packpath).read(12))
 		# END handle header
 
 	def _iter_objects(self, start_offset, as_stream=True):
@@ -545,14 +548,23 @@ def info(self, sha):
 		:param sha: 20 byte sha1
 		:raise BadObject:
 		:return: OInfo instance, with 20 byte sha"""
-		return self._object(sha, as_stream=False)
+		return self._object(sha, False)
 
 	def stream(self, sha):
 		"""Retrieve an object stream along with its information as identified by the given sha
 		:param sha: 20 byte sha1
 		:raise BadObject: 
 		:return: OStream instance, with 20 byte sha"""
-		return self._object(sha, as_stream=True)
+		return self._object(sha, True)
+
+	def info_at_index(self, index):
+		"""As ``info``, but uses a PackIndexFile compatible index to refer to the object"""
+		return self._object(None, False, index)
+	
+	def stream_at_index(self, index):
+		"""As ``stream``, but uses a PackIndexFile compatible index to refer to the 
+		object"""
+		return self._object(None, True, index)
 
 	#} END Read-Database like Interface
 

test/db/test_pack.py

@@ -1,12 +1,51 @@
 from lib import *
 from gitdb.db import PackedDB
-		
+from gitdb.test.lib import fixture_path
+
+import os
+import random
+
 class TestPackDB(TestDBBase):
 
 	@with_rw_directory
 	@with_packs_rw
 	def test_writing(self, path):
-		ldb = PackedDB(path)
-		# TODO
+		pdb = PackedDB(path)
+		
+		# on demand, we init our pack cache
+		num_packs = 2
+		assert len(pdb._entities) == num_packs
+		assert pdb._st_mtime != 0
+		
+		# test pack directory changed: 
+		# packs removed - rename a file, should affect the glob
+		pack_path = pdb._entities[0][1].pack().path()
+		new_pack_path = pack_path + "renamed"
+		os.rename(pack_path, new_pack_path)
 
+		pdb.update_pack_entity_cache(force=True)
+		assert len(pdb._entities) == num_packs - 1
+		
+		# packs added
+		os.rename(new_pack_path, pack_path)
+		pdb.update_pack_entity_cache(force=True)
+		assert len(pdb._entities) == num_packs
 
+		# bang on the cache
+		# access the Entities directly, as there is no iteration interface
+		# yet ( or required for now )
+		sha_list = list()
+		for entity in (item[1] for item in pdb._entities):
+			for index in xrange(entity.index().size()):
+				
+				sha_list.append(entity.index().sha(index))
+			# END for each index
+		# END for each entity
+		
+		# hit all packs in random order
+		random.shuffle(sha_list)
+		
+		for sha in sha_list:
+			info = pdb.info(sha)
+			stream = pdb.stream(sha)
+		# END for each sha to query

test/test_pack.py

@@ -5,16 +5,19 @@
 					with_packs_rw,
 					fixture_path
 				)
-from gitdb.stream import (
-							DeltaApplyReader
-						)
+from gitdb.stream import DeltaApplyReader
 
 from gitdb.pack import (
 							PackEntity,
 							PackIndexFile,
 							PackFile
 						)
 
+from gitdb.base import (
+							OInfo, 
+							OStream,
+						)
+
 from gitdb.fun import delta_types
 from gitdb.exc import UnsupportedOperation
 from gitdb.util import to_bin_sha
@@ -140,6 +143,11 @@ def test_pack_entity(self):
 				# we return fully resolved items, which is implied by the sha centric access
 				assert not info.type_id in delta_types
 
+				# try all calls
+				assert len(entity.collect_streams(info.sha))
+				assert isinstance(entity.info(info.sha), OInfo)
+				assert isinstance(entity.stream(info.sha), OStream)
+				
 				# verify the stream
 				try:
 					assert entity.is_valid_stream(info.sha, use_crc=True)