PackedDB: added sha_iter and size methods, these should move to the ObjectDBR actually

Added performance test, packed stream reading still runs into errors, which is interesting as it dealt with the sample packs very well before

Author: Byron

db/pack.py

@@ -28,7 +28,9 @@ class PackedDB(FileDBBase, ObjectDBR, LazyMixin):
 	"""A database operating on a set of object packs"""
 
 	# sort the priority list every N queries
-	_sort_interval = 15
+	# Higher values are better, performance tests don't show this has 
+	# any effect, but it should have one
+	_sort_interval = 500
 
 	def __init__(self, root_path):
 		super(PackedDB, self).__init__(root_path)
@@ -156,4 +158,19 @@ def update_pack_entity_cache(self, force=False):
 		self._sort_entities()
 		return True
 
+	def sha_iter(self):
+		"""Return iterator yielding 20 byte shas for the packed objects in this data base"""
+		sha_list = list()
+		for entity in (item[1] for item in self._entities):
+			index = entity.index()
+			sha_by_index = index.sha
+			for index in xrange(index.size()):
+				yield sha_by_index(index)
+			# END for each index
+		# END for each entity
+	
+	def size(self):
+		""":return: amount of packed objects in this database"""
+		sizes = [item[1].index().size() for item in self._entities]
+		return reduce(lambda x,y: x+y, sizes)
 	#} END interface

test/db/test_pack.py

@@ -34,13 +34,8 @@ def test_writing(self, path):
 		# bang on the cache
 		# access the Entities directly, as there is no iteration interface
 		# yet ( or required for now )
-		sha_list = list()
-		for entity in (item[1] for item in pdb._entities):
-			for index in xrange(entity.index().size()):
-				
-				sha_list.append(entity.index().sha(index))
-			# END for each index
-		# END for each entity
+		sha_list = list(pdb.sha_iter())
+		assert len(sha_list) == pdb.size()
 
 		# hit all packs in random order
 		random.shuffle(sha_list)

test/performance/lib.py

@@ -44,6 +44,7 @@ def setUpAll(cls):
 		except AttributeError:
 			pass
 		cls.gitrepopath = resolve_or_fail(k_env_git_repo)
+		assert cls.gitrepopath.endswith('.git')
 
 
 #} END base classes

test/performance/test_db.py

@@ -1,15 +1,71 @@
 """Performance tests for object store"""
+from lib import (
+	TestBigRepoR 
+	)
+
+from gitdb.db.pack import PackedDB
 
 import sys
+import os
 from time import time
-
-from lib import (
-	TestBigRepoR
-	)
+import random
 
 class TestGitDBPerformance(TestBigRepoR):
 
-	def test_random_access(self):
-		pass
-		# TODO: use the actual db for this
+	def test_pack_random_access(self):
+		pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack"))
+		assert len(pdb._entities) > 1
+		
+		# sha lookup
+		st = time()
+		sha_list = list(pdb.sha_iter())
+		elapsed = time() - st
+		ns = len(sha_list)
+		print >> sys.stderr, "PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed)
+		
+		
+		# sha lookup: best-case and worst case access
+		pdb_pack_info = pdb._pack_info
+		access_times = list()
+		for rand in range(2):
+			if rand:
+				random.shuffle(sha_list)
+			# END shuffle shas
+			st = time()
+			for sha in sha_list:
+				pdb_pack_info(sha)
+			# END for each sha to look up
+			elapsed = time() - st
+			access_times.append(elapsed)
+			
+			# discard cache
+			del(pdb._entities)
+			pdb._entities
+			print >> sys.stderr, "PDB: looked up %i sha (random=%i) in %f s ( %f shas/s )" % (ns, rand, elapsed, ns / elapsed)
+		# END for each random mode
+		elapsed_order, elapsed_rand = access_times
+		
+		# well, its never really sequencial regarding the memory patterns, but it 
+		# shows how well the prioriy cache performs
+		print >> sys.stderr, "PDB: sequential access is %f %% faster than random-access" % (100 - ((elapsed_order / elapsed_rand) * 100))
+		
+		
+		# query info and streams only
+		max_items = 10000			# can wait longer when testing memory
+		for pdb_fun in (pdb.info, pdb.stream):
+			st = time()
+			for sha in sha_list[:max_items]:
+				pdb_fun(sha)
+			elapsed = time() - st
+			print >> sys.stderr, "PDB: Obtained %i object %s by sha in %f s ( %f info/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed)
+		# END for each function
 
+		# retrieve stream and read all
+		max_items = 5000
+		pdb_stream = pdb.stream
+		st = time()
+		for sha in sha_list[:max_items]:
+			stream = pdb_stream(sha)
+			stream.read()
+		elapsed = time() - st
+		print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes in %f s ( %f info/s )" % (max_items, elapsed, max_items / elapsed)