CompoundDB: implemented simple dict base first-level cache which really helps to improve performance in real-world applications, which need to quickly determine whether objects are in or out for instance, as it happens during index_to_tree conversion

Added separate pack streaming test which shows only a throughput of 250 streams / s in a densely packed pack, and about 3.5 MiB of data throughput. Performance tests show that half the time is spent in collecting the numerous deltas, the other one in decompressing and applying them
Fixed broken performance tests

Author: Byron

db/base.py

@@ -183,19 +183,29 @@ class CompoundDB(ObjectDBR, LazyMixin, CachingDB):
 	
 	Databases are stored in the lazy-loaded _dbs attribute.
 	Define _set_cache_ to update it with your databases"""
-	
 	def _set_cache_(self, attr):
 		if attr == '_dbs':
 			self._dbs = list()
+		elif attr == '_db_cache':
+			self._db_cache = dict()
+		else:
+			super(CompoundDB, self)._set_cache_(attr)
 
 	def _db_query(self, sha):
 		""":return: database containing the given 20 or 40 byte sha
 		:raise BadObject:"""
 		# most databases use binary representations, prevent converting 
 		# it everytime a database is being queried
 		sha = to_bin_sha(sha)
+		try:
+			return self._db_cache[sha]
+		except KeyError:
+			pass
+		# END first level cache
+		
 		for db in self._dbs:
 			if db.has_object(sha):
+				self._db_cache[sha] = db
 				return db
 		# END for each database
 		raise BadObject(sha)
@@ -232,6 +242,8 @@ def databases(self):
 		return tuple(self._dbs)
 
 	def update_cache(self, force=False):
+		# something might have changed, clear everything
+		self._db_cache.clear()
 		stat = False
 		for db in self._dbs:
 			if isinstance(db, CachingDB):

db/git.py

@@ -57,8 +57,9 @@ def _set_cache_(self, attr):
 
 			# finally set the value
 			self._loose_db = loose_db
-			
-		# END handle dbs
+		else:
+			super(GitDB, self)._set_cache_(attr)
+		# END handle attrs
 
 	#{ ObjectDBW interface
 

db/pack.py

@@ -46,7 +46,7 @@ def __init__(self, root_path):
 	def _set_cache_(self, attr):
 		if attr == '_entities':
 			self._entities = list()
-			self.update_cache()
+			self.update_cache(force=True)
 		# END handle entities initialization
 
 	def _sort_entities(self):

db/ref.py

@@ -21,7 +21,9 @@ def _set_cache_(self, attr):
 		if attr == '_dbs':
 			self._dbs = list()
 			self._update_dbs_from_ref_file()
-		# END handle dbs
+		else:
+			super(ReferenceDB, self)._set_cache_(attr)
+		# END handle attrs
 
 	def _update_dbs_from_ref_file(self):
 		dbcls = self.ObjectDBCls

exc.py

@@ -1,4 +1,5 @@
 """Module with common exceptions"""
+from util import to_hex_sha
 
 class ODBError(Exception):
 	"""All errors thrown by the object database"""
@@ -7,7 +8,11 @@ class InvalidDBRoot(ODBError):
 	"""Thrown if an object database cannot be initialized at the given path"""
 
 class BadObject(ODBError):
-	"""The object with the given SHA does not exist"""
+	"""The object with the given SHA does not exist. Instantiate with the 
+	failed sha"""
+	
+	def __str__(self):
+		return "BadObject: %s" % to_hex_sha(self.args[0])
 
 class BadObjectType(ODBError):
 	"""The object had an unsupported type"""

test/performance/test_pack.py

@@ -72,8 +72,10 @@ def test_pack_random_access(self):
 		total_kib = total_size / 1000
 		print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed)
 
-	
-		print >> sys.stderr, "Endurance run: verify streaming of %i objects (crc and sha)" % ns
+	def _disabled_test_correctness(self):
+		pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack"))
+		# disabled for now as it used to work perfectly, checking big repositories takes a long time
+		print >> sys.stderr, "Endurance run: verify streaming of objects (crc and sha)"
 		for crc in range(2):
 			count = 0
 			st = time()

test/performance/test_pack_streaming.py

@@ -0,0 +1,33 @@
+"""Specific test for pack streams only"""
+from lib import (
+	TestBigRepoR 
+	)
+
+from gitdb.db.pack import PackedDB
+
+import os
+import sys
+from time import time
+
+class TestPackStreamingPerformance(TestBigRepoR):
+	
+	def test_stream_reading(self):
+		pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack"))
+		
+		# streaming only, meant for --with-profile runs
+		ni = 5000
+		count = 0
+		pdb_stream = pdb.stream
+		total_size = 0
+		st = time()
+		for sha in pdb.sha_iter():
+			if count == ni:
+				break
+			stream = pdb_stream(sha)
+			stream.read()
+			total_size += stream.size
+			count += 1
+		elapsed = time() - st
+		total_kib = total_size / 1000
+		print >> sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (ni, total_kib, total_kib/elapsed , elapsed, ni / elapsed)
+		

test/performance/test_stream.py

@@ -1,6 +1,7 @@
 """Performance data streaming performance"""
 from lib import TestBigRepoR
 from gitdb.db import *
+from gitdb.base import *
 from gitdb.stream import *
 from gitdb.util import pool
 from gitdb.typ import str_blob_type