Added multi-threading performance tests which show that, during compression and decompression, it is not a tiny bit faster than without, which is due to the GIL and even separately locked zlib module implementations. The only way to make this faster update the resepctive c modules to drop the gil, and their own locks where possible

Author: Byron

ext/async

@@ -3,6 +3,14 @@
 from lib import TestBigRepoR
 from gitdb.db import *
 from gitdb.stream import *
+from gitdb.util import pool
+from gitdb.typ import str_blob_type
+from gitdb.fun import chunk_size
+
+from async import ( 
+	IteratorReader, 
+	ChannelThreadTask,
+	)
 
 from cStringIO import StringIO
 from time import time
@@ -19,6 +27,30 @@
 	)
 
 
+#{ Utilities
+def read_chunked_stream(stream):
+	total = 0
+	while True:
+		chunk = stream.read(chunk_size)
+		total += len(chunk)
+		if len(chunk) < chunk_size:
+			break
+	# END read stream loop
+	assert total == stream.size
+	return stream
+	
+	
+class TestStreamReader(ChannelThreadTask):
+	"""Expects input streams and reads them in chunks. It will read one at a time, 
+	requireing a queue chunk of size 1"""
+	def __init__(self, *args):
+		super(TestStreamReader, self).__init__(*args)
+		self.fun = read_chunked_stream
+		self.max_chunksize = 1
+	
+
+#} END utilities
+
 class TestObjDBPerformance(TestBigRepoR):
 
 	large_data_size_bytes = 1000*1000*10		# some MiB should do it
@@ -27,14 +59,17 @@ class TestObjDBPerformance(TestBigRepoR):
 	@with_rw_directory
 	def test_large_data_streaming(self, path):
 		ldb = LooseObjectDB(path)
+		string_ios = list()			# list of streams we previously created
 
+		# serial mode 
 		for randomize in range(2):
 			desc = (randomize and 'random ') or ''
 			print >> sys.stderr, "Creating %s data ..." % desc
 			st = time()
 			size, stream = make_memory_file(self.large_data_size_bytes, randomize)
 			elapsed = time() - st
 			print >> sys.stderr, "Done (in %f s)" % elapsed
+			string_ios.append(stream)
 
 			# writing - due to the compression it will seem faster than it is 
 			st = time()
@@ -78,7 +113,70 @@ def test_large_data_streaming(self, path):
 			cs_kib = cs / 1000
 			print >> sys.stderr, "Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks)
 
-			# del db file so git has something to do
+			# del db file so we keep something to do
 			os.remove(db_file)
-			
 		# END for each randomization factor
+		
+		
+		# multi-threaded mode
+		# want two, should be supported by most of todays cpus
+		pool.set_size(2)
+		total_kib = 0
+		nsios = len(string_ios)
+		for stream in string_ios:
+			stream.seek(0)
+			total_kib += len(stream.getvalue()) / 1000
+		# END rewind
+		
+		def istream_iter():
+			for stream in string_ios:
+				stream.seek(0)
+				yield IStream(str_blob_type, len(stream.getvalue()), stream)
+			# END for each stream
+		# END util
+		
+		# write multiple objects at once, involving concurrent compression
+		reader = IteratorReader(istream_iter())
+		istream_reader = ldb.store_async(reader)
+		istream_reader.task().max_chunksize = 1
+		
+		st = time()
+		istreams = istream_reader.read(nsios)
+		assert len(istreams) == nsios
+		elapsed = time() - st
+		
+		print >> sys.stderr, "Threads(%i): Compressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed)
+		
+		
+		# decompress multiple at once, by reading them
+		istream_reader = IteratorReader(iter([ i.sha for i in istreams ]))
+		ostream_reader = ldb.stream_async(istream_reader)
+		
+		chunk_task = TestStreamReader(ostream_reader, "chunker", None)
+		output_reader = pool.add_task(chunk_task)
+		
+		st = time()
+		assert len(output_reader.read(nsios)) == nsios
+		elapsed = time() - st
+		
+		print >> sys.stderr, "Threads(%i): Decompressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed)
+		
+		# store the files, and read them back. For the reading, we use a task 
+		# as well which is chunked into one item per task. Reading all will
+		# very quickly result in two threads handling two bytestreams of 
+		# chained compression/decompression streams
+		reader = IteratorReader(istream_iter())
+		istream_reader = ldb.store_async(reader)
+		
+		istream_to_sha = lambda items: [ i.sha for i in items ]
+		istream_reader.set_post_cb(istream_to_sha)
+		
+		ostream_reader = ldb.stream_async(istream_reader)
+		chunk_task = TestStreamReader(ostream_reader, "chunker", None)
+		output_reader = pool.add_task(chunk_task)
+		
+		st = time()
+		assert len(output_reader.read(nsios)) == nsios
+		elapsed = time() - st
+		
+		print >> sys.stderr, "Threads(%i): Compressed and decompressed and read %i KiB of data in loose odb in %f s ( %f Combined KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed)