Now using the async zlib module if it is available to allow performance gains through multi-threading.

Author: Byron

ext/async

@@ -6,7 +6,7 @@
 	BadObjectType
 	)
 
-import zlib
+from util import zlib
 decompressobj = zlib.decompressobj
 
 

fun.py

@@ -1,4 +1,4 @@
-import zlib
+
 from cStringIO import StringIO
 import errno
 
@@ -7,7 +7,8 @@
 		to_bin_sha,
 		make_sha,
 		write, 
-		close
+		close,
+		zlib
 	)
 
 __all__ = ('OInfo', 'OStream', 'IStream', 'InvalidOInfo', 'InvalidOStream', 

stream.py

@@ -3,9 +3,9 @@
 	OStream, 
 	)
 from gitdb.stream import Sha1Writer
+from gitdb.util import zlib
 
 import sys
-import zlib
 import random
 from array import array
 from cStringIO import StringIO

test/lib.py

@@ -1,5 +1,4 @@
 """Performance data streaming performance"""
-
 from lib import TestBigRepoR
 from gitdb.db import *
 from gitdb.stream import *
@@ -53,7 +52,7 @@ def __init__(self, *args):
 
 class TestObjDBPerformance(TestBigRepoR):
 
-	large_data_size_bytes = 1000*1000*10		# some MiB should do it
+	large_data_size_bytes = 1000*1000*50		# some MiB should do it
 	moderate_data_size_bytes = 1000*1000*1		# just 1 MiB
 
 	@with_rw_directory
@@ -147,33 +146,39 @@ def istream_iter():
 
 		print >> sys.stderr, "Threads(%i): Compressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed)
 
-		
 		# decompress multiple at once, by reading them
+		# chunk size is not important as the stream will not really be decompressed
+		
+		# until its read
 		istream_reader = IteratorReader(iter([ i.sha for i in istreams ]))
 		ostream_reader = ldb.stream_async(istream_reader)
 
 		chunk_task = TestStreamReader(ostream_reader, "chunker", None)
 		output_reader = pool.add_task(chunk_task)
+		output_reader.task().max_chunksize = 1
 
 		st = time()
 		assert len(output_reader.read(nsios)) == nsios
 		elapsed = time() - st
 
-		print >> sys.stderr, "Threads(%i): Decompressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed)
+		print >> sys.stderr, "Threads(%i): Decompressed %i KiB of data in loose odb in %f s ( %f Read KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed)
 
 		# store the files, and read them back. For the reading, we use a task 
 		# as well which is chunked into one item per task. Reading all will
 		# very quickly result in two threads handling two bytestreams of 
 		# chained compression/decompression streams
 		reader = IteratorReader(istream_iter())
 		istream_reader = ldb.store_async(reader)
+		istream_reader.task().max_chunksize = 1
 
 		istream_to_sha = lambda items: [ i.sha for i in items ]
 		istream_reader.set_post_cb(istream_to_sha)
 
 		ostream_reader = ldb.stream_async(istream_reader)
+		
 		chunk_task = TestStreamReader(ostream_reader, "chunker", None)
 		output_reader = pool.add_task(chunk_task)
+		output_reader.max_chunksize = 1
 
 		st = time()
 		assert len(output_reader.read(nsios)) == nsios

test/performance/test_stream.py

@@ -13,14 +13,14 @@
 	NULL_HEX_SHA
 	)
 
+from gitdb.util import zlib
 from gitdb.typ import (
 	str_blob_type
 	)
 
 from cStringIO import StringIO
 import tempfile
 import os
-import zlib
 
 
 

test/test_stream.py

@@ -2,6 +2,12 @@
 import os
 import errno
 
+try:
+	import async.mod.zlib as zlib
+except ImportError:
+	import zlib
+# END try async zlib
+
 from async import ThreadPool
 
 try: