delta-apply now works after fixing a stupid type, instead of i + 1 I wrote i + i ... argh \!

Author: Byron

fun.py

@@ -155,14 +155,16 @@ def stream_copy(read, write, size, chunk_size):
 	return dbw
 
 
-def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file):
+def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file, 
+						target_size):
 	"""Apply data from a delta buffer using a source buffer to the target file, 
 	which will be written to
 	:param src_buf: random access data from which the delta was created
 	:param src_buf_size: size of the source buffer in bytes
 	:param delta_buf_size: size fo the delta buffer in bytes
 	:param delta_buf: random access delta data
 	:param target_file: file like object to write the result to
+	:param target_size: size of the target buffer
 	:note: transcribed to python from the similar routine in patch-delta.c"""
 	i = 0
 	twrite = target_file.write
@@ -180,7 +182,7 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi
 				i += 1
 			if (c & 0x04):
 				cp_off |= (ord(db[i]) << 16)
-				i += i
+				i += 1
 			if (c & 0x08):
 				cp_off |= (ord(db[i]) << 24)
 				i += 1
@@ -196,14 +198,20 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi
 
 			if not cp_size: 
 				cp_size = 0x10000
-			# maybe skip this check ?
-			if (cp_off + cp_size < cp_size or
-			    cp_off + cp_size > src_buf_size):
+			
+			rbound = cp_off + cp_size
+			if (rbound < cp_size or
+			    rbound > src_buf_size or 
+			    cp_size > target_size):
 				break
 			twrite(buffer(src_buf, cp_off, cp_size))
+			target_size -= cp_size
 		elif c:
+			if c > target_size:
+				break
 			twrite(db[i:i+c])
 			i += c
+			target_size -= c
 		else:
 			raise ValueError("unexpected delta opcode 0")
 		# END handle command byte

stream.py

@@ -327,19 +327,15 @@ def __init__(self, stream_list):
 
 	def _set_cache_(self, attr):
 		"""If we are here, we apply the actual deltas"""
-		# fill in delta info structures, providing the source and target buffer
-		# sizes.
-		buffer_offset_list = list()
-		final_target_size = None
+		
+		# prefetch information
+		buffer_info_list = list()
 		max_target_size = 0
 		for dstream in self._dstreams:
 			buf = dstream.read(512)			# read the header information + X
 			offset, src_size = msb_size(buf)
 			offset, target_size = msb_size(buf, offset)
-			if final_target_size is None:
-				final_target_size = target_size
-			# END set final target size
-			buffer_offset_list.append((buffer(buf, offset), offset))
+			buffer_info_list.append((buffer(buf, offset), offset, src_size, target_size))
 			max_target_size = max(max_target_size, target_size)
 		# END for each delta stream
 
@@ -358,7 +354,7 @@ def _set_cache_(self, attr):
 		# Allocate private memory map big enough to hold the first base buffer
 		# We need random access to it
 		bbuf = allocate_memory(base_size)
-		stream_copy(self._bstream.read, bbuf.write, base_size, 256*mmap.PAGESIZE)
+		stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE)
 
 		# allocate memory map large enough for the largest (intermediate) target
 		# We will use it as scratch space for all delta ops. If the final 
@@ -370,7 +366,8 @@ def _set_cache_(self, attr):
 		# work on the op-codes to reconstruct everything.
 		# For the actual copying, we use a seek and write pattern of buffer
 		# slices.
-		for (dbuf, offset), dstream in reversed(zip(buffer_offset_list, self._dstreams)):
+		final_target_size = None
+		for (dbuf, offset, src_size, target_size), dstream in reversed(zip(buffer_info_list, self._dstreams)):
 			# allocate a buffer to hold all delta data - fill in the data for 
 			# fast access. We do this as we know that reading individual bytes
 			# from our stream would be slower than necessary ( although possible )
@@ -381,15 +378,16 @@ def _set_cache_(self, attr):
 			# read the rest from the stream. The size we give is larger than necessary
 			stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE)
 
-			################################################################
-			apply_delta_data(bbuf, len(bbuf), ddata, len(ddata), tbuf)
-			################################################################
+			#######################################################################
+			apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf, target_size)
+			#######################################################################
 
 			# finally, swap out source and target buffers. The target is now the 
 			# base for the next delta to apply
 			bbuf, tbuf = tbuf, bbuf
 			bbuf.seek(0)
 			tbuf.seek(0)
+			final_target_size = target_size
 		# END for each delta to apply
 
 		# its already seeked to 0, constrain it to the actual size

test/performance/test_db.py

@@ -63,9 +63,12 @@ def test_pack_random_access(self):
 		# retrieve stream and read all
 		max_items = 5000
 		pdb_stream = pdb.stream
+		total_size = 0
 		st = time()
 		for sha in sha_list[:max_items]:
 			stream = pdb_stream(sha)
 			stream.read()
+			total_size += stream.size
 		elapsed = time() - st
-		print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes in %f s ( %f info/s )" % (max_items, elapsed, max_items / elapsed)
+		total_kib = total_size / 1000
+		print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed)