Base implementation and stubs added for git-like db, as well as the reference db ( for the alternates implementation )

Author: Byron

db/base.py

@@ -9,7 +9,7 @@
 	)
 
 
-__all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB')
+__all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB')
 
 
 class ObjectDBR(object):
@@ -66,6 +66,14 @@ def stream_async(self, reader):
 		# base implementation just uses the stream method repeatedly
 		task = ChannelThreadTask(reader, str(self.stream_async), self.stream)
 		return pool.add_task(task)
+	
+	def size(self):
+		""":return: amount of objects in this database"""
+		raise NotImplementedError()
+		
+	def sha_iter(self):
+		"""Return iterator yielding 20 byte shas for all objects in this data base"""
+		raise NotImplementedError()
 
 	#} END query interface
 
@@ -150,6 +158,63 @@ def db_path(self, rela_path):
 	#} END interface
 
 
-class CompoundDB(ObjectDBR):
-	"""A database which delegates calls to sub-databases"""
-	# TODO
+class CachingDB(object):
+	"""A database which uses caches to speed-up access"""
+	
+	#{ Interface 
+	def update_cache(self, force=False):
+		"""Call this method if the underlying data changed to trigger an update
+		of the internal caching structures.
+		:param force: if True, the update must be performed. Otherwise the implementation
+			may decide not to perform an update if it thinks nothing has changed.
+		:return: True if an update was performed as something change indeed"""
+		
+	# END interface
+
+
+class CompoundDB(ObjectDBR, LazyMixin, CachingDB):
+	"""A database which delegates calls to sub-databases.
+	
+	Databases are stored in the lazy-loaded _dbs attribute.
+	Define _set_cache_ to update it with your databases"""
+	
+	def _set_cache_(self, attr):
+		if attr == '_dbs':
+			self._dbs = list()
+	
+	#{ ObjectDBR interface 
+	
+	def has_object(self, sha):
+		raise NotImplementedError("To be implemented in subclass")
+		
+	def info(self, sha):
+		raise NotImplementedError("To be implemented in subclass")
+		
+	def stream(self, sha):
+		raise NotImplementedError()
+
+	def size(self):
+		raise NotImplementedError()
+		
+	def sha_iter(self):
+		raise NotImplementedError()
+		
+	#} END object DBR Interface
+	
+	#{ Interface
+	
+	def databases(self):
+		""":return: tuple of database instances we use for lookups"""
+		return tuple(self._dbs)
+
+	def update_cache(self, force=False):
+		stat = False
+		for db in self._dbs:
+			if isinstance(db, CachingDB):
+				stat |= db.update_cache(force)
+			# END if is caching db
+		# END for each database to update
+		return stat
+	#} END interface
+		
+

db/git.py

@@ -1,33 +1,49 @@
-
-from gitdb.base import (
-								OInfo,
-								OStream
-							)
+from base import (
+					CompoundDB,
+					FileDBBase,
+				)
 
 from loose import LooseObjectDB
+from pack import PackedDB
+from ref import ReferenceDB
+
+from gitdb.util import LazyMixin
+from gitdb.exc import InvalidDBRoot
+import os
 
-__all__ = ('GitObjectDB', )
+__all__ = ('GitDB', )
 
-#class GitObjectDB(CompoundDB, ObjectDBW):
-class GitObjectDB(LooseObjectDB):
-	"""A database representing the default git object store, which includes loose 
-	objects, pack files and an alternates file
+class GitDB(FileDBBase, CompoundDB):
+	"""A git-style object database, which contains all objects in the 'objects'
+	subdirectory"""
+	# Configuration
+	PackDBCls = PackedDB
+	LooseDBCls = LooseObjectDB
+	ReferenceDBCls = ReferenceDB
 
-	It will create objects only in the loose object database.
-	:note: for now, we use the git command to do all the lookup, just until he 
-		have packs and the other implementations
-	"""
-	def __init__(self, root_path, git):
-		"""Initialize this instance with the root and a git command"""
-		super(GitObjectDB, self).__init__(root_path)
-		self._git = git
+	# Directories
+	packs_dir = 'packs'
+	loose_dir = ''
+	alternates_dir = os.path.join('info', 'alternates')
+	
+	def __init__(self, root_path):
+		"""Initialize ourselves on a git objects directory"""
+		super(GitDB, self).__init__(root_path)
 
-	def info(self, sha):
-		t = self._git.get_object_header(sha)
-		return OInfo(*t)
+	def _set_cache_(self, attr):
+		if attr == '_dbs':
+			self._dbs = list()
+			for subpath, dbcls in ((self.packs_dir, self.PackDBCls), 
+									(self.loose_dir, self.LooseDBCls),
+									(self.alternates_dir, self.ReferenceDBCls)):
+				path = self.db_path(subpath)
+				if os.path.exists(path):
+					self._dbs.append(dbcls(path))
+				# END check path exists
+			# END for each db type
+			
+			# should have at least one subdb
+			if not self._dbs:
+				raise InvalidDBRoot(self.root_path())
+		# END handle dbs
 
-	def stream(self, sha):
-		"""For now, all lookup is done by git itself"""
-		t = self._git.stream_object_data(sha)
-		return OStream(*t)
-	

db/loose.py

@@ -24,11 +24,13 @@
 from gitdb.util import (
 		ENOENT,
 		to_hex_sha,
+		hex_to_bin,
 		exists,
 		isdir,
 		mkdir,
 		rename,
 		dirname,
+		basename,
 		join
 	)
 
@@ -186,4 +188,21 @@ def store(self, istream):
 
 		istream.sha = sha
 		return istream
+		
+	def sha_iter(self):
+		# find all files which look like an object, extract sha from there
+		for root, dirs, files in os.walk(self.root_path()):
+			root_base = basename(root)
+			if len(root_base) != 2:
+				continue
+				
+			for f in files:
+				if len(f) != 38:
+					continue
+				yield hex_to_bin(root_base + f)
+			# END for each file
+		# END for each walk iteration
+		
+	def size(self):
+		return len(tuple(self.sha_iter()))
 

db/pack.py

@@ -1,7 +1,8 @@
 """Module containing a database to deal with packs"""
 from base import (
 						FileDBBase, 
-						ObjectDBR
+						ObjectDBR, 
+						CachingDB
 				)
 
 from gitdb.util import (
@@ -18,13 +19,13 @@
 
 import os
 import glob
-__all__ = ('PackedDB', )
 
+__all__ = ('PackedDB', )
 
 #{ Utilities
 
 
-class PackedDB(FileDBBase, ObjectDBR, LazyMixin):
+class PackedDB(FileDBBase, ObjectDBR, CachingDB, LazyMixin):
 	"""A database operating on a set of object packs"""
 
 	# sort the priority list every N queries
@@ -43,9 +44,10 @@ def __init__(self, root_path):
 		self._st_mtime = 0				# last modification data of our root path
 
 	def _set_cache_(self, attr):
-		# currently it can only be our _entities attribute
-		self._entities = list()
-		self.update_pack_entity_cache()
+		if attr == '_entities':
+			self._entities = list()
+			self.update_cache()
+		# END handle entities initialization
 
 	def _sort_entities(self):
 		self._entities.sort(key=lambda l: l[0], reverse=True)
@@ -95,6 +97,20 @@ def info(self, sha):
 	def stream(self, sha):
 		entity, index = self._pack_info(sha)
 		return entity.stream_at_index(index)
+		
+	def sha_iter(self):
+		sha_list = list()
+		for entity in self.entities():
+			index = entity.index()
+			sha_by_index = index.sha
+			for index in xrange(index.size()):
+				yield sha_by_index(index)
+			# END for each index
+		# END for each entity
+	
+	def size(self):
+		sizes = [item[1].index().size() for item in self._entities]
+		return reduce(lambda x,y: x+y, sizes)
 
 	#} END object db read
 
@@ -115,7 +131,7 @@ def store_async(self, reader):
 
 	#{ Interface 
 
-	def update_pack_entity_cache(self, force=False):
+	def update_cache(self, force=False):
 		"""Update our cache with the acutally existing packs on disk. Add new ones, 
 		and remove deleted ones. We keep the unchanged ones
 		:param force: If True, the cache will be updated even though the directory
@@ -162,19 +178,4 @@ def entities(self):
 		""":return: list of pack entities operated upon by this database"""
 		return [ item[1] for item in self._entities ]
 
-	def sha_iter(self):
-		"""Return iterator yielding 20 byte shas for the packed objects in this data base"""
-		sha_list = list()
-		for entity in self.entities():
-			index = entity.index()
-			sha_by_index = index.sha
-			for index in xrange(index.size()):
-				yield sha_by_index(index)
-			# END for each index
-		# END for each entity
-	
-	def size(self):
-		""":return: amount of packed objects in this database"""
-		sizes = [item[1].index().size() for item in self._entities]
-		return reduce(lambda x,y: x+y, sizes)
 	#} END interface

db/ref.py

@@ -1,7 +1,64 @@
-from base import CompoundDB
+from base import (
+					CompoundDB,
+				)
 
+import os
 __all__ = ('CompoundDB', )
 
 class ReferenceDB(CompoundDB):
 	"""A database consisting of database referred to in a file"""
-
+	
+	# Configuration
+	# Specifies the object database to use for the paths found in the alternates
+	# file. If None, it defaults to the GitDB
+	ObjectDBCls = None
+	
+	def __init__(self, ref_file):
+		super(ReferenceDB, self).__init__()
+		self._ref_file = ref_file
+		
+	def _set_cache_(self, attr):
+		if attr == '_dbs':
+			self._dbs = list()
+			self._update_dbs_from_ref_file()
+		# END handle dbs
+		
+	def _update_dbs_from_ref_file(self):
+		dbcls = self.ObjectDBCls
+		if dbcls is None:
+			# late import
+			from git import GitDB
+			dbcls = GitDB
+		# END get db type
+		
+		# try to get as many as possible, don't fail if some are unavailable
+		ref_paths = list()
+		try:
+			ref_paths = [l.strip() for l in open(self._ref_file, 'r').readlines()]
+		except OSError:
+			pass
+		# END handle alternates
+		
+		ref_paths_set = set(ref_paths)
+		cur_ref_paths_set = set(db.root_path() for db in self._dbs)
+		
+		# remove existing
+		for path in (cur_ref_paths_set - ref_paths_set):
+			for i, db in enumerate(self._dbs[:]):
+				if db.root_path() == path:
+					del(self._dbs[i])
+					continue
+				# END del matching db
+		# END for each path to remove
+		
+		# add new
+		# sort them to maintain order
+		added_paths = sorted(ref_paths_set - cur_ref_paths_set, key=lambda p: ref_paths.index(p))
+		for path in added_paths:
+			self._dbs.append(dbcls(path))
+		# END for each path to add
+		
+	def update_cache(self, force=False):
+		# re-read alternates and update databases
+		self._update_dbs_from_ref_file()
+		return super(ReferenceDB, self).update_cache(force)

test/db/lib.py

@@ -3,6 +3,7 @@
 	with_rw_directory,
 	with_packs_rw,
 	ZippedStoreShaWriter,
+	fixture_path,
 	TestBase
 	)
 

test/db/test_git.py

@@ -0,0 +1,9 @@
+from lib import *
+from gitdb.db import GitDB
+		
+class TestGitDB(TestBase):
+	
+	def test_reading(self):
+		ldb = GitDB(fixture_path('../../.git/objects')
+		self.fail("todo")
+		

test/db/test_loose.py

@@ -11,3 +11,8 @@ def test_writing(self, path):
 		self._assert_object_writing(ldb)
 		self._assert_object_writing_async(ldb)
 
+		# verify sha iteration and size
+		shas = list(ldb.sha_iter())
+		assert shas and len(shas[0]) == 20
+		
+		assert len(shas) == ldb.size()