Implemented gitdb, it should be a fully functional git database with full read support, and the ability to write loose objects

Author: Byron

db/base.py

@@ -1,13 +1,19 @@
 """Contains implementations of database retrieveing objects"""
 from gitdb.util import (
 		pool,
-		join
+		join, 
+		LazyMixin, 
+		to_bin_sha
 	)
 
+from gitdb.exc import BadObject
+
 from async import (
 		ChannelThreadTask
 	)
 
+from itertools import chain
+
 
 __all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB')
 
@@ -182,22 +188,40 @@ def _set_cache_(self, attr):
 		if attr == '_dbs':
 			self._dbs = list()
 
+	def _db_query(self, sha):
+		""":return: database containing the given 20 or 40 byte sha
+		:raise BadObject:"""
+		# most databases use binary representations, prevent converting 
+		# it everytime a database is being queried
+		sha = to_bin_sha(sha)
+		for db in self._dbs:
+			if db.has_object(sha):
+				return db
+		# END for each database
+		raise BadObject(sha)
+	
 	#{ ObjectDBR interface 
 
 	def has_object(self, sha):
-		raise NotImplementedError("To be implemented in subclass")
+		try:
+			self._db_query(sha)
+			return True
+		except BadObject:
+			return False
+		# END handle exceptions
 
 	def info(self, sha):
-		raise NotImplementedError("To be implemented in subclass")
+		return self._db_query(sha).info(sha)
 
 	def stream(self, sha):
-		raise NotImplementedError()
+		return self._db_query(sha).stream(sha)
 
 	def size(self):
-		raise NotImplementedError()
+		""":return: total size of all contained databases"""
+		return reduce(lambda x,y: x+y, (db.size() for db in self._dbs), 0)
 
 	def sha_iter(self):
-		raise NotImplementedError()
+		return chain(*(db.sha_iter() for db in self._dbs))
 
 	#} END object DBR Interface
 

db/git.py

@@ -1,7 +1,8 @@
 from base import (
-					CompoundDB,
-					FileDBBase,
-				)
+						CompoundDB, 
+						ObjectDBW, 
+						FileDBBase
+					)
 
 from loose import LooseObjectDB
 from pack import PackedDB
@@ -13,7 +14,7 @@
 
 __all__ = ('GitDB', )
 
-class GitDB(FileDBBase, CompoundDB):
+class GitDB(FileDBBase, ObjectDBW, CompoundDB):
 	"""A git-style object database, which contains all objects in the 'objects'
 	subdirectory"""
 	# Configuration
@@ -22,7 +23,7 @@ class GitDB(FileDBBase, CompoundDB):
 	ReferenceDBCls = ReferenceDB
 
 	# Directories
-	packs_dir = 'packs'
+	packs_dir = 'pack'
 	loose_dir = ''
 	alternates_dir = os.path.join('info', 'alternates')
 
@@ -31,19 +32,42 @@ def __init__(self, root_path):
 		super(GitDB, self).__init__(root_path)
 
 	def _set_cache_(self, attr):
-		if attr == '_dbs':
+		if attr == '_dbs' or attr == '_loose_db':
 			self._dbs = list()
+			loose_db = None
 			for subpath, dbcls in ((self.packs_dir, self.PackDBCls), 
 									(self.loose_dir, self.LooseDBCls),
 									(self.alternates_dir, self.ReferenceDBCls)):
 				path = self.db_path(subpath)
 				if os.path.exists(path):
 					self._dbs.append(dbcls(path))
+					if dbcls is self.LooseDBCls:
+						loose_db = self._dbs[-1]
+					# END remember loose db
 				# END check path exists
 			# END for each db type
 
 			# should have at least one subdb
 			if not self._dbs:
 				raise InvalidDBRoot(self.root_path())
+			# END handle error
+			
+			# we the first one should have the store method
+			assert loose_db is not None and hasattr(loose_db, 'store'), "First database needs store functionality"
+			
+			# finally set the value
+			self._loose_db = loose_db
+			
 		# END handle dbs
 
+	#{ ObjectDBW interface
+		
+	def store(self, istream):
+		return self._loose_db.store(istream)
+		
+	def ostream(self):
+		return self._loose_db.ostream()
+	
+	def set_ostream(self, ostream):
+		return self._loose_db.set_ostream(ostream)
+	#} END objectdbw interface

db/pack.py

@@ -110,7 +110,7 @@ def sha_iter(self):
 
 	def size(self):
 		sizes = [item[1].index().size() for item in self._entities]
-		return reduce(lambda x,y: x+y, sizes)
+		return reduce(lambda x,y: x+y, sizes, 0)
 
 	#} END object db read
 

db/ref.py

@@ -3,7 +3,7 @@
 				)
 
 import os
-__all__ = ('CompoundDB', )
+__all__ = ('ReferenceDB', )
 
 class ReferenceDB(CompoundDB):
 	"""A database consisting of database referred to in a file"""
@@ -35,7 +35,7 @@ def _update_dbs_from_ref_file(self):
 		ref_paths = list()
 		try:
 			ref_paths = [l.strip() for l in open(self._ref_file, 'r').readlines()]
-		except OSError:
+		except (OSError, IOError):
 			pass
 		# END handle alternates
 
@@ -55,7 +55,16 @@ def _update_dbs_from_ref_file(self):
 		# sort them to maintain order
 		added_paths = sorted(ref_paths_set - cur_ref_paths_set, key=lambda p: ref_paths.index(p))
 		for path in added_paths:
-			self._dbs.append(dbcls(path))
+			try:
+				db = dbcls(path)
+				# force an update to verify path
+				if isinstance(db, CompoundDB):
+					db.databases()
+				# END verification
+				self._dbs.append(db)
+			except Exception, e:
+				# ignore invalid paths or issues
+				pass
 		# END for each path to add
 
 	def update_cache(self, force=False):

test/db/lib.py

@@ -22,7 +22,7 @@
 from cStringIO import StringIO
 
 
-__all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw' )
+__all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw', 'fixture_path')
 
 class TestDBBase(TestBase):
 	"""Base class providing testing routines on databases"""

test/db/test_git.py

@@ -1,9 +1,26 @@
 from lib import *
 from gitdb.db import GitDB
+from gitdb.base import OStream, OInfo
 
-class TestGitDB(TestBase):
+class TestGitDB(TestDBBase):
 
 	def test_reading(self):
-		ldb = GitDB(fixture_path('../../.git/objects')
-		self.fail("todo")
+		gdb = GitDB(fixture_path('../../.git/objects'))
 
+		# we have packs and loose objects, alternates doesn't necessarily exist
+		assert 1 < len(gdb.databases()) < 4
+		
+		# access should be possible
+		gitdb_sha = "5690fd0d3304f378754b23b098bd7cb5f4aa1976"
+		assert isinstance(gdb.info(gitdb_sha), OInfo)
+		assert isinstance(gdb.stream(gitdb_sha), OStream)
+		assert gdb.size() > 200
+		assert len(list(gdb.sha_iter())) == gdb.size()
+		
+	@with_rw_directory
+	def test_writing(self, path):
+		gdb = GitDB(path)
+		
+		# its possible to write objects
+		self._assert_object_writing(gdb)
+		self._assert_object_writing_async(gdb)

test/db/test_ref.py

@@ -1,21 +1,53 @@
 from lib import *
 from gitdb.db import ReferenceDB
 
-class TestReferenceDB(TestBase):
+import os
+		
+class TestReferenceDB(TestDBBase):
+	
+	def make_alt_file(self, alt_path, alt_list):
+		"""Create an alternates file which contains the given alternates.
+		The list can be empty"""
+		alt_file = open(alt_path, "wb")
+		for alt in alt_list:
+			alt_file.write(alt + "\n")
+		alt_file.close()
 
 	@with_rw_directory
 	def test_writing(self, path):
-		# TODO: setup alternate file
-		alternates = 
-		ldb = ReferenceDB(path)
+		null_sha_bin = '\0'  * 20
+		null_sha_hex = "0" * 40
+		
+		alt_path = os.path.join(path, 'alternates')
+		rdb = ReferenceDB(alt_path)
+		assert len(rdb.databases()) == 0
+		assert rdb.size() == 0
+		assert len(list(rdb.sha_iter())) == 0
 
 		# try empty, non-existing
+		assert not rdb.has_object(null_sha_hex)
+		assert not rdb.has_object(null_sha_bin)
 
+		
+		# setup alternate file
 		# add two, one is invalid
+		own_repo_path = fixture_path('../../.git/objects')		# use own repo
+		self.make_alt_file(alt_path, [own_repo_path, "invalid/path"])
+		rdb.update_cache()
+		assert len(rdb.databases()) == 1
+		
+		# we should now find a default revision of ours
+		gitdb_sha = "5690fd0d3304f378754b23b098bd7cb5f4aa1976"
+		assert rdb.has_object(gitdb_sha)
 
 		# remove valid
+		self.make_alt_file(alt_path, ["just/one/invalid/path"])
+		rdb.update_cache()
+		assert len(rdb.databases()) == 0
 
 		# add valid
+		self.make_alt_file(alt_path, [own_repo_path])
+		rdb.update_cache()
+		assert len(rdb.databases()) == 1
 
-		self.fail("todo")