Fixed implementation after design change to deal with it - all tests run, but next there will have to be more through testing

Author: Byron

lib/git/__init__.py

@@ -22,5 +22,8 @@
 from git.index import *
 from git.utils import LockFile, BlockingLockFile
 
+# odb is NOT imported intentionally - if you really want it, you should get it 
+# yourself as its part of the core
+
 __all__ = [ name for name, obj in locals().items()
             if not (name.startswith('_') or inspect.ismodule(obj)) ]

lib/git/objects/base.py

@@ -76,12 +76,14 @@ def _set_cache_(self, attr):
 		Retrieve object information
 		"""
 		if attr	 == "size":
-			typename, self.size = self.repo.odb.info(self.sha)
-			assert typename == self.type, _assertion_msg_format % (self.sha, typename, self.type)
+			oinfo = self.repo.odb.info(self.sha)
+			self.size = oinfo.size
+			assert oinfo.type == self.type, _assertion_msg_format % (self.sha, oinfo.type, self.type)
 		elif attr == "data":
-			typename, self.size, stream = self.repo.odb.stream(self.sha)
-			self.data = stream.read()	# once we have an own odb, we can delay reading
-			assert typename == self.type, _assertion_msg_format % (self.sha, typename, self.type)
+			ostream = self.repo.odb.stream(self.sha)
+			self.size = ostream.size
+			self.data = ostream.read()
+			assert ostream.type == self.type, _assertion_msg_format % (self.sha, ostream.type, self.type)
 		else:
 			super(Object,self)._set_cache_(attr)
 
@@ -124,14 +126,13 @@ def __repr__(self):
 	def data_stream(self):
 		""" :return:  File Object compatible stream to the uncompressed raw data of the object
 		:note: returned streams must be read in order"""
-		type, size, stream = self.repo.odb.stream(self.sha)
-		return stream
+		return self.repo.odb.stream(self.sha)
 
 	def stream_data(self, ostream):
 		"""Writes our data directly to the given output stream
 		:param ostream: File object compatible stream object.
 		:return: self"""
-		type, size, istream = self.repo.odb.stream(self.sha)
+		istream = self.repo.odb.stream(self.sha)
 		stream_copy(istream, ostream)
 		return self
 

lib/git/objects/commit.py

@@ -9,6 +9,7 @@
 import git.stats as stats
 from git.actor import Actor
 from tree import Tree
+from git.odb import IStream
 from cStringIO import StringIO
 import base
 import utils
@@ -346,7 +347,8 @@ def create_from_tree(cls, repo, tree, message, parent_commits=None, head=False):
 		streamlen = stream.tell()
 		stream.seek(0)
 
-		new_commit.sha = repo.odb.store(cls.type, streamlen, stream, sha_as_hex=True)
+		istream = repo.odb.store(IStream(cls.type, streamlen, stream))
+		new_commit.sha = istream.sha
 
 		if head:
 			try:

lib/git/odb/__init__.py

@@ -1,2 +1,6 @@
 """Initialize the object database module"""
 
+# default imports
+from db import *
+from stream import *
+

lib/git/odb/db.py

@@ -8,7 +8,10 @@
 
 from stream import (
 		DecompressMemMapReader,
-		FDCompressedSha1Writer
+		FDCompressedSha1Writer,
+		Sha1Writer,
+		OStream,
+		OInfo
 	)
 
 from utils import (
@@ -34,11 +37,13 @@
 import os
 
 
+__all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'LooseObjectDB', 'PackedDB', 
+			'CompoundDB', 'ReferenceDB', 'GitObjectDB' )
+
 class ObjectDBR(object):
 	"""Defines an interface for object database lookup.
 	Objects are identified either by hex-sha (40 bytes) or 
 	by sha (20 bytes)"""
-	__slots__ = tuple()
 
 	def __contains__(self, sha):
 		return self.has_obj
@@ -52,35 +57,34 @@ def has_object(self, sha):
 		raise NotImplementedError("To be implemented in subclass")
 
 	def info(self, sha):
-		""" :return: ODB_Info instance
+		""" :return: OInfo instance
 		:param sha: 40 bytes hexsha or 20 bytes binary sha
 		:raise BadObject:"""
 		raise NotImplementedError("To be implemented in subclass")
 
 	def info_async(self, input_channel):
 		"""Retrieve information of a multitude of objects asynchronously
 		:param input_channel: Channel yielding the sha's of the objects of interest
-		:return: Channel yielding ODB_Info|InvalidODB_Info, in any order"""
+		:return: Channel yielding OInfo|InvalidOInfo, in any order"""
 		raise NotImplementedError("To be implemented in subclass")
 
 	def stream(self, sha):
-		""":return: ODB_OStream instance
+		""":return: OStream instance
 		:param sha: 40 bytes hexsha or 20 bytes binary sha
 		:raise BadObject:"""
 		raise NotImplementedError("To be implemented in subclass")
 
 	def stream_async(self, input_channel):
-		"""Retrieve the ODB_OStream of multiple objects
+		"""Retrieve the OStream of multiple objects
 		:param input_channel: see ``info``
 		:param max_threads: see ``ObjectDBW.store``
-		:return: Channel yielding ODB_OStream|InvalidODB_OStream instances in any order"""
+		:return: Channel yielding OStream|InvalidOStream instances in any order"""
 		raise NotImplementedError("To be implemented in subclass")
 
 	#} END query interface
 
 class ObjectDBW(object):
 	"""Defines an interface to create objects in the database"""
-	__slots__ = "_ostream"
 
 	def __init__(self, *args, **kwargs):
 		self._ostream = None
@@ -99,12 +103,12 @@ def set_ostream(self, stream):
 	def ostream(self):
 		""":return: overridden output stream this instance will write to, or None
 			if it will write to the default stream"""
-			return self._ostream
+		return self._ostream
 
 	def store(self, istream):
 		"""Create a new object in the database
 		:return: the input istream object with its sha set to its corresponding value
-		:param istream: ODB_IStream compatible instance. If its sha is already set 
+		:param istream: IStream compatible instance. If its sha is already set 
 			to a value, the object will just be stored in the our database format, 
 			in which case the input stream is expected to be in object format ( header + contents ).
 		:raise IOError: if data could not be written"""
@@ -115,30 +119,23 @@ def store_async(self, input_channel):
 		return right away, returning an output channel which receives the results as 
 		they are computed.
 		
-		:return: Channel yielding your ODB_IStream which served as input, in any order.
+		:return: Channel yielding your IStream which served as input, in any order.
 			The IStreams sha will be set to the sha it received during the process, 
 			or its error attribute will be set to the exception informing about the error.
-		:param input_channel: Channel yielding ODB_IStream instance.
+		:param input_channel: Channel yielding IStream instance.
 			As the same instances will be used in the output channel, you can create a map
 			between the id(istream) -> istream
 		:note:As some ODB implementations implement this operation as atomic, they might 
 			abort the whole operation if one item could not be processed. Hence check how 
 			many items have actually been produced."""
-		# a trivial implementation, ignoring the threads for now
-		# TODO: add configuration to the class to determine whether we may 
-		# actually use multiple threads, default False of course. If the add
-		shas = list()
-		for args in iter_info:
-			shas.append(self.store(dry_run=dry_run, sha_as_hex=sha_as_hex, *args))
-		return shas
+		raise NotImplementedError("To be implemented in subclass")
 
 	#} END edit interface
 
 
 class FileDBBase(object):
 	"""Provides basic facilities to retrieve files of interest, including 
 	caching facilities to help mapping hexsha's to objects"""
-	__slots__ = ('_root_path', )
 
 	def __init__(self, root_path):
 		"""Initialize this instance to look for its files at the given root path
@@ -164,15 +161,11 @@ def db_path(self, rela_path):
 		return join(self._root_path, rela_path)
 	#} END interface
 
-	#{ Utiltities
-	
-		
-	#} END utilities
 
 
 class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW):
 	"""A database which operates on loose object files"""
-	__slots__ = ('_hexsha_to_file', '_fd_open_flags')
+	
 	# CONFIGURATION
 	# chunks in which data will be copied between streams
 	stream_chunk_size = chunk_size
@@ -238,21 +231,26 @@ def _map_loose_object(self, sha):
 		finally:
 			os.close(fd)
 		# END assure file is closed
+		
+	def set_ostream(self, stream):
+		""":raise TypeError: if the stream does not support the Sha1Writer interface"""
+		if stream is not None and not isinstance(stream, Sha1Writer):
+			raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__)
+		return super(LooseObjectDB, self).set_ostream(stream)
 
 	def info(self, sha):
 		m = self._map_loose_object(sha)
 		try:
-			return loose_object_header_info(m)
+			type, size = loose_object_header_info(m)
+			return OInfo(sha, type, size)
 		finally:
 			m.close()
 		# END assure release of system resources
 
-	def object(self, sha):
+	def stream(self, sha):
 		m = self._map_loose_object(sha)
-		reader = DecompressMemMapReader(m, close_on_deletion = True)
-		type, size = reader.initialize()
-		
-		return type, size, reader
+		type, size, stream = DecompressMemMapReader.new(m, close_on_deletion = True)
+		return OStream(sha, type, size, stream)
 
 	def has_object(self, sha):
 		try:
@@ -263,27 +261,33 @@ def has_object(self, sha):
 		# END check existance
 
 	def store(self, istream):
-		# open a tmp file to write the data to
-		# todo: implement ostream properly
-		fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path)
-		writer = FDCompressedSha1Writer(fd)
+		"""note: The sha we produce will be hex by nature"""
+		assert istream.sha is None, "Direct istream writing not yet implemented"
+		tmp_path = None
+		writer = self.ostream()
+		if writer is None:
+			# open a tmp file to write the data to
+			fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path)
+			writer = FDCompressedSha1Writer(fd)
+		# END handle custom writer
 
 		try:
-			write_object(type, size, stream, writer,
-							close_target_stream=True, chunk_size=self.stream_chunk_size)
-		except:
-			os.remove(tmp_path)
-			raise
-		# END assure tmpfile removal on error
-		
+			try:
+				write_object(istream.type, istream.size, istream.read, writer.write,
+								chunk_size=self.stream_chunk_size)
+			except:
+				if tmp_path:
+					os.remove(tmp_path)
+				raise
+			# END assure tmpfile removal on error
+		finally:
+			if tmp_path:
+				writer.close()
+		# END assure target stream is closed
 
-		# in dry-run mode, we delete the file afterwards
 		sha = writer.sha(as_hex=True)
 
-		if dry_run:
-			os.remove(tmp_path)
-		else:
-			# rename the file into place
+		if tmp_path:
 			obj_path = self.db_path(self.object_path(sha))
 			obj_dir = dirname(obj_path)
 			if not isdir(obj_dir):
@@ -292,11 +296,8 @@ def store(self, istream):
 			rename(tmp_path, obj_path)
 		# END handle dry_run
 
-		if not sha_as_hex:
-			sha = hex_to_bin(sha)
-		# END handle sha format
-		
-		return sha
+		istream.sha = sha
+		return istream
 
 
 class PackedDB(FileDBBase, ObjectDBR):
@@ -320,18 +321,17 @@ class GitObjectDB(LooseObjectDB):
 	:note: for now, we use the git command to do all the lookup, just until he 
 		have packs and the other implementations
 	"""
-	__slots__ = ('_git', )
 	def __init__(self, root_path, git):
 		"""Initialize this instance with the root and a git command"""
 		super(GitObjectDB, self).__init__(root_path)
 		self._git = git
 
 	def info(self, sha):
-		discard, type, size = self._git.get_object_header(sha)
-		return type, size
+		t = self._git.get_object_header(sha)
+		return OInfo(t[0], t[1], t[2])
 
-	def object(self, sha):
+	def stream(self, sha):
 		"""For now, all lookup is done by git itself"""
-		discard, type, size, stream = self._git.stream_object_data(sha)
-		return type, size, stream
+		t = self._git.stream_object_data(sha)
+		return OStream(t[0], t[1], t[2], t[3])
 

lib/git/odb/fun.py

@@ -21,6 +21,8 @@
 # used when dealing with larger streams
 chunk_size = 1000*1000
 
+__all__ = ('is_loose_object', 'loose_object_header_info', 'object_header_info', 
+			'write_object' )
 
 #{ Routines
 
@@ -73,42 +75,34 @@ def object_header_info(m):
 		raise BadObjectType(type_id)
 	# END handle exceptions
 
-def write_object(type, size, source_stream, target_stream, close_target_stream=True, 
-					chunk_size=chunk_size):
+def write_object(type, size, read, write, chunk_size=chunk_size):
 	"""Write the object as identified by type, size and source_stream into the 
 	target_stream
 	
 	:param type: type string of the object
 	:param size: amount of bytes to write from source_stream
-	:param source_stream: stream as file-like object providing at least size bytes
-	:param target_stream: stream as file-like object to receive the data
+	:param read: read method of a stream providing the content data
+	:param write: write method of the output stream
 	:param close_target_stream: if True, the target stream will be closed when 
 		the routine exits, even if an error is thrown
-	:param chunk_size: size of chunks to read from source. Larger values can be beneficial
-		for io performance, but cost more memory as well
 	:return: The actual amount of bytes written to stream, which includes the header and a trailing newline"""
 	tbw = 0												# total num bytes written
 	dbw = 0												# num data bytes written
-	try:
-		# WRITE HEADER: type SP size NULL
-		tbw += target_stream.write("%s %i\0" % (type, size))
-	
-		# WRITE ALL DATA UP TO SIZE
-		while True:
-			cs = min(chunk_size, size-dbw)
-			data_len = target_stream.write(source_stream.read(cs))
-			dbw += data_len
-			if data_len < cs or dbw == size:
-				tbw += dbw
-				break
-			# END check for stream end
-		# END duplicate data
-		return tbw
-	finally:
-		if close_target_stream:
-			target_stream.close()
-		# END handle stream closing
-	# END assure file was closed
 
+	# WRITE HEADER: type SP size NULL
+	tbw += write("%s %i\0" % (type, size))
+
+	# WRITE ALL DATA UP TO SIZE
+	while True:
+		cs = min(chunk_size, size-dbw)
+		data_len = write(read(cs))
+		dbw += data_len
+		if data_len < cs or dbw == size:
+			tbw += dbw
+			break
+		# END check for stream end
+	# END duplicate data
+	return tbw
+
 
 #} END routines