Removed compression flag from IStream and OStream types, as a valid object will always be compressed if generated by the system ( even future memory db's will compress it )

loose db: implemented direct stream copy, indicated by a sha set in the IStream, including test. This will be the case once Packs are exploded for instance

Author: Byron

lib/git/odb/db.py

@@ -29,7 +29,8 @@
 from fun import ( 
 	chunk_size,
 	loose_object_header_info, 
-	write_object
+	write_object,
+	stream_copy
 	)
 
 import tempfile
@@ -262,7 +263,6 @@ def has_object(self, sha):
 
 	def store(self, istream):
 		"""note: The sha we produce will be hex by nature"""
-		assert istream.sha is None, "Direct istream writing not yet implemented"
 		tmp_path = None
 		writer = self.ostream()
 		if writer is None:
@@ -273,8 +273,13 @@ def store(self, istream):
 
 		try:
 			try:
-				write_object(istream.type, istream.size, istream.read, writer.write,
-								chunk_size=self.stream_chunk_size)
+				if istream.sha is not None:
+					stream_copy(istream.read, writer.write, istream.size, self.stream_chunk_size)
+				else:
+					# write object with header, we have to make a new one
+					write_object(istream.type, istream.size, istream.read, writer.write,
+									chunk_size=self.stream_chunk_size)
+				# END handle direct stream copies
 			except:
 				if tmp_path:
 					os.remove(tmp_path)
@@ -285,7 +290,7 @@ def store(self, istream):
 				writer.close()
 		# END assure target stream is closed
 
-		sha = writer.sha(as_hex=True)
+		sha = istream.sha or writer.sha(as_hex=True)
 
 		if tmp_path:
 			obj_path = self.db_path(self.object_path(sha))

lib/git/odb/stream.py

@@ -69,15 +69,6 @@ def __new__(cls, sha, type, size, stream, *args, **kwargs):
 
 	def __init__(self, *args, **kwargs):
 		tuple.__init__(self)
-	#{ Interface 
-	
-	def is_compressed(self):
-		""":return: True if reads of this stream yield zlib compressed data. Default False
-		:note: this does not imply anything about the actual internal storage.
-			Hence the data could be uncompressed, but read compressed, or vice versa"""
-		return False
-		
-	#} END interface
 
 	#{ Stream Reader Interface 
 
@@ -97,11 +88,11 @@ class IStream(list):
 	The only method your content stream must support is 'read'"""
 	__slots__ = tuple()
 
-	def __new__(cls, type, size, stream, sha=None, compressed=False):
-		return list.__new__(cls, (sha, type, size, stream, compressed, None))
+	def __new__(cls, type, size, stream, sha=None):
+		return list.__new__(cls, (sha, type, size, stream, None))
 
-	def __init__(self, type, size, stream, sha=None, compressed=None):
-		list.__init__(self, (sha, type, size, stream, compressed, None))
+	def __init__(self, type, size, stream, sha=None):
+		list.__init__(self, (sha, type, size, stream, None))
 
 	#{ Interface 
 
@@ -117,11 +108,11 @@ def binsha(self):
 
 	def _error(self):
 		""":return: the error that occurred when processing the stream, or None"""
-		return self[5]
+		return self[4]
 
 	def _set_error(self, exc):
 		"""Set this input stream to the given exc, may be None to reset the error"""
-		self[5] = exc
+		self[4] = exc
 
 	error = property(_error, _set_error)
 
@@ -172,13 +163,6 @@ def _set_stream(self, stream):
 	stream = property(_stream, _set_stream)
 
 	#} END odb info interface 
-	
-	#{ OStream interface 
-	
-	def is_compressed(self):
-		return self[4]
-		
-	#} END OStream interface
 
 
 class InvalidOInfo(tuple):
@@ -397,6 +381,7 @@ def read(self, size=-1):
 class Sha1Writer(object):
 	"""Simple stream writer which produces a sha whenever you like as it degests
 	everything it is supposed to write"""
+	__slots__ = "sha1"
 
 	def __init__(self):
 		self.sha1 = make_sha("")

test/git/test_odb.py

@@ -31,7 +31,8 @@ def close(self):
 
 		def _assert(self):
 			assert self.was_read
-	
+
+
 class DeriveTest(OStream):
 	def __init__(self, sha, type, size, stream, *args, **kwargs):
 		self.myarg = kwargs.pop('myarg')
@@ -41,6 +42,27 @@ def _assert(self):
 		assert self.args
 		assert self.myarg
 
+
+class ZippedStoreShaWriter(Sha1Writer):
+	"""Remembers everything someone writes to it"""
+	__slots__ = ('buf', 'zip')
+	def __init__(self):
+		Sha1Writer.__init__(self)
+		self.buf = StringIO()
+		self.zip = zlib.compressobj(1)	# fastest
+	
+	def __getattr__(self, attr):
+		return getattr(self.buf, attr)
+	
+	def write(self, data):
+		alen = Sha1Writer.write(self, data)
+		self.buf.write(self.zip.compress(data))
+		return alen
+		
+	def close(self):
+		self.buf.write(self.zip.flush())
+
+
 #} END stream utilitiess
 
 
@@ -68,15 +90,11 @@ def test_streams(self):
 		ostream.read(20)
 		assert stream.bytes == 20
 
-		# defaults false
-		assert not ostream.is_compressed()
-		
 		# derive with own args
 		DeriveTest(sha, Blob.type, s, stream, 'mine',myarg = 3)._assert()
 
 		# test istream
 		istream = IStream(Blob.type, s, stream)
-		assert not istream.is_compressed()
 		assert istream.sha == None
 		istream.sha = sha
 		assert istream.sha == sha
@@ -94,6 +112,10 @@ def test_streams(self):
 		istream.stream = None
 		assert istream.stream is None
 
+		assert istream.error is None
+		istream.error = Exception()
+		assert isinstance(istream.error, Exception)
+		
 	def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None):
 		"""Make stream tests - the orig_stream is seekable, allowing it to be 
 		rewound and reused
@@ -218,13 +240,14 @@ def _assert_object_writing(self, db):
 		"""General tests to verify object writing, compatible to ObjectDBW
 		:note: requires write access to the database"""
 		# start in 'dry-run' mode, using a simple sha1 writer
-		ostreams = (Sha1Writer, None)
+		ostreams = (ZippedStoreShaWriter, None)
 		for ostreamcls in ostreams:
 			for data in self.all_data:
 				dry_run = ostreamcls is not None
 				ostream = None
 				if ostreamcls is not None:
 					ostream = ostreamcls()
+					assert isinstance(ostream, Sha1Writer)
 				# END create ostream
 
 				prev_ostream = db.set_ostream(ostream)
@@ -252,6 +275,26 @@ def _assert_object_writing(self, db):
 				else:
 					self.failUnlessRaises(BadObject, db.info, sha)
 					self.failUnlessRaises(BadObject, db.stream, sha)
+					
+					# DIRECT STREAM COPY
+					# our data hase been written in object format to the StringIO
+					# we pasesd as output stream. No physical database representation
+					# was created.
+					# Test direct stream copy of object streams, the result must be 
+					# identical to what we fed in
+					ostream.seek(0)
+					istream.stream = ostream
+					assert istream.sha is not None
+					prev_sha = istream.sha
+					
+					db.set_ostream(ZippedStoreShaWriter())
+					db.store(istream)
+					assert istream.sha == prev_sha
+					new_ostream = db.ostream()
+					
+					# note: only works as long our store write uses the same compression
+					# level, which is zip
+					assert ostream.getvalue() == new_ostream.getvalue()
 			# END for each data set
 		# END for each dry_run mode