index.add: added progress function support and made sure progress is sent just in time

This adds a dependency to a git update index fix - and I hope it will be put in in time ( including windows version )

Author: Byron

doc/intro.rst

@@ -15,6 +15,7 @@ Requirements
 ============
 
 * Git_ tested with 1.5.3.7
+* Requires Git_ 1.6.5.4 or newer if index.add function is to be used 
 * `Python Nose`_ - used for running the tests
 * `Mock by Michael Foord`_ used for tests. Requires 0.5
 

lib/git/index.py

@@ -16,8 +16,10 @@
 import sys
 import stat
 import subprocess
+import glob
 import git.diff as diff
 
+from errors import GitCommandError
 from git.objects import Blob, Tree, Object, Commit
 from git.utils import SHA1Writer, LazyMixin, ConcurrentWriteOperation, join_path_native
 
@@ -662,7 +664,7 @@ def _preprocess_add_items(self, items):
 
 	@clear_cache
 	@default_index
-	def add(self, items, force=True, **kwargs):
+	def add(self, items, force=True, fprogress=lambda *args: None):
 		"""
 		Add files from the working tree, specific blobs or BaseIndexEntries 
 		to the index. The underlying index file will be written immediately, hence
@@ -717,43 +719,130 @@ def add(self, items, force=True, **kwargs):
 			as the API user usually wants the item to be added even though 
 			they might be excluded.
 		
-		``**kwargs``
-			Additional keyword arguments to be passed to git-update-index, such 
-			as index_only.
-			
+		``fprogress``
+			Function with signature f(path, done=False, item=item) called for each 
+			path to be added, once once it is about to be added where done==False
+			and once after it was added where done=True.
+			item is set to the actual item we handle, either a Path or a BaseIndexEntry
+			Please note that the processed path is not guaranteed to be present 
+			in the index already as the index is currently being processed.
+		
 		Returns
 			List(BaseIndexEntries) representing the entries just actually added.
-		"""
+			
+		Raises
+			GitCommandError if a supplied Path did not exist. Please note that BaseIndexEntry
+			Objects that do not have a null sha will be added even if their paths 
+			do not exist.
+		"""
+		# UTILITIES 
+		def raise_exc(e):
+			raise e
+			
+		def expand_paths(paths):
+			"""Expand the directories in list of paths to the corresponding paths accordingly, 
+			they will always be relative to the repository"""
+			out = list()
+			r = self.repo.git.git_dir
+			rs = r + '/'
+			for path in paths:
+				abs_path = path
+				if not os.path.isabs(abs_path):
+					abs_path = os.path.join(r, path)
+				# END make absolute path
+				
+				# resolve globs if possible
+				if '?' in path or '*' in path or '[' in path:
+					out.extend(f.replace(rs, '') for f in expand_paths(glob.glob(abs_path)))
+					continue
+				# END glob handling 
+				try:
+					for root, dirs, files in os.walk(abs_path, onerror=raise_exc):
+						for rela_file in files:
+							# add relative paths only
+							out.append(os.path.join(root.replace(rs, ''), rela_file))
+						# END for each file in subdir
+					# END for each subdirectory
+				except OSError:
+					# was a file or something that could not be iterated
+					out.append(path)
+				# END path exception handling 
+			# END for each path
+			
+			# NOTE: git will add items multiple times even if a glob overlapped 
+			# with manually specified paths or if paths where specified multiple 
+			# times - we respect that and do not prune
+			return out
+		# END expand helper method
+		
+		def write_path_to_stdin(proc, filepath, item, fmakeexc):
+			"""Write path to proc.stdin and make sure it processes the item, including progress.
+			@return: stdout string"""
+			fprogress(filepath, False, item)
+			try:
+				proc.stdin.write("%s\n" % filepath)
+			except IOError:
+				# pipe broke, usually because some error happend
+				raise fmakeexc()
+			# END write exception handling 
+			proc.stdin.flush()
+			# NOTE: if this hangs, you need at lest git 1.6.5.4 as a git-bugfix
+			# is needed for this
+			# TODO: Rewrite this using hash-object and update index to get 
+			# rid of the bug-fix dependency, updaet intro.rst requirements
+			rval = proc.stdout.readline().strip()	# trigger operation
+			fprogress(filepath, True, item)
+			return rval
+		# END write_path_to_stdin
+			
+		
 		# sort the entries into strings and Entries, Blobs are converted to entries
 		# automatically
 		# paths can be git-added, for everything else we use git-update-index
 		entries_added = list()
 		paths, entries = self._preprocess_add_items(items)
 
+		# HANDLE PATHS 
 		if paths:
-			git_add_output = self.repo.git.add(paths, v=True)
-			# force rereading our entries
+			# to get suitable progress information, pipe paths to stdin
+			args = ("--add", "--replace", "--verbose", "--stdin")
+			proc = self.repo.git.update_index(*args, **{'as_process':True, 'istream':subprocess.PIPE})
+			make_exc = lambda : GitCommandError(("git-update-index",)+args, 128, proc.stderr.readline())
+			filepaths=expand_paths(paths)
+			added_files = list()
+			
+			for filepath in filepaths:
+				write_path_to_stdin(proc, filepath, filepath, make_exc)
+				added_files.append(filepath)
+			# END for each filepath
+			self._flush_stdin_and_wait(proc)	# ignore stdout
+			
+			# force rereading our entries once it is all done
 			del(self.entries)
-			for line in git_add_output.splitlines():
-				# line contains:
-				# add '<path>'
-				added_file = line[5:-1]
-				entries_added.append(self.entries[(added_file,0)])
-			# END for each line
+			entries_added.extend(self.entries[(f,0)] for f in added_files)
 		# END path handling
 
+		# HANDLE ENTRIES
 		if entries:
 			null_mode_entries = [ e for e in entries if e.mode == 0 ]
 			if null_mode_entries:
 				raise ValueError("At least one Entry has a null-mode - please use index.remove to remove files for clarity")
 			# END null mode should be remove
 
+			# HANLDE ENTRY OBJECT CREATION
 			# create objects if required, otherwise go with the existing shas
 			null_entries_indices = [ i for i,e in enumerate(entries) if e.sha == Object.NULL_HEX_SHA ]
 			if null_entries_indices:
-				hash_proc = self.repo.git.hash_object(w=True, stdin_paths=True, istream=subprocess.PIPE, as_process=True)
-				hash_proc.stdin.write('\n'.join(entries[i].path for i in null_entries_indices))
-				obj_ids = self._flush_stdin_and_wait(hash_proc).splitlines()
+				# creating object ids is the time consuming part. Hence we will
+				# send progress for these now.
+				args = ("-w", "--stdin-paths")
+				proc = self.repo.git.hash_object(*args, **{'istream':subprocess.PIPE, 'as_process':True})
+				make_exc = lambda : GitCommandError(("git-hash-object",)+args, 128, proc.stderr.readline())
+				obj_ids = list()
+				for ei in null_entries_indices:
+					entry = entries[ei]
+					obj_ids.append(write_path_to_stdin(proc, entry.path, entry, make_exc))
+				# END for each entry index
 				assert len(obj_ids) == len(null_entries_indices), "git-hash-object did not produce all requested objects: want %i, got %i" % ( len(null_entries_indices), len(obj_ids) )
 
 				# update IndexEntries with new object id
@@ -764,11 +853,22 @@ def add(self, items, force=True, **kwargs):
 				# END for each index
 			# END null_entry handling
 
-			# feed all the data to stdin
-			update_index_proc = self.repo.git.update_index(index_info=True, istream=subprocess.PIPE, as_process=True, **kwargs)
-			update_index_proc.stdin.write('\n'.join(str(e) for e in entries))
+			# feed pure entries to stdin
+			proc = self.repo.git.update_index(index_info=True, istream=subprocess.PIPE, as_process=True)
+			for i, entry in enumerate(entries):
+				progress_sent = i in null_entries_indices
+				if not progress_sent:
+					fprogress(entry.path, False, entry)
+				# it cannot handle too-many newlines in this mode
+				if i != 0:
+					proc.stdin.write('\n')
+				proc.stdin.write(str(entry))
+				proc.stdin.flush()
+				if not progress_sent:
+					fprogress(entry.path, True, entry)
+			# END for each enty
+			self._flush_stdin_and_wait(proc)
 			entries_added.extend(entries)
-			self._flush_stdin_and_wait(update_index_proc)
 		# END if there are base entries
 
 		return entries_added

test/git/test_index.py

@@ -13,8 +13,35 @@
 import glob
 from stat import *
 
+
 class TestTree(TestBase):
 
+	def __init__(self, *args):
+		super(TestTree, self).__init__(*args)
+		self._reset_progress()
+	
+	def _assert_add_progress(self, entries):
+		assert len(entries) == len(self._add_progress_map)
+		for path, call_count in self._add_progress_map.iteritems():
+			assert call_count == 2
+		self._reset_progress()
+
+	def _add_progress(self, path, done, item):
+		"""Called as progress func - we keep track of the proper 
+		call order"""
+		assert item is not None
+		self._add_progress_map.setdefault(path, 0)
+		curval = self._add_progress_map[path]
+		if curval == 0:
+			assert not done
+		if curval == 1:
+			assert done
+		self._add_progress_map[path] = curval + 1
+		
+	def _reset_progress(self):
+		# maps paths to the count of calls
+		self._add_progress_map = dict()
+	
 	def test_index_file_base(self):
 		# read from file
 		index = IndexFile(self.rorepo, fixture_path("index"))
@@ -297,19 +324,29 @@ def mixed_iterator():
 		assert os.path.isfile(os.path.join(rw_repo.git.git_dir, lib_file_path))
 
 		# directory
-		entries = index.add(['lib'])
+		entries = index.add(['lib'], fprogress=self._add_progress)
+		self._assert_add_progress(entries)
 		assert len(entries)>1
 
 		# glob 
-		entries = index.reset(new_commit).add(['lib/*.py'])
+		entries = index.reset(new_commit).add(['lib/git/*.py'], fprogress=self._add_progress)
+		self._assert_add_progress(entries)
 		assert len(entries) == 14
 
+		# same file 
+		entries = index.reset(new_commit).add(['lib/git/head.py']*2, fprogress=self._add_progress)
+		# would fail, test is too primitive to handle this case
+		# self._assert_add_progress(entries)
+		self._reset_progress()
+		assert len(entries) == 2
+		
 		# missing path
 		self.failUnlessRaises(GitCommandError, index.reset(new_commit).add, ['doesnt/exist/must/raise'])
 
 		# blob from older revision overrides current index revision
 		old_blob = new_commit.parents[0].tree.blobs[0]
-		entries = index.reset(new_commit).add([old_blob])
+		entries = index.reset(new_commit).add([old_blob], fprogress=self._add_progress)
+		self._assert_add_progress(entries)
 		assert index.entries[(old_blob.path,0)].sha == old_blob.sha and len(entries) == 1 
 
 		# mode 0 not allowed
@@ -319,14 +356,16 @@ def mixed_iterator():
 		# add new file
 		new_file_relapath = "my_new_file"
 		new_file_path = self._make_file(new_file_relapath, "hello world", rw_repo)
-		entries = index.reset(new_commit).add([BaseIndexEntry((010644, null_sha, 0, new_file_relapath))])
+		entries = index.reset(new_commit).add([BaseIndexEntry((010644, null_sha, 0, new_file_relapath))], fprogress=self._add_progress)
+		self._assert_add_progress(entries)
 		assert len(entries) == 1 and entries[0].sha != null_sha
 
 		# add symlink
 		if sys.platform != "win32":
 			link_file = os.path.join(rw_repo.git.git_dir, "my_real_symlink")
 			os.symlink("/etc/that", link_file)
-			entries = index.reset(new_commit).add([link_file])
+			entries = index.reset(new_commit).add([link_file], fprogress=self._add_progress)
+			self._assert_add_progress(entries)
 			assert len(entries) == 1 and S_ISLNK(entries[0].mode)
 			print "%o" % entries[0].mode
 		# END real symlink test 
@@ -336,7 +375,8 @@ def mixed_iterator():
 		link_target = "/etc/that"
 		fake_symlink_path = self._make_file(fake_symlink_relapath, link_target, rw_repo)
 		fake_entry = BaseIndexEntry((0120000, null_sha, 0, fake_symlink_relapath))
-		entries = index.reset(new_commit).add([fake_entry])
+		entries = index.reset(new_commit).add([fake_entry], fprogress=self._add_progress)
+		self._assert_add_progress(entries)
 		assert entries[0].sha != null_sha
 		assert len(entries) == 1 and S_ISLNK(entries[0].mode)
 
@@ -363,4 +403,3 @@ def mixed_iterator():
 			open(fake_symlink_path,'rb').read() == link_target 
 		else:
 			assert S_ISLNK(os.lstat(fake_symlink_path)[ST_MODE])
-