Skip to content

Commit 369e564

Browse files
ByronByron
committed
fix(cmd): don't open stdout when fetching
This allows us to use the main thread to parse stderr to get progress, and resolve assertion failures hopefully once and for all. Relates to gitpython-developers#301
1 parent 36dbe7e commit 369e564

File tree

4 files changed

+105
-96
lines changed

4 files changed

+105
-96
lines changed

‎git/cmd.py‎

Lines changed: 75 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141

4242
execute_kwargs = ('istream', 'with_keep_cwd', 'with_extended_output',
4343
'with_exceptions', 'as_process', 'stdout_as_string',
44-
'output_stream')
44+
'output_stream', 'with_stdout')
4545

4646
log = logging.getLogger('git.cmd')
4747
log.addHandler(logging.NullHandler())
@@ -65,84 +65,6 @@ def _bchr(c):
6565
# Documentation
6666
## @{
6767

68-
def _parse_lines_from_buffer(buf):
69-
line = b''
70-
bi = 0
71-
lb = len(buf)
72-
while bi < lb:
73-
char = _bchr(buf[bi])
74-
bi += 1
75-
76-
if char in (b'\r', b'\n') and line:
77-
yield bi, line
78-
line = b''
79-
else:
80-
line += char
81-
# END process parsed line
82-
# END while file is not done reading
83-
# end
84-
85-
def _read_lines_from_fno(fno, last_buf_list):
86-
buf = os.read(fno, mmap.PAGESIZE)
87-
buf = last_buf_list[0] + buf
88-
89-
bi = 0
90-
for bi, line in _parse_lines_from_buffer(buf):
91-
yield line
92-
# for each line to parse from the buffer
93-
94-
# keep remainder
95-
last_buf_list[0] = buf[bi:]
96-
97-
def _dispatch_single_line(line, handler):
98-
line = line.decode(defenc)
99-
if line and handler:
100-
try:
101-
handler(line)
102-
except Exception:
103-
# Keep reading, have to pump the lines empty nontheless
104-
log.error("Line handler exception on line: %s", line, exc_info=True)
105-
# end
106-
# end dispatch helper
107-
# end single line helper
108-
109-
def _dispatch_lines(fno, handler, buf_list):
110-
lc = 0
111-
last_buf = buf_list[0]
112-
while True:
113-
for line in _read_lines_from_fno(fno, buf_list):
114-
_dispatch_single_line(line, handler)
115-
lc += 1
116-
# for each line
117-
118-
if last_buf == buf_list[0]:
119-
break
120-
121-
last_buf = buf_list[0]
122-
# end endless loop
123-
return lc
124-
# end
125-
126-
def _deplete_buffer(fno, handler, buf_list, wg=None):
127-
lc = 0
128-
while True:
129-
line_count = _dispatch_lines(fno, handler, buf_list)
130-
lc += line_count
131-
if line_count == 0:
132-
break
133-
# end deplete buffer
134-
135-
if buf_list[0]:
136-
_dispatch_single_line(buf_list[0], handler)
137-
lc += 1
138-
# end
139-
140-
if wg:
141-
wg.done()
142-
143-
return lc
144-
# end
145-
14668
def handle_process_output(process, stdout_handler, stderr_handler, finalizer):
14769
"""Registers for notifications to lean that process output is ready to read, and dispatches lines to
14870
the respective line handlers. We are able to handle carriage returns in case progress is sent by that
@@ -156,6 +78,76 @@ def handle_process_output(process, stdout_handler, stderr_handler, finalizer):
15678
fdmap = {process.stdout.fileno(): (stdout_handler, [b'']),
15779
process.stderr.fileno(): (stderr_handler, [b''])}
15880

81+
def _parse_lines_from_buffer(buf):
82+
line = b''
83+
bi = 0
84+
lb = len(buf)
85+
while bi < lb:
86+
char = _bchr(buf[bi])
87+
bi += 1
88+
89+
if char in (b'\r', b'\n') and line:
90+
yield bi, line
91+
line = b''
92+
else:
93+
line += char
94+
# END process parsed line
95+
# END while file is not done reading
96+
# end
97+
98+
def _read_lines_from_fno(fno, last_buf_list):
99+
buf = os.read(fno, mmap.PAGESIZE)
100+
buf = last_buf_list[0] + buf
101+
102+
bi = 0
103+
for bi, line in _parse_lines_from_buffer(buf):
104+
yield line
105+
# for each line to parse from the buffer
106+
107+
# keep remainder
108+
last_buf_list[0] = buf[bi:]
109+
110+
def _dispatch_single_line(line, handler):
111+
line = line.decode(defenc)
112+
if line and handler:
113+
try:
114+
handler(line)
115+
except Exception:
116+
# Keep reading, have to pump the lines empty nontheless
117+
log.error("Line handler exception on line: %s", line, exc_info=True)
118+
# end
119+
# end dispatch helper
120+
# end single line helper
121+
122+
def _dispatch_lines(fno, handler, buf_list):
123+
lc = 0
124+
for line in _read_lines_from_fno(fno, buf_list):
125+
_dispatch_single_line(line, handler)
126+
lc += 1
127+
# for each line
128+
return lc
129+
# end
130+
131+
def _deplete_buffer(fno, handler, buf_list, wg=None):
132+
lc = 0
133+
while True:
134+
line_count = _dispatch_lines(fno, handler, buf_list)
135+
lc += line_count
136+
if line_count == 0:
137+
break
138+
# end deplete buffer
139+
140+
if buf_list[0]:
141+
_dispatch_single_line(buf_list[0], handler)
142+
lc += 1
143+
# end
144+
145+
if wg:
146+
wg.done()
147+
148+
return lc
149+
# end
150+
159151
if hasattr(select, 'poll'):
160152
# poll is preferred, as select is limited to file handles up to 1024 ... . This could otherwise be
161153
# an issue for us, as it matters how many handles our own process has
@@ -483,6 +475,7 @@ def execute(self, command,
483475
as_process=False,
484476
output_stream=None,
485477
stdout_as_string=True,
478+
with_stdout=True,
486479
**subprocess_kwargs
487480
):
488481
"""Handles executing the command on the shell and consumes and returns
@@ -536,6 +529,8 @@ def execute(self, command,
536529
some of the valid kwargs are already set by this method, the ones you
537530
specify may not be the same ones.
538531
532+
:param with_stdout: If True, default True, we open stdout on the created process
533+
539534
:return:
540535
* str(output) if extended_output = False (Default)
541536
* tuple(int(status), str(stdout), str(stderr)) if extended_output = True
@@ -586,7 +581,7 @@ def execute(self, command,
586581
cwd=cwd,
587582
stdin=istream,
588583
stderr=PIPE,
589-
stdout=PIPE,
584+
stdout=with_stdout and PIPE or None,
590585
shell=self.USE_SHELL,
591586
close_fds=(os.name == 'posix'), # unsupported on windows
592587
**subprocess_kwargs

‎git/remote.py‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -550,7 +550,7 @@ def _get_fetch_info_from_stderr(self, proc, progress):
550550

551551
progress_handler = progress.new_message_handler()
552552

553-
def my_progress_handler(line):
553+
for line in proc.stderr.readlines():
554554
for pline in progress_handler(line):
555555
if line.startswith('fatal:') or line.startswith('error:'):
556556
raise GitCommandError(("Error when fetching: %s" % line,), 2)
@@ -563,9 +563,7 @@ def my_progress_handler(line):
563563
# end for each comand code we know
564564
# end for each line progress didn't handle
565565
# end
566-
567-
# We are only interested in stderr here ...
568-
handle_process_output(proc, None, my_progress_handler, finalize_process)
566+
finalize_process(proc)
569567

570568
# read head information
571569
fp = open(join(self.repo.git_dir, 'FETCH_HEAD'), 'rb')
@@ -651,7 +649,8 @@ def fetch(self, refspec=None, progress=None, **kwargs):
651649
else:
652650
args = [refspec]
653651

654-
proc = self.repo.git.fetch(self, *args, with_extended_output=True, as_process=True, v=True, **kwargs)
652+
proc = self.repo.git.fetch(self, *args, with_extended_output=True, as_process=True, with_stdout=True, v=True,
653+
**kwargs)
655654
res = self._get_fetch_info_from_stderr(proc, progress or RemoteProgress())
656655
if hasattr(self.repo.odb, 'update_cache'):
657656
self.repo.odb.update_cache()

‎git/test/fixtures/cat_file.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
import sys
2+
3+
for line in open(sys.argv[1]).readlines():
4+
sys.stdout.write(line)
5+
sys.stderr.write(line)

‎git/test/test_git.py‎

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@
55
# This module is part of GitPython and is released under
66
# the BSD License: http://www.opensource.org/licenses/bsd-license.php
77
import os
8+
import sys
89
import mock
10+
import subprocess
911

1012
from git.test.lib import (
1113
TestBase,
@@ -22,7 +24,6 @@
2224
GitCommandNotFound,
2325
Repo
2426
)
25-
from git.cmd import _deplete_buffer
2627
from gitdb.test.lib import with_rw_directory
2728

2829
from git.compat import PY3
@@ -206,16 +207,25 @@ def test_environment(self, rw_dir):
206207
# end
207208
# end if select.poll exists
208209

209-
def test_dispatch_lines(self):
210+
def test_handle_process_output(self):
211+
from git.cmd import handle_process_output
212+
210213
line_count = 5002
211-
count = [0]
212-
def counter(line):
213-
count[0] += 1
214+
count = [None, 0, 0]
215+
216+
def counter_stdout(line):
217+
count[1] += 1
218+
219+
def counter_stderr(line):
220+
count[2] += 1
221+
222+
proc = subprocess.Popen([sys.executable, fixture_path('cat_file.py'), str(fixture_path('issue-301_stderr'))],
223+
stdin=None,
224+
stdout=subprocess.PIPE,
225+
stderr=subprocess.PIPE,
226+
shell=False)
214227

215-
fd = os.open(fixture_path('issue-301_stderr'), os.O_RDONLY)
216-
buf_list = [b'']
217-
lines_parsed = _deplete_buffer(fd, counter, buf_list)
218-
os.close(fd)
228+
handle_process_output(proc, counter_stdout, counter_stderr, lambda proc: proc.wait())
219229

220-
assert lines_parsed == line_count
221-
assert count[0] == line_count
230+
assert count[1] == line_count
231+
assert count[2] == line_count

0 commit comments

Comments
 (0)