gitpython-developers#648 max_chunk_size can be now set to control output_stream behavior

Author: bpiotr

git/cmd.py

@@ -47,7 +47,7 @@
 execute_kwargs = set(('istream', 'with_extended_output',
                       'with_exceptions', 'as_process', 'stdout_as_string',
                       'output_stream', 'with_stdout', 'kill_after_timeout',
-                      'universal_newlines', 'shell', 'env'))
+                      'universal_newlines', 'shell', 'env', 'max_chunk_size'))
 
 log = logging.getLogger(__name__)
 log.addHandler(logging.NullHandler())
@@ -175,8 +175,6 @@ def __setstate__(self, d):
         dict_to_slots_and__excluded_are_none(self, d, excluded=self._excluded_)
 
     # CONFIGURATION
-    # The size in bytes read from stdout when copying git's output to another stream
-    max_chunk_size = io.DEFAULT_BUFFER_SIZE
 
     git_exec_name = "git"           # default that should work on linux and windows
 
@@ -598,6 +596,7 @@ def execute(self, command,
                 universal_newlines=False,
                 shell=None,
                 env=None,
+                max_chunk_size=io.DEFAULT_BUFFER_SIZE,
                 **subprocess_kwargs
                 ):
         """Handles executing the command on the shell and consumes and returns
@@ -643,6 +642,11 @@ def execute(self, command,
 
         :param env:
             A dictionary of environment variables to be passed to `subprocess.Popen`.
+            
+        :param max_chunk_size:
+            Maximum number of bytes in one chunk of data passed to the output_stream in 
+            one invocation of write() method. If the given number is not positive then
+            the default value is used.
 
         :param subprocess_kwargs:
             Keyword arguments to be passed to subprocess.Popen. Please note that
@@ -789,7 +793,8 @@ def _kill_process(pid):
                     stderr_value = stderr_value[:-1]
                 status = proc.returncode
             else:
-                stream_copy(proc.stdout, output_stream, self.max_chunk_size)
+                max_chunk_size = max_chunk_size if max_chunk_size and max_chunk_size > 0 else io.DEFAULT_BUFFER_SIZE
+                stream_copy(proc.stdout, output_stream, max_chunk_size)
                 stdout_value = output_stream
                 stderr_value = proc.stderr.read()
                 # strip trailing "\n"

git/test/test_repo.py

@@ -5,6 +5,7 @@
 # This module is part of GitPython and is released under
 # the BSD License: http://www.opensource.org/licenses/bsd-license.php
 import glob
+import io
 from io import BytesIO
 import itertools
 import os
@@ -201,6 +202,21 @@ def _assert_empty_repo(self, repo):
             pass
         # END test repos with working tree
 
+    @with_rw_repo('HEAD')
+    def test_max_chunk_size(self, repo):
+        class TestOutputStream(object):
+            def __init__(self, max_chunk_size):
+                self.max_chunk_size = max_chunk_size
+
+            def write(self, b):
+                assert_true(len(b) <= self.max_chunk_size)
+
+        for chunk_size in [16, 128, 1024]:
+            repo.git.status(output_stream=TestOutputStream(chunk_size), max_chunk_size=chunk_size)
+
+        repo.git.log(n=100, output_stream=TestOutputStream(io.DEFAULT_BUFFER_SIZE), max_chunk_size=None)
+        repo.git.log(n=100, output_stream=TestOutputStream(io.DEFAULT_BUFFER_SIZE))
+
     def test_init(self):
         prev_cwd = os.getcwd()
         os.chdir(tempfile.gettempdir())
@@ -242,7 +258,7 @@ def test_init(self):
                     # when relative paths are used, the clone may actually be inside
                     # of the parent directory
                     pass
-                # END exception handling
+                    # END exception handling
 
             # END for each path
 
@@ -259,7 +275,7 @@ def test_init(self):
             except OSError:
                 pass
             os.chdir(prev_cwd)
-        # END restore previous state
+            # END restore previous state
 
     def test_bare_property(self):
         self.rorepo.bare
@@ -296,8 +312,8 @@ def test_is_dirty(self):
             for working_tree in (0, 1):
                 for untracked_files in (0, 1):
                     assert self.rorepo.is_dirty(index, working_tree, untracked_files) in (True, False)
-                # END untracked files
-            # END working tree
+                    # END untracked files
+                    # END working tree
         # END index
         orig_val = self.rorepo._bare
         self.rorepo._bare = True
@@ -456,7 +472,7 @@ def test_untracked_files(self, rwrepo):
                 untracked_files = [win_encode(f) for f in untracked_files]
             repo_add(untracked_files)
             self.assertEqual(len(rwrepo.untracked_files), (num_recently_untracked - len(files)))
-        # end for each run
+            # end for each run
 
     def test_config_reader(self):
         reader = self.rorepo.config_reader()                # all config files
@@ -514,7 +530,7 @@ def test_tilde_and_env_vars_in_repo_path(self, rw_dir):
             if ph:
                 os.environ['HOME'] = ph
                 del os.environ['FOO']
-        # end assure HOME gets reset to what it was
+                # end assure HOME gets reset to what it was
 
     def test_git_cmd(self):
         # test CatFileContentStream, just to be very sure we have no fencepost errors
@@ -681,7 +697,7 @@ def test_rev_parse(self):
                     print("failed on %s" % path_section)
                     # is fine, in case we have something like 112, which belongs to remotes/rname/merge-requests/112
                     pass
-                # END exception handling
+                    # END exception handling
             # END for each token
             if ref_no == 3 - 1:
                 break
@@ -827,8 +843,8 @@ def last_commit(repo, rev, path):
             for repo_type in (GitCmdObjectDB, GitDB):
                 repo = Repo(self.rorepo.working_tree_dir, odbt=repo_type)
                 last_commit(repo, 'master', 'git/test/test_base.py')
-            # end for each repository type
-        # end for each iteration
+                # end for each repository type
+                # end for each iteration
 
     def test_remote_method(self):
         self.failUnlessRaises(ValueError, self.rorepo.remote, 'foo-blue')