Fetch only required remotes. Saves disk space and bandwidth by default.

README.rst

@@ -42,6 +42,21 @@ Create a ``repos.yaml`` file:
  - acsone 80_connector_flow_ir_cron_able-lmi
  - acsone 8.0_connector_flow_improve_eval_config
  target: acsone aggregated_branch_name
+ fetch_all:
+ - oca
+
+If any of your merges refer to a specific commit, you will probably need to
+fetch all remotes from the corresponding remote or [use any other strategy to
+get that fetch working](http://stackoverflow.com/a/30701724/1468388), but we
+recommend to simply add this like in the example above:
+
+ fetch_all:
+ - oca
+ - other-remote
+
+You can specify that you want to fetch all references from all remotes you have defined with:
+
+ fetch_all: true
 
 Aggregate you repositories at any time:
 
@@ -101,6 +116,7 @@ Contributors
 ------------
 
  * Cyril Gaudin (camptocamp)
+ * Jairo Llopis <[email protected]>
 
 Maintainer
 ----------

git_aggregator/config.py

@@ -72,6 +72,12 @@ def get_repos(config):
  else:
  raise ConfigException(
  '%s: merges is not defined.' % directory)
+ # Only fetch required remotes by default
+ repo_dict["fetch_all"] = repo_data.get("fetch_all", False)
+ if isinstance(repo_dict["fetch_all"], string_types):
+ repo_dict["fetch_all"] = frozenset((repo_dict["fetch_all"],))
+ elif isinstance(repo_dict["fetch_all"], list):
+ repo_dict["fetch_all"] = frozenset(repo_dict["fetch_all"])
  if 'target' not in repo_data:
  raise ConfigException('%s: No target defined.' % directory)
  parts = (repo_data.get('target') or "") .split(' ')

git_aggregator/repo.py

@@ -33,7 +33,7 @@ class Repo(object):
 
  _git_version = None
 
- def __init__(self, cwd, remotes, merges, target,
+ def __init__(self, cwd, remotes, merges, target, fetch_all=False,
  shell_command_after=None):
  """Initialize a git repository aggregator
 
@@ -43,11 +43,19 @@ def __init__(self, cwd, remotes, merges, target,
  :param: merges list of merge to apply to build the aggregated
  repository. A merge is a dict {'remote': '', 'ref': ''}
  :param target:
- :patam shell_command_after: an optional list of shell command to
+ :param fetch_all:
+ Can be an iterable (recommended: ``frozenset``) that yields names
+ of remotes where all refs should be fetched, or ``True`` to do it
+ for every configured remote.
+ :param shell_command_after: an optional list of shell command to
  execute after the aggregation
  """
  self.cwd = cwd
  self.remotes = remotes
+ if fetch_all is True:
+ self.fetch_all = frozenset(r["name"] for r in remotes)
+ else:
+ self.fetch_all = fetch_all or frozenset()
  self.merges = merges
  self.target = target
  self.shell_command_after = shell_command_after or []
@@ -160,7 +168,7 @@ def aggregate(self):
  self._switch_to_branch(self.target['branch'])
  for r in self.remotes:
  self._set_remote(**r)
- self.fetch_all()
+ self.fetch()
  merges = self.merges
  if not is_new:
  # reset to the first merge
@@ -176,9 +184,14 @@ def init_repository(self, target_dir):
  logger.info('Init empty git repository in %s', target_dir)
  self.log_call(['git', 'init', target_dir])
 
- def fetch_all(self):
- logger.info('Fetching all remotes')
- self.log_call(['git', 'fetch', '--all'], cwd=self.cwd)
+ def fetch(self):
+ basecmd = ("git", "fetch")
+ logger.info("Fetching required remotes")
+ for merge in self.merges:
+ cmd = basecmd + (merge["remote"],)
+ if merge["remote"] not in self.fetch_all:
+ cmd += (merge["ref"],)
+ self.log_call(cmd, cwd=self.cwd)
 
  def push(self):
  remote = self.target['remote']

git_aggregator/utils.py

@@ -25,4 +25,5 @@ def __exit__(self, *exc_args):
  os.chdir(self.wd)
  self.active = False
 
+
 working_directory_keeper = WorkingDirectoryKeeper()

tests/test_config.py

@@ -5,6 +5,7 @@
 import tempfile
 import unittest
 import kaptan
+from textwrap import dedent
 
 from git_aggregator import config
 from git_aggregator.exception import ConfigException
@@ -39,6 +40,7 @@ def test_load(self):
  self.assertDictEqual(
  repos[0],
  {'cwd': '/product_attribute',
+ 'fetch_all': False,
  'merges': [{'ref': '8.0', 'remote': 'oca'},
  {'ref': 'refs/pull/105/head', 'remote': 'oca'},
  {'ref': 'refs/pull/106/head', 'remote': 'oca'}],
@@ -249,3 +251,46 @@ def test_import_config(self):
  finally:
  if os.path.exists(config_path):
  os.remove(config_path)
+
+ def test_fetch_all_string(self):
+ config_yaml = """
+ ./test:
+ remotes:
+ oca: https:/test/test.git
+ merges:
+ - oca 8.0
+ target: oca aggregated_branch_name
+ fetch_all: oca
+ """
+ config_yaml = dedent(config_yaml)
+ repos = config.get_repos(self._parse_config(config_yaml))
+ self.assertSetEqual(repos[0]["fetch_all"], {"oca"})
+
+ def test_fetch_all_list(self):
+ config_yaml = """
+ ./test:
+ remotes:
+ oca: https:/test/test.git
+ merges:
+ - oca 8.0
+ target: oca aggregated_branch_name
+ fetch_all:
+ - oca
+ """
+ config_yaml = dedent(config_yaml)
+ repos = config.get_repos(self._parse_config(config_yaml))
+ self.assertSetEqual(repos[0]["fetch_all"], {"oca"})
+
+ def test_fetch_all_true(self):
+ config_yaml = """
+ ./test:
+ remotes:
+ oca: https:/test/test.git
+ merges:
+ - oca 8.0
+ target: oca aggregated_branch_name
+ fetch_all: yes
+ """
+ config_yaml = dedent(config_yaml)
+ repos = config.get_repos(self._parse_config(config_yaml))
+ self.assertIs(repos[0]["fetch_all"], True)

tests/test_repo.py

@@ -136,7 +136,7 @@ def test_simple_merge(self):
  'remote': 'r1',
  'branch': 'agg'
  }
- repo = Repo(self.cwd, remotes, merges, target)
+ repo = Repo(self.cwd, remotes, merges, target, True)
  repo.aggregate()
  last_rev = git_get_last_rev(self.cwd)
  self.assertEqual(last_rev, self.commit_3_sha)
@@ -171,7 +171,7 @@ def test_update_aggregate(self):
  'remote': 'r1',
  'branch': 'agg'
  }
- repo = Repo(self.cwd, remotes, merges, target)
+ repo = Repo(self.cwd, remotes, merges, target, True)
  repo.aggregate()
  self.assertTrue(os.path.isfile(os.path.join(self.cwd, 'tracked')))
  self.assertTrue(os.path.isfile(os.path.join(self.cwd, 'tracked2')))
@@ -205,7 +205,7 @@ def test_update_aggregate_2(self):
  'remote': 'r1',
  'branch': 'agg'
  }
- repo = Repo(self.cwd, remotes, merges, target)
+ repo = Repo(self.cwd, remotes, merges, target, True)
  repo.aggregate()
  self.assertTrue(os.path.isfile(os.path.join(self.cwd, 'tracked')))
  self.assertTrue(os.path.isfile(os.path.join(self.cwd, 'tracked2')))