Commit 7478091e808f30c08c15e5c7a1ad8b584eaec04b - ros-rosdistro

Merge tag 'upstream/0.4.7' Upstream version 0.4.7 Jochen Sprickerhof 7 years ago

9 changed file(s) with 274 addition(s) and 30 deletion(s). Raw diff Collapse all Expand all

+87

-0

scripts/rosdistro_freeze_source less more

	0	#!/usr/bin/env python
	1
	2	# Software License Agreement (BSD License)
	3	#
	4	# Copyright (c) 2016, Clearpath Robotics
	5	# All rights reserved.
	6	#
	7	# Redistribution and use in source and binary forms, with or without
	8	# modification, are permitted provided that the following conditions
	9	# are met:
	10	#
	11	# * Redistributions of source code must retain the above copyright
	12	# notice, this list of conditions and the following disclaimer.
	13	# * Redistributions in binary form must reproduce the above
	14	# copyright notice, this list of conditions and the following
	15	# disclaimer in the documentation and/or other materials provided
	16	# with the distribution.
	17	# * Neither the name of Open Source Robotics Foundation, Inc. nor
	18	# the names of its contributors may be used to endorse or promote
	19	# products derived from this software without specific prior
	20	# written permission.
	21	#
	22	# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	23	# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	24	# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
	25	# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
	26	# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
	27	# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	28	# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	29	# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
	30	# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	31	# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
	32	# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	33	# POSSIBILITY OF SUCH DAMAGE.
	34
	35	import argparse
	36	import os.path
	37	import sys
	38	import yaml
	39
	40	from rosdistro import get_distribution
	41	from rosdistro.index import Index
	42	from rosdistro.freeze_source import freeze_distribution_sources, CONCURRENT_DEFAULT
	43	from rosdistro.writer import yaml_from_distribution_file
	44
	45
	46	def parse_args(args=sys.argv[1:]):
	47	parser = argparse.ArgumentParser(
	48	description='''
	49	Freeze a rosdistro\'s source branch versions to hashes or tags. If neither --release-version
	50	nor --release-tag are specified, the hashes of the current devel branches are used.
	51	''')
	52	parser.add_argument('index', type=argparse.FileType(),
	53	help='Path to a local index.yaml file.')
	54	parser.add_argument('dist_names', nargs='*', help='The names of the distributions (default: all)')
	55	parser.add_argument('-j', '--jobs', type=int, default=CONCURRENT_DEFAULT,
	56	help='How many worker threads to use.')
	57	parser.add_argument('-q', '--quiet', action="store_true",
	58	help='Suppress updating status bar (for script/CI usage).')
	59
	60	mode = parser.add_mutually_exclusive_group()
	61	mode.add_argument('--release-version', action="store_true",
	62	help='Freeze to the hash of current release tag.')
	63	mode.add_argument('--release-tag', action="store_true",
	64	help='Freeze to name of current release tag.')
	65	return parser.parse_args(args)
	66
	67
	68	def main():
	69	args = parse_args()
	70	index = Index(yaml.safe_load(args.index),
	71	'file://%s' % os.path.dirname(os.path.abspath(args.index.name)))
	72
	73	if not args.dist_names:
	74	args.dist_names = sorted(index.distributions.keys())
	75
	76	for dist_name in args.dist_names:
	77	dist = get_distribution(index, dist_name)
	78	freeze_distribution_sources(dist, release_version=args.release_version, release_tag=args.release_tag,
	79	concurrent_ops=args.jobs, quiet=args.quiet)
	80	dist_file_local = index.distributions[dist_name]['distribution'].split('://')[1]
	81	with open(dist_file_local, 'w') as f:
	82	f.write(yaml_from_distribution_file(dist))
	83
	84
	85	if __name__ == '__main__':
	86	main()

-0

setup.py less more

18	18	scripts=[
19	19	# 'scripts/rosdistro',
20	20	'scripts/rosdistro_build_cache',
	21	'scripts/rosdistro_freeze_source',
21	22	# 'scripts/rosdistro_convert',
22	23	# 'scripts/rosdistro_generate_cache',
23	24	'scripts/rosdistro_migrate_to_rep_141',

-3

src/rosdistro/__init__.py less more

163	163
164	164	def get_distribution_cache(index, dist_name):
165	165	if dist_name not in index.distributions.keys():
166		raise RuntimeError("Unknown distribution: '{0}'. Valid distribution names are: {1}".format(dist_name, ', '.join(["'%s'" % d for d in index.distributions.keys()])))
	166	raise RuntimeError("Unknown distribution: '{0}'. Valid distribution names are: {1}".format(dist_name, ', '.join(sorted(index.distributions.keys()))))
167	167	dist = index.distributions[dist_name]
168	168	if 'distribution_cache' not in dist.keys():
169	169	raise RuntimeError("Distribution has no cache: '{0}'".format(dist_name))

222	222	def get_release_cache(index, dist_name):
223	223	print('# rosdistro.get_release_cache() has been deprecated in favor of the new function rosdistro.get_distribution_cache() - please check that you have the latest versions of the Python tools (e.g. on Ubuntu/Debian use: sudo apt-get update && sudo apt-get install --only-upgrade python-bloom python-rosdep python-rosinstall python-rosinstall-generator)', file=sys.stderr)
224	224	if dist_name not in index.distributions.keys():
225		raise RuntimeError("Unknown release: '{0}'. Valid release names are: {1}".format(dist_name, ', '.join(["'%s'" % d for d in index.distributions.keys()])))
	225	raise RuntimeError("Unknown release: '{0}'. Valid release names are: {1}".format(dist_name, ', '.join(sorted(index.distributions.keys()))))
226	226	dist = index.distributions[dist_name]
227	227	if 'distribution_cache' not in dist.keys():
228	228	raise RuntimeError("Release has no cache: '{0}'".format(dist_name))

304	304
305	305	def _get_dist_file_data(index, dist_name, type_):
306	306	if dist_name not in index.distributions.keys():
307		raise RuntimeError("Unknown release: '{0}'. Valid release names are: {1}".format(dist_name, ', '.join(["'%s'" % d for d in index.distributions.keys()])))
	307	raise RuntimeError("Unknown release: '{0}'. Valid release names are: {1}".format(dist_name, ', '.join(sorted(index.distributions.keys()))))
308	308	dist = index.distributions[dist_name]
309	309	if type_ not in dist.keys():
310	310	raise RuntimeError('unknown release type "%s"' % type_)

-1

src/rosdistro/_version.py less more

0		__version__ = '0.4.4'
	0	__version__ = '0.4.7'

+15

-0

src/rosdistro/distribution_cache.py less more

77	77	# remove packages which are not in the old distribution file
78	78	self._remove_obsolete_entries()
79	79
	80	# determine differences in doc and source entries
	81	if len(distribution_file_data) == len(self._distribution_file_data):
	82	for old_data, new_data in zip(self._distribution_file_data, distribution_file_data):
	83	for repo_name in sorted(new_data['repositories'].keys()):
	84	repo = new_data['repositories'][repo_name]
	85	for section in ['doc', 'source']:
	86	if section not in repo:
	87	continue
	88	if repo_name in old_data['repositories'] and \
	89	section in old_data['repositories'][repo_name] and \
	90	old_data['repositories'][repo_name][section] == repo[section]:
	91	continue
	92	# section is either different or does't exist before
	93	print(" - updated '%s' entry for repository '%s'" % (section, repo_name))
	94
80	95	self._distribution_file_data = distribution_file_data
81	96	dist_file = create_distribution_file(self.distribution_file.name, self._distribution_file_data)
82	97

+14

-4

src/rosdistro/distribution_cache_generator.py less more

83	83	sys.stdout.write('.')
84	84	sys.stdout.flush()
85	85	# check that package.xml is fetchable
	86	old_package_xml = None
	87	if cache and pkg_name in cache.release_package_xmls:
	88	old_package_xml = cache.release_package_xmls[pkg_name]
86	89	package_xml = dist.get_release_package_xml(pkg_name)
87	90	if not package_xml:
88	91	errors.append('%s: missing package.xml file for package "%s"' % (dist_name, pkg_name))

96	99	# check that version numbers match (at least without deb inc)
97	100	if not re.match('^%s-[\dA-z~\+\.]+$' % re.escape(pkg.version), repo.version):
98	101	errors.append('%s: different version in package.xml (%s) for package "%s" than for the repository (%s) (after removing the debian increment)' % (dist_name, pkg.version, pkg_name, repo.version))
	102
	103	if package_xml != old_package_xml:
	104	print(" - updated manifest of package '%s' to version '%s'" % (pkg_name, pkg.version))
99	105
100	106	if not debug:
101	107	print('')

129	135	print('- trying to fetch cache')
130	136	# get distribution cache
131	137	cache = get_distribution_cache(index, dist_name)
132		# get current distribution file
133		rel_file_data = _get_dist_file_data(index, dist_name, 'distribution')
134		# update cache with current distribution file
135		cache.update_distribution(rel_file_data)
136	138	except Exception as e:
137	139	print('- failed to fetch old cache: %s' % e)
	140
138	141	if cache:
139	142	print('- update cache')
	143	# get current distribution file
	144	rel_file_data = _get_dist_file_data(index, dist_name, 'distribution')
	145	# since format 2 of the index file might contain a single value rather then a list
	146	if not isinstance(rel_file_data, list):
	147	rel_file_data = [rel_file_data]
	148	# update cache with current distribution file
	149	cache.update_distribution(rel_file_data)
140	150	else:
141	151	print('- build cache from scratch')
142	152	# get empty cache with distribution file

+108

-0

src/rosdistro/freeze_source.py less more

	0	#!/usr/bin/env python
	1
	2	# Software License Agreement (BSD License)
	3	#
	4	# Copyright (c) 2016, Clearpath Robotics
	5	# All rights reserved.
	6	#
	7	# Redistribution and use in source and binary forms, with or without
	8	# modification, are permitted provided that the following conditions
	9	# are met:
	10	#
	11	# * Redistributions of source code must retain the above copyright
	12	# notice, this list of conditions and the following disclaimer.
	13	# * Redistributions in binary form must reproduce the above
	14	# copyright notice, this list of conditions and the following
	15	# disclaimer in the documentation and/or other materials provided
	16	# with the distribution.
	17	# * Neither the name of Open Source Robotics Foundation, Inc. nor
	18	# the names of its contributors may be used to endorse or promote
	19	# products derived from this software without specific prior
	20	# written permission.
	21	#
	22	# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	23	# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	24	# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
	25	# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
	26	# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
	27	# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	28	# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	29	# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
	30	# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	31	# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
	32	# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	33	# POSSIBILITY OF SUCH DAMAGE.
	34
	35	from __future__ import print_function
	36
	37	import sys
	38	import subprocess
	39	import threading
	40	import time
	41	import yaml
	42
	43	try:
	44	import queue
	45	except ImportError:
	46	import Queue as queue
	47
	48	CONCURRENT_DEFAULT = 16
	49
	50
	51	def freeze_distribution_sources(dist, release_version=False, release_tag=False,
	52	concurrent_ops=CONCURRENT_DEFAULT, quiet=False):
	53	# Populate this queue with tuples of repositories instances to be updated,
	54	# so that this work can be spread across multiple threads.
	55	work_queue = queue.Queue()
	56	for repo_name, repo in dist.repositories.iteritems():
	57	# Only manipulate distribution entries with a source repo listed.
	58	if repo.source_repository:
	59	# Decide which git ref string we'll be using as the replacement match.
	60	if repo.release_repository and (release_version or release_tag):
	61	version = repo.release_repository.version.split('-')[0]
	62	else:
	63	version = repo.source_repository.version
	64	work_queue.put((repo.source_repository, version, release_tag))
	65
	66	total_items = work_queue.qsize()
	67
	68	for i in range(concurrent_ops):
	69	threading.Thread(target=_worker, args=[work_queue]).start()
	70
	71	# Wait until the threads have done all the work and exited.
	72	while not work_queue.empty():
	73	time.sleep(0.1)
	74	if not quiet:
	75	sys.stdout.write("Updating source repo versions (%d/%d) \r" %
	76	(total_items - work_queue.qsize(), total_items))
	77	sys.stdout.flush()
	78	work_queue.join()
	79
	80	# Clear past the updating line.
	81	if not quiet:
	82	print("")
	83
	84
	85	def _worker(work_queue):
	86	while True:
	87	try:
	88	source_repo, freeze_version, freeze_to_tag = work_queue.get(block=False)
	89	cmd = ['git', 'ls-remote', source_repo.url]
	90	ls_remote_lines = subprocess.check_output(cmd).splitlines()
	91	for line in ls_remote_lines:
	92	hash, ref = line.split('\t', 1)
	93	if ref.endswith(freeze_version):
	94	if freeze_to_tag and ref.startswith('refs/tags/'):
	95	source_repo.version = ref.split('refs/tags/')[1]
	96	else:
	97	source_repo.version = hash
	98	break
	99
	100	work_queue.task_done()
	101
	102	except subprocess.CalledProcessError:
	103	print("Non-zero return code for: %s" % ' '.join(cmd), file=sys.stderr)
	104	work_queue.task_done()
	105
	106	except queue.Empty:
	107	break

+37

-18

src/rosdistro/manifest_provider/git.py less more

30	30	# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31	31	# POSSIBILITY OF SUCH DAMAGE.
32	32
	33	from distutils.version import LooseVersion
33	34	import os
34	35	import shutil
35	36	import subprocess

52	53	raise RuntimeError('Unable to fetch package.xml: %s' % e)
53	54
54	55
	56	def _git_version_gte(version):
	57	global _git_client_version
	58	if not _git_client_version:
	59	cmd = [_git_client_executable, '--version']
	60	result = _run_command(cmd)
	61	_git_client_version = result['output'].split()[-1]
	62	return LooseVersion(_git_client_version) >= LooseVersion(version)
	63
	64
55	65	def _get_package_xml(url, tag):
56	66	base = tempfile.mkdtemp('rosdistro')
57	67	try:
58		# git 1.7.9 does not support cloning a tag directly, so doing it in two steps
59	68	assert _git_client_executable is not None, "'git' not found"
60		cmd = [_git_client_executable, 'clone', url, base]
61		result = _run_command(cmd, base)
62		if result['returncode'] != 0:
63		raise RuntimeError('Could not clone repository "%s"' % url)
	69	if _git_version_gte('1.8.0'):
	70	# Directly clone the required tag with least amount of additional history. This behaviour
	71	# has been available since git 1.8.0:
	72	# https://git.kernel.org/cgit/git/git.git/tree/Documentation/git-clone.txt?h=v1.8.0#n158
	73	cmd = [_git_client_executable, 'clone', url, base, '--depth', '1', '--branch', tag]
	74	result = _run_command(cmd, base)
	75	if result['returncode'] != 0:
	76	raise RuntimeError('Could not clone repository "%s" at tag "%s"' % (url, tag))
	77	else:
	78	# Old git doesn't support cloning a tag directly, so check it out after a full clone.
	79	cmd = [_git_client_executable, 'clone', url, base]
	80	result = _run_command(cmd, base)
	81	if result['returncode'] != 0:
	82	raise RuntimeError('Could not clone repository "%s"' % url)
64	83
65		cmd = [_git_client_executable, 'tag', '-l']
66		result = _run_command(cmd, base)
67		if result['returncode'] != 0:
68		raise RuntimeError('Could not get tags of repository "%s"' % url)
	84	cmd = [_git_client_executable, 'tag', '-l']
	85	result = _run_command(cmd, base)
	86	if result['returncode'] != 0:
	87	raise RuntimeError('Could not get tags of repository "%s"' % url)
69	88
70		if tag not in result['output'].splitlines():
71		raise RuntimeError('Specified tag "%s" is not a git tag of repository "%s"' % (tag, url))
	89	if tag not in result['output'].splitlines():
	90	raise RuntimeError('Specified tag "%s" is not a git tag of repository "%s"' % (tag, url))
72	91
73		cmd = [_git_client_executable, 'checkout', tag]
74		result = _run_command(cmd, base)
75		if result['returncode'] != 0:
76		raise RuntimeError('Could not checkout tag "%s" of repository "%s"' % (tag, url))
	92	cmd = [_git_client_executable, 'checkout', tag]
	93	result = _run_command(cmd, base)
	94	if result['returncode'] != 0:
	95	raise RuntimeError('Could not checkout tag "%s" of repository "%s"' % (tag, url))
77	96
78	97	filename = os.path.join(base, 'package.xml')
79	98	if not os.path.exists(filename):
80	99	raise RuntimeError('Could not find package.xml in repository "%s"' % url)
81	100	with open(filename, 'r') as f:
82		package_xml = f.read()
83		return package_xml
	101	return f.read()
84	102	finally:
85	103	shutil.rmtree(base)
86	104

103	121	return False
104	122
105	123
106		def _run_command(cmd, cwd, env=None):
	124	def _run_command(cmd, cwd=None, env=None):
107	125	result = {'cmd': ' '.join(cmd), 'cwd': cwd}
108	126	try:
109	127	proc = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env)

126	144	return None
127	145
128	146	_git_client_executable = _find_executable('git')
	147	_git_client_version = None

-4

src/rosdistro/release_cache_generator.py less more

128	128	print('- trying to fetch cache')
129	129	# get release cache
130	130	cache = get_release_cache(index, dist_name)
131		# get current release file
132		rel_file_data = _get_dist_file_data(index, dist_name, 'release')
133		# update cache with current release file
134		cache.update_distribution(rel_file_data)
135	131	except:
136	132	print('- failed to fetch old cache')
	133
137	134	if cache:
138	135	print('- update cache')
	136	# get current release file
	137	rel_file_data = _get_dist_file_data(index, dist_name, 'release')
	138	# since format 2 of the index file might contain a single value rather then a list
	139	if not isinstance(rel_file_data, list):
	140	rel_file_data = [rel_file_data]
	141	# update cache with current release file
	142	cache.update_distribution(rel_file_data)
139	143	else:
140	144	print('- build cache from scratch')
141	145	# get empty cache with distribution file