Commit 4de751a173d12a2be4d8ba450f2f92a11c51511c - ros-bloom

+0

-2

.travis.yml less more

12	12	- cd -
13	13	- sudo `which rosdep` init
14	14	- rosdep update
15		- git config --global user.email "test@example.com"
16		- git config --global user.name "Test User"
17	15	# command to run tests
18	16	script:
19	17	- BLOOM_VERBOSE=1 python setup.py nosetests -s --tests test

+13

-0

CHANGELOG.rst less more

	0	0.8.0 (2019-04-12 13:45:00 -0800)
	1	---------------------------------
	2	- Start release increment at 1. `#528 <https://github.com/ros-infrastructure/bloom/pull/528>`_
	3	- Evaluate conditions in package.xml before resolving dependencies. `#519 <https://github.com/ros-infrastructure/bloom/pull/519>`_
	4	- Update to prevent overwriting template files that exist in source. `#516 <https://github.com/ros-infrastructure/bloom/pull/516>`_
	5	- Update debian templates to add trailing newline. `#523 <https://github.com/ros-infrastructure/bloom/pull/523>`_
	6	- Fix str/bytes issue in Python 3 auth. `#522 <https://github.com/ros-infrastructure/bloom/pull/522>`_
	7	- Use distribution type from index v4 to set ROS 2-specific behavior. `#502 <https://github.com/ros-infrastructure/bloom/pull/502>`_
	8	- Fix tests to allow them to run outside of a git context. `#515 <https://github.com/ros-infrastructure/bloom/pull/515>`_
	9	- Fix tests to allow empty git environment. `#514 <https://github.com/ros-infrastructure/bloom/pull/514>`_
	10	- Invoke scripts using the current python executable. `#513 <https://github.com/ros-infrastructure/bloom/pull/513>`_
	11	- Drop support for older distributions. (Retroactive to 0.6.8) `#512 <https://github.com/ros-infrastructure/bloom/pull/512>`_
	12
0	13	0.7.2 (2019-01-26 07:45:00 -0800)
1	14	---------------------------------
2	15	- Updated a test to support mixed rosdistro index. `#510 <https://github.com/ros-infrastructure/bloom/pull/510>`_

+1

-1

bloom/commands/git/release.py less more

226	226	settings['ros_distro'] = track_dict['ros_distro']
227	227	# Release increment
228	228	if 'last_version' in track_dict and track_dict['last_version'] != version:
229		next_release_inc = str(0)
	229	next_release_inc = str(1)
230	230	else:
231	231	next_release_inc = str(int(track_dict['release_inc']) + 1)
232	232	settings['release_inc'] = release_inc_override or next_release_inc

+19

-223

bloom/commands/release.py less more

35	35
36	36	import argparse
37	37	import atexit
38		import base64
39	38	import datetime
40	39	import difflib
41		import getpass
42		import json
43	40	import os
44	41	import pkg_resources
45	42	import platform

75	72	from bloom.git import inbranch
76	73	from bloom.git import ls_tree
77	74
78		from bloom.github import auth_header_from_basic_auth
79		from bloom.github import auth_header_from_oauth_token
80		from bloom.github import Github
81	75	from bloom.github import GithubException
82		from bloom.github import GitHubAuthException
	76	from bloom.github import get_gh_info
	77	from bloom.github import get_github_interface
83	78
84	79	from bloom.logging import debug
85	80	from bloom.logging import error

92	87
93	88	from bloom.packages import get_package_data
94	89	from bloom.packages import get_ignored_packages
	90
	91	from bloom.rosdistro_api import get_distribution_file
	92	from bloom.rosdistro_api import get_index
	93	from bloom.rosdistro_api import get_most_recent
	94	from bloom.rosdistro_api import get_rosdistro_index_commit
	95	from bloom.rosdistro_api import get_rosdistro_index_original_branch
95	96
96	97	from bloom.summary import commit_summary
97	98	from bloom.summary import get_summary_file

117	118	import vcstools.__version__
118	119	from vcstools.vcs_abstraction import get_vcs_client
119	120
120		try:
121		import rosdistro
122		if parse_version(rosdistro.__version__) < parse_version('0.7.0'):
123		error("rosdistro version 0.7.0 or greater is required, found '{0}' from '{1}'."
124		.format(rosdistro.__version__, os.path.dirname(rosdistro.__file__)),
125		exit=True)
126		except ImportError:
127		debug(traceback.format_exc())
128		error("rosdistro was not detected, please install it.", file=sys.stderr,
129		exit=True)
	121	from rosdistro import DistributionFile
	122	from rosdistro import get_distribution_files
	123	from rosdistro import get_index_url
130	124	from rosdistro.writer import yaml_from_distribution_file
131	125
132	126	try:

161	155	if os.path.exists(repo_path):
162	156	shutil.rmtree(repo_path)
163	157
164		_rosdistro_index = None
165		_rosdistro_distribution_files = {}
166		_rosdistro_index_commit = None
167		_rosdistro_index_original_branch = None
168
169
170		def get_index_url():
171		global _rosdistro_index_commit, _rosdistro_index_original_branch
172		index_url = rosdistro.get_index_url()
173		pr = urlparse(index_url)
174		if pr.netloc in ['raw.github.com', 'raw.githubusercontent.com']:
175		# Try to determine what the commit hash was
176		tokens = [x for x in pr.path.split('/') if x]
177		if len(tokens) <= 3:
178		debug("Failed to get commit for rosdistro index file: index url")
179		debug(tokens)
180		return index_url
181		owner = tokens[0]
182		repo = tokens[1]
183		branch = tokens[2]
184		gh = get_github_interface(quiet=True)
185		if gh is None:
186		# Failed to get it with auth, try without auth (may fail)
187		gh = Github(username=None, auth=None)
188		try:
189		data = gh.get_branch(owner, repo, branch)
190		except GithubException:
191		debug(traceback.format_exc())
192		debug("Failed to get commit for rosdistro index file: api")
193		return index_url
194		_rosdistro_index_commit = data.get('commit', {}).get('sha', None)
195		if _rosdistro_index_commit is not None:
196		info("ROS Distro index file associate with commit '{0}'"
197		.format(_rosdistro_index_commit))
198		# Also mutate the index_url to use the commit (rather than the moving branch name)
199		base_info = get_gh_info(index_url)
200		base_branch = base_info['branch']
201		rosdistro_index_commit = _rosdistro_index_commit # Copy global into local for substitution
202		middle = "{org}/{repo}".format(**base_info)
203		index_url = index_url.replace("{pr.netloc}/{middle}/{base_branch}/".format(**locals()),
204		"{pr.netloc}/{middle}/{rosdistro_index_commit}/".format(**locals()))
205		info("New ROS Distro index url: '{0}'".format(index_url))
206		_rosdistro_index_original_branch = base_branch
207		else:
208		debug("Failed to get commit for rosdistro index file: json")
209		return index_url
210
211
212		def get_index():
213		global _rosdistro_index
214		if _rosdistro_index is None:
215		_rosdistro_index = rosdistro.get_index(get_index_url())
216		if _rosdistro_index.version == 1:
217		error("This version of bloom does not support rosdistro version "
218		"'{0}', please use an older version of bloom."
219		.format(_rosdistro_index.version), exit=True)
220		if _rosdistro_index.version > 4:
221		error("This version of bloom does not support rosdistro version "
222		"'{0}', please update bloom.".format(_rosdistro_index.version), exit=True)
223		return _rosdistro_index
224
225
226		def list_distributions():
227		return sorted(get_index().distributions.keys())
228
229
230		def get_distribution_type(distro):
231		return get_index().distributions[distro].get('distribution_type')
232
233
234		def get_most_recent(thing_name, repository, reference_distro):
235		reference_distro_type = get_distribution_type(reference_distro)
236		distros_with_entry = {}
237		get_things = {
238		'release': lambda r: None if r.release_repository is None else r.release_repository,
239		'doc': lambda r: None if r.doc_repository is None else r.doc_repository,
240		'source': lambda r: None if r.source_repository is None else r.source_repository,
241		}
242		get_thing = get_things[thing_name]
243		for distro in list_distributions():
244		# skip distros with a different type if the information is available
245		if reference_distro_type is not None:
246		if get_distribution_type(distro) != reference_distro_type:
247		continue
248		distro_file = get_distribution_file(distro)
249		if repository in distro_file.repositories:
250		thing = get_thing(distro_file.repositories[repository])
251		if thing is not None:
252		distros_with_entry[distro] = thing
253		# Choose the alphabetical last distro which contained a release of this repository
254		default_distro = (sorted(distros_with_entry.keys()) or [None])[-1]
255		default_thing = distros_with_entry.get(default_distro, None)
256		return default_distro, default_thing
257
258
259		def get_distribution_file(distro):
260		global _rosdistro_distribution_files
261		if distro not in _rosdistro_distribution_files:
262		# REP 143, get list of distribution files and take the last one
263		files = rosdistro.get_distribution_files(get_index(), distro)
264		if not files:
265		error("No distribution files listed for distribution '{0}'."
266		.format(distro), exit=True)
267		_rosdistro_distribution_files[distro] = files[-1]
268		return _rosdistro_distribution_files[distro]
269	158
270	159	_rosdistro_distribution_file_urls = {}
271	160

425	314
426	315	def get_relative_distribution_file_path(distro):
427	316	distribution_file_url = urlparse(get_distribution_file_url(distro))
428		index_file_url = urlparse(rosdistro.get_index_url())
	317	index_file_url = urlparse(get_index_url())
429	318	return os.path.relpath(distribution_file_url.path,
430	319	os.path.commonprefix([index_file_url.path, distribution_file_url.path]))
431	320

626	515
627	516	# Do the diff
628	517	distro_file_name = get_relative_distribution_file_path(distro)
629		updated_distribution_file = rosdistro.DistributionFile(distro, distribution_dict)
	518	updated_distribution_file = DistributionFile(distro, distribution_dict)
630	519	distro_dump = yaml_from_distribution_file(updated_distribution_file)
631	520	distro_file_raw = load_url_to_file_handle(get_distribution_file_url(distro)).read().decode('utf-8')
632	521	if distro_file_raw != distro_dump:

678	567	return None
679	568
680	569
681		def get_gh_info(url):
682		o = urlparse(url)
683		if 'raw.github.com' not in o.netloc and 'raw.githubusercontent.com' not in o.netloc:
684		return None
685		url_paths = o.path.split('/')
686		if len(url_paths) < 5:
687		return None
688		return {'server': 'github.com',
689		'org': url_paths[1],
690		'repo': url_paths[2],
691		'branch': url_paths[3],
692		'path': '/'.join(url_paths[4:])}
693
694
695	570	def get_repo_info(distro_url):
696	571	gh_info = get_gh_info(distro_url)
697	572	if gh_info:
698	573	return gh_info
699
700
701		_gh = None
702
703
704		def get_github_interface(quiet=False):
705		def mfa_prompt(oauth_config_path, username):
706		"""Explain how to create a token for users with Multi-Factor Authentication configured."""
707		warning("Receiving 401 when trying to create an oauth token can be caused by the user "
708		"having two-factor authentication enabled.")
709		warning("If 2FA is enabled, the user will have to create an oauth token manually.")
710		warning("A token can be created at https://github.com/settings/tokens")
711		warning("The resulting token can be placed in the '{oauth_config_path}' file as such:"
712		.format(**locals()))
713		info("")
714		warning('{{"github_user": "{username}", "oauth_token": "TOKEN_GOES_HERE"}}'
715		.format(**locals()))
716		info("")
717
718		global _gh
719		if _gh is not None:
720		return _gh
721		# First check to see if the oauth token is stored
722		oauth_config_path = os.path.join(os.path.expanduser('~'), '.config', 'bloom')
723		config = {}
724		if os.path.exists(oauth_config_path):
725		with open(oauth_config_path, 'r') as f:
726		config = json.loads(f.read())
727		token = config.get('oauth_token', None)
728		username = config.get('github_user', None)
729		if token and username:
730		return Github(username, auth=auth_header_from_oauth_token(token), token=token)
731		if not os.path.isdir(os.path.dirname(oauth_config_path)):
732		os.makedirs(os.path.dirname(oauth_config_path))
733		if quiet:
734		return None
735		# Ok, now we have to ask for the user name and pass word
736		info("")
737		warning("Looks like bloom doesn't have an oauth token for you yet.")
738		warning("Therefore bloom will require your GitHub username and password just this once.")
739		warning("With your GitHub username and password bloom will create an oauth token on your behalf.")
740		warning("The token will be stored in `~/.config/bloom`.")
741		warning("You can delete the token from that file to have a new token generated.")
742		warning("Guard this token like a password, because it allows someone/something to act on your behalf.")
743		warning("If you need to unauthorize it, remove it from the 'Applications' menu in your GitHub account page.")
744		info("")
745		if not maybe_continue('y', "Would you like to create an OAuth token now"):
746		return None
747		token = None
748		while token is None:
749		try:
750		username = getpass.getuser()
751		username = safe_input("GitHub username [{0}]: ".format(username)) or username
752		password = getpass.getpass("GitHub password (never stored): ")
753		except (KeyboardInterrupt, EOFError):
754		return None
755		if not password:
756		error("No password was given, aborting.")
757		return None
758		gh = Github(username, auth=auth_header_from_basic_auth(username, password))
759		try:
760		token = gh.create_new_bloom_authorization(update_auth=True)
761		with open(oauth_config_path, 'w') as f:
762		config.update({'oauth_token': token, 'github_user': username})
763		f.write(json.dumps(config))
764		info("The token '{token}' was created and stored in the bloom config file: '{oauth_config_path}'"
765		.format(**locals()))
766		except GitHubAuthException as exc:
767		error("{0}".format(exc))
768		mfa_prompt(oauth_config_path, username)
769		except GithubException as exc:
770		error("{0}".format(exc))
771		info("")
772		if hasattr(exc, 'resp') and '{0}'.format(exc.resp.status) in ['401']:
773		mfa_prompt(oauth_config_path, username)
774		warning("This sometimes fails when the username or password are incorrect, try again?")
775		if not maybe_continue():
776		return None
777		_gh = gh
778		return gh
779	574
780	575
781	576	def get_changelog_summary(release_tag):

813	608
814	609
815	610	def open_pull_request(track, repository, distro, interactive, override_release_repository_url):
816		global _rosdistro_index_commit
817	611	# Get the diff
818	612	distribution_file = get_distribution_file(distro)
819	613	if repository in distribution_file.repositories and \

836	630	return
837	631
838	632	# If we did replace the branch in the url with a commit, restore that now
839		if _rosdistro_index_original_branch is not None:
840		base_info['branch'] = _rosdistro_index_original_branch
	633	rosdistro_index_original_branch = get_rosdistro_index_original_branch()
	634	if rosdistro_index_original_branch is not None:
	635	base_info['branch'] = rosdistro_index_original_branch
841	636
842	637	# Create content for PR
843	638	title = "{0}: {1} in '{2}' [bloom]".format(repository, version, base_info['path'])

946	741	_my_run('git checkout -b {new_branch}'.format(**locals()))
947	742	_my_run("git pull {rosdistro_url} {base_info[branch]}".format(**locals()),
948	743	"Pulling latest rosdistro branch")
949		if _rosdistro_index_commit is not None:
950		_my_run('git reset --hard {_rosdistro_index_commit}'.format(**globals()))
	744	rosdistro_index_commit = get_rosdistro_index_commit()
	745	if rosdistro_index_commit is not None:
	746	_my_run('git reset --hard {rosdistro_index_commit}'.format(**locals()))
951	747	with open('{0}'.format(base_info['path']), 'w') as f:
952	748	info(fmt("@{bf}@!==> @\|@!Writing new distribution file: ") + str(base_info['path']))
953	749	f.write(updated_distro_file_yaml)

+1

-1

bloom/config.py less more

189	189	'patches': PromptEntry('Patches Directory', spec=config_spec['patches']),
190	190	'ros_distro': PromptEntry('ROS Distro', default='indigo', spec=config_spec['ros_distro']),
191	191	'release_repo_url': PromptEntry('Release Repository Push URL', spec=config_spec['release_repo_url']),
192		'release_inc': -1,
	192	'release_inc': 0,
193	193	'actions': [
194	194	'bloom-export-upstream :{vcs_local_uri} :{vcs_type}'
195	195	' --tag :{release_tag} --display-uri :{vcs_uri}'

+17

-0

bloom/generators/common.py less more

39	39	from bloom.logging import error
40	40	from bloom.logging import info
41	41
	42	from bloom.rosdistro_api import get_distribution_type
	43
42	44	from bloom.util import code
43	45	from bloom.util import maybe_continue
44	46	from bloom.util import print_exc

114	116	default_os_installer)
115	117	assert inst_key in os_installers
116	118	return installer.resolve(rule), inst_key, default_os_installer
	119
	120
	121	def package_conditional_context(ros_distro):
	122	distribution_type = get_distribution_type(ros_distro)
	123	if distribution_type == 'ros1':
	124	ros_version = '1'
	125	elif distribution_type == 'ros2':
	126	ros_version = '2'
	127	else:
	128	error("Bloom cannot cope with distribution_type '{0}'".format(
	129	distribution_type), exit=True)
	130	return {
	131	'ROS_VERSION': ros_version,
	132	'ROS_DISTRO': ros_distro,
	133	}
117	134
118	135
119	136	def resolve_rosdep_key(

+40

-21

bloom/generators/debian/generator.py less more

57	57
58	58	from bloom.generators.common import default_fallback_resolver
59	59	from bloom.generators.common import invalidate_view_cache
	60	from bloom.generators.common import package_conditional_context
60	61	from bloom.generators.common import resolve_rosdep_key
61	62
62	63	from bloom.git import inbranch

146	147	if not os.path.exists(dst):
147	148	os.makedirs(dst)
148	149	if os.path.exists(template_dst):
149		debug("Removing existing file '{0}'".format(template_dst))
150		os.remove(template_dst)
151		with io.open(template_dst, 'w', encoding='utf-8') as f:
152		if not isinstance(template, str):
153		template = template.decode('utf-8')
154		# Python 2 API needs a `unicode` not a utf-8 string.
155		elif sys.version_info.major == 2:
156		template = template.decode('utf-8')
157		f.write(template)
158		shutil.copystat(template_abs_path, template_dst)
	150	debug("Not overwriting existing file '{0}'".format(template_dst))
	151	else:
	152	with io.open(template_dst, 'w', encoding='utf-8') as f:
	153	if not isinstance(template, str):
	154	template = template.decode('utf-8')
	155	# Python 2 API needs a `unicode` not a utf-8 string.
	156	elif sys.version_info.major == 2:
	157	template = template.decode('utf-8')
	158	f.write(template)
	159	shutil.copystat(template_abs_path, template_dst)
159	160
160	161
161	162	def place_template_files(path, build_type, gbp=False):

311	312	# Installation prefix
312	313	data['InstallationPrefix'] = installation_prefix
313	314	# Resolve dependencies
314		depends = package.run_depends + package.buildtool_export_depends
315		build_depends = package.build_depends + package.buildtool_depends + package.test_depends
316
317		unresolved_keys = depends + build_depends + package.replaces + package.conflicts
	315	package.evaluate_conditions(package_conditional_context(ros_distro))
	316	depends = [
	317	dep for dep in (package.run_depends + package.buildtool_export_depends)
	318	if dep.evaluated_condition]
	319	build_depends = [
	320	dep for dep in (package.build_depends + package.buildtool_depends + package.test_depends)
	321	if dep.evaluated_condition]
	322
	323	unresolved_keys = [
	324	dep for dep in (depends + build_depends + package.replaces + package.conflicts)
	325	if dep.evaluated_condition]
318	326	# The installer key is not considered here, but it is checked when the keys are checked before this
319	327	resolved_deps = resolve_dependencies(unresolved_keys, os_name,
320	328	os_version, ros_distro,

646	654	update_rosdep()
647	655	self.has_run_rosdep = True
648	656
649		def _check_all_keys_are_valid(self, peer_packages):
	657	def _check_all_keys_are_valid(self, peer_packages, ros_distro):
650	658	keys_to_resolve = []
651	659	key_to_packages_which_depends_on = collections.defaultdict(list)
652	660	keys_to_ignore = set()
653	661	for package in self.packages.values():
654		depends = package.run_depends + package.buildtool_export_depends
655		build_depends = package.build_depends + package.buildtool_depends + package.test_depends
656		unresolved_keys = depends + build_depends + package.replaces + package.conflicts
657		keys_to_ignore = keys_to_ignore.union(package.replaces + package.conflicts)
	662	package.evaluate_conditions(package_conditional_context(ros_distro))
	663	depends = [
	664	dep for dep in (package.run_depends + package.buildtool_export_depends)
	665	if dep.evaluated_condition]
	666	build_depends = [
	667	dep for dep in (package.build_depends + package.buildtool_depends + package.test_depends)
	668	if dep.evaluated_condition]
	669	unresolved_keys = [
	670	dep for dep in (depends + build_depends + package.replaces + package.conflicts)
	671	if dep.evaluated_condition]
	672	keys_to_ignore = {
	673	dep for dep in keys_to_ignore.union(package.replaces + package.conflicts)
	674	if dep.evaluated_condition}
658	675	keys = [d.name for d in unresolved_keys]
659	676	keys_to_resolve.extend(keys)
660	677	for key in keys:

698	715
699	716	peer_packages = [p.name for p in self.packages.values()]
700	717
701		while not self._check_all_keys_are_valid(peer_packages):
	718	while not self._check_all_keys_are_valid(peer_packages, self.rosdistro):
702	719	error("Some of the dependencies for packages in this repository could not be resolved by rosdep.")
703	720	error("You can try to address the issues which appear above and try again if you wish.")
704	721	try:

837	854	place_template_files('.', build_type, gbp=True)
838	855	# Commit results
839	856	execute_command('git add ' + debian_dir)
840		execute_command('git commit -m "Placing debian template files"')
	857	_, has_files, _ = execute_command('git diff --cached --name-only', return_io=True)
	858	if has_files:
	859	execute_command('git commit -m "Placing debian template files"')
841	860
842	861	def get_releaser_history(self):
843	862	# Assumes that this is called in the target branch

+1

-1

bloom/generators/debian/templates/ament_cmake/compat.em less more

0		@(debhelper_version)⏎
	0	@(debhelper_version)

+1

-1

bloom/generators/debian/templates/ament_python/compat.em less more

0		@(debhelper_version)⏎
	0	@(debhelper_version)

+1

-1

bloom/generators/debian/templates/catkin/compat.em less more

0		@(debhelper_version)⏎
	0	@(debhelper_version)

+1

-1

bloom/generators/debian/templates/cmake/compat.em less more

0		@(debhelper_version)⏎
	0	@(debhelper_version)

+8

-4

bloom/generators/rosdebian.py less more

43	43
44	44	from bloom.logging import info
45	45
	46	from bloom.rosdistro_api import get_index
	47
46	48	from bloom.util import get_distro_list_prompt
47	49
48	50

87	89	subs['Package'] = rosify_package_name(subs['Package'], self.rosdistro)
88	90
89	91	# ROS 2 specific bloom extensions.
90		# TODO(nuclearsandwich) explore other ways to enable these extensions, reduce their necessity,
91		# or make them configurable rather than relying on distro names.
92		if self.rosdistro in ['r2b2', 'r2b3', 'ardent', 'bouncy', 'crystal']:
	92	ros2_distros = [
	93	name for name, values in get_index().distributions.items()
	94	if values.get('distribution_type') == 'ros2']
	95	if self.rosdistro in ros2_distros:
93	96	# Add ros-workspace package as a dependency to any package other
94	97	# than ros_workspace and its dependencies.
95	98	if package.name not in ['ament_cmake_core', 'ament_package', 'ros_workspace']:

99	102
100	103	# Add packages necessary to build vendor typesupport for rosidl_interface_packages to their
101	104	# build dependencies.
102		if self.rosdistro in ['bouncy', 'crystal'] and \
	105	if self.rosdistro in ros2_distros and \
	106	self.rosdistro not in ('r2b2', 'r2b3', 'ardent') and \
103	107	'rosidl_interface_packages' in [p.name for p in package.member_of_groups]:
104	108	ROS2_VENDOR_TYPESUPPORT_DEPENDENCIES = [
105	109	'rmw-connext-cpp',

+26

-9

bloom/generators/rpm/generator.py less more

59	59
60	60	from bloom.generators.common import default_fallback_resolver
61	61	from bloom.generators.common import invalidate_view_cache
	62	from bloom.generators.common import package_conditional_context
62	63	from bloom.generators.common import resolve_rosdep_key
63	64
64	65	from bloom.git import inbranch

230	231	# Installation prefix
231	232	data['InstallationPrefix'] = installation_prefix
232	233	# Resolve dependencies
233		depends = package.run_depends + package.buildtool_export_depends
234		build_depends = package.build_depends + package.buildtool_depends + package.test_depends
235		unresolved_keys = depends + build_depends + package.replaces + package.conflicts
	234	package.evaluate_conditions(package_conditional_context(ros_distro))
	235	depends = [
	236	dep for dep in (package.run_depends + package.buildtool_export_depends)
	237	if dep.evaluated_condition]
	238	build_depends = [
	239	dep for dep in (package.build_depends + package.buildtool_depends + package.test_depends)
	240	if dep.evaluated_condition]
	241	unresolved_keys = [
	242	dep for dep in (depends + build_depends + package.replaces + package.conflicts)
	243	if dep.evaluated_condition]
236	244	# The installer key is not considered here, but it is checked when the keys are checked before this
237	245	resolved_deps = resolve_dependencies(unresolved_keys, os_name,
238	246	os_version, ros_distro,

525	533	update_rosdep()
526	534	self.has_run_rosdep = True
527	535
528		def _check_all_keys_are_valid(self, peer_packages):
	536	def _check_all_keys_are_valid(self, peer_packages, rosdistro):
529	537	keys_to_resolve = []
530	538	key_to_packages_which_depends_on = collections.defaultdict(list)
531	539	keys_to_ignore = set()
532	540	for package in self.packages.values():
533		depends = package.run_depends + package.buildtool_export_depends
534		build_depends = package.build_depends + package.buildtool_depends + package.test_depends
535		unresolved_keys = depends + build_depends + package.replaces + package.conflicts
536		keys_to_ignore = keys_to_ignore.union(package.replaces + package.conflicts)
	541	package.evaluate_conditions(package_conditional_context(rosdistro))
	542	depends = [
	543	dep for dep in (package.run_depends + package.buildtool_export_depends)
	544	if dep.evaluated_condition]
	545	build_depends = [
	546	dep for dep in (package.build_depends + package.buildtool_depends + package.test_depends)
	547	if dep.evaluated_condition]
	548	unresolved_keys = [
	549	dep for dep in (depends + build_depends + package.replaces + package.conflicts)
	550	if dep.evaluated_condition]
	551	keys_to_ignore = {
	552	dep for dep in keys_to_ignore.union(package.replaces + package.conflicts)
	553	if dep.evaluated_condition}
537	554	keys = [d.name for d in unresolved_keys]
538	555	keys_to_resolve.extend(keys)
539	556	for key in keys:

577	594
578	595	peer_packages = [p.name for p in self.packages.values()]
579	596
580		while not self._check_all_keys_are_valid(peer_packages):
	597	while not self._check_all_keys_are_valid(peer_packages, self.rosdistro):
581	598	error("Some of the dependencies for packages in this repository could not be resolved by rosdep.")
582	599	error("You can try to address the issues which appear above and try again if you wish, "
583	600	"or continue without releasing into RPM-based distributions (e.g. Fedora 24).")

+109

-1

bloom/github.py less more

38	38
39	39	import base64
40	40	import datetime
	41	import getpass
41	42	import json
	43	import os
42	44	import socket
43	45	import sys
	46
	47	from bloom.logging import error
	48	from bloom.logging import info
	49	from bloom.logging import warning
	50
	51	from bloom.util import maybe_continue
	52	from bloom.util import safe_input
44	53
45	54
46	55	try:

49	58	from urllib2 import HTTPError
50	59	from urllib2 import Request, urlopen
51	60	from urllib2 import URLError
	61	from urlparse import urlparse
52	62	from urlparse import urlunsplit
53	63	except ImportError:
54	64	# Python3
55	65	from urllib.error import HTTPError
56	66	from urllib.error import URLError
	67	from urllib.parse import urlparse
57	68	from urllib.parse import urlunsplit
58	69	from urllib.request import Request, urlopen
59	70

61	72
62	73
63	74	def auth_header_from_basic_auth(user, password):
64		return "Basic {0}".format(base64.b64encode('{0}:{1}'.format(user, password)))
	75	auth_str = '{0}:{1}'.format(user, password)
	76	if sys.version_info >= (3, 0):
	77	auth_str = auth_str.encode()
	78	return "Basic {0}".format(base64.b64encode(auth_str))
65	79
66	80
67	81	def auth_header_from_oauth_token(token):

239	253	raise GithubException("Failed to create pull request", resp)
240	254	resp_json = json_loads(resp)
241	255	return resp_json['html_url']
	256
	257
	258	def get_gh_info(url):
	259	o = urlparse(url)
	260	if 'raw.github.com' not in o.netloc and 'raw.githubusercontent.com' not in o.netloc:
	261	return None
	262	url_paths = o.path.split('/')
	263	if len(url_paths) < 5:
	264	return None
	265	return {'server': 'github.com',
	266	'org': url_paths[1],
	267	'repo': url_paths[2],
	268	'branch': url_paths[3],
	269	'path': '/'.join(url_paths[4:])}
	270
	271
	272	_gh = None
	273
	274
	275	def get_github_interface(quiet=False):
	276	def mfa_prompt(oauth_config_path, username):
	277	"""Explain how to create a token for users with Multi-Factor Authentication configured."""
	278	warning("Receiving 401 when trying to create an oauth token can be caused by the user "
	279	"having two-factor authentication enabled.")
	280	warning("If 2FA is enabled, the user will have to create an oauth token manually.")
	281	warning("A token can be created at https://github.com/settings/tokens")
	282	warning("The resulting token can be placed in the '{oauth_config_path}' file as such:"
	283	.format(**locals()))
	284	info("")
	285	warning('{{"github_user": "{username}", "oauth_token": "TOKEN_GOES_HERE"}}'
	286	.format(**locals()))
	287	info("")
	288
	289	global _gh
	290	if _gh is not None:
	291	return _gh
	292	# First check to see if the oauth token is stored
	293	oauth_config_path = os.path.join(os.path.expanduser('~'), '.config', 'bloom')
	294	config = {}
	295	if os.path.exists(oauth_config_path):
	296	with open(oauth_config_path, 'r') as f:
	297	config = json.loads(f.read())
	298	token = config.get('oauth_token', None)
	299	username = config.get('github_user', None)
	300	if token and username:
	301	return Github(username, auth=auth_header_from_oauth_token(token), token=token)
	302	if not os.path.isdir(os.path.dirname(oauth_config_path)):
	303	os.makedirs(os.path.dirname(oauth_config_path))
	304	if quiet:
	305	return None
	306	# Ok, now we have to ask for the user name and pass word
	307	info("")
	308	warning("Looks like bloom doesn't have an oauth token for you yet.")
	309	warning("Therefore bloom will require your GitHub username and password just this once.")
	310	warning("With your GitHub username and password bloom will create an oauth token on your behalf.")
	311	warning("The token will be stored in `~/.config/bloom`.")
	312	warning("You can delete the token from that file to have a new token generated.")
	313	warning("Guard this token like a password, because it allows someone/something to act on your behalf.")
	314	warning("If you need to unauthorize it, remove it from the 'Applications' menu in your GitHub account page.")
	315	info("")
	316	if not maybe_continue('y', "Would you like to create an OAuth token now"):
	317	return None
	318	token = None
	319	while token is None:
	320	try:
	321	username = getpass.getuser()
	322	username = safe_input("GitHub username [{0}]: ".format(username)) or username
	323	password = getpass.getpass("GitHub password (never stored): ")
	324	except (KeyboardInterrupt, EOFError):
	325	return None
	326	if not password:
	327	error("No password was given, aborting.")
	328	return None
	329	gh = Github(username, auth=auth_header_from_basic_auth(username, password))
	330	try:
	331	token = gh.create_new_bloom_authorization(update_auth=True)
	332	with open(oauth_config_path, 'w') as f:
	333	config.update({'oauth_token': token, 'github_user': username})
	334	f.write(json.dumps(config))
	335	info("The token '{token}' was created and stored in the bloom config file: '{oauth_config_path}'"
	336	.format(**locals()))
	337	except GitHubAuthException as exc:
	338	error("{0}".format(exc))
	339	mfa_prompt(oauth_config_path, username)
	340	except GithubException as exc:
	341	error("{0}".format(exc))
	342	info("")
	343	if hasattr(exc, 'resp') and '{0}'.format(exc.resp.status) in ['401']:
	344	mfa_prompt(oauth_config_path, username)
	345	warning("This sometimes fails when the username or password are incorrect, try again?")
	346	if not maybe_continue():
	347	return None
	348	_gh = gh
	349	return gh

+182

-0

bloom/rosdistro_api.py less more

	0	# Software License Agreement (BSD License)
	1	#
	2	# Copyright (c) 2014, Open Source Robotics Foundation, Inc.
	3	# Copyright (c) 2013, Willow Garage, Inc.
	4	# All rights reserved.
	5	#
	6	# Redistribution and use in source and binary forms, with or without
	7	# modification, are permitted provided that the following conditions
	8	# are met:
	9	#
	10	# * Redistributions of source code must retain the above copyright
	11	# notice, this list of conditions and the following disclaimer.
	12	# * Redistributions in binary form must reproduce the above
	13	# copyright notice, this list of conditions and the following
	14	# disclaimer in the documentation and/or other materials provided
	15	# with the distribution.
	16	# * Neither the name of Willow Garage, Inc. nor the names of its
	17	# contributors may be used to endorse or promote products derived
	18	# from this software without specific prior written permission.
	19	#
	20	# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	21	# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	22	# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
	23	# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
	24	# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
	25	# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	26	# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	27	# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
	28	# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	29	# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
	30	# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	31	# POSSIBILITY OF SUCH DAMAGE.
	32
	33	from __future__ import print_function
	34	from __future__ import unicode_literals
	35
	36	import os
	37	import sys
	38	import traceback
	39
	40	from pkg_resources import parse_version
	41
	42	# python2/3 compatibility
	43	try:
	44	from urllib.parse import urlparse
	45	except ImportError:
	46	from urlparse import urlparse
	47
	48	from bloom.github import Github
	49	from bloom.github import GithubException
	50	from bloom.github import get_gh_info
	51	from bloom.github import get_github_interface
	52
	53	from bloom.logging import debug
	54	from bloom.logging import error
	55	from bloom.logging import info
	56
	57
	58	try:
	59	import rosdistro
	60	if parse_version(rosdistro.__version__) < parse_version('0.7.0'):
	61	error("rosdistro version 0.7.0 or greater is required, found '{0}' from '{1}'."
	62	.format(rosdistro.__version__, os.path.dirname(rosdistro.__file__)),
	63	exit=True)
	64	except ImportError:
	65	debug(traceback.format_exc())
	66	error("rosdistro was not detected, please install it.", file=sys.stderr,
	67	exit=True)
	68
	69	_rosdistro_index = None
	70	_rosdistro_distribution_files = {}
	71	_rosdistro_index_commit = None
	72	_rosdistro_index_original_branch = None
	73
	74
	75	def get_index_url():
	76	global _rosdistro_index_commit, _rosdistro_index_original_branch
	77	index_url = rosdistro.get_index_url()
	78	pr = urlparse(index_url)
	79	if pr.netloc in ['raw.github.com', 'raw.githubusercontent.com']:
	80	# Try to determine what the commit hash was
	81	tokens = [x for x in pr.path.split('/') if x]
	82	if len(tokens) <= 3:
	83	debug("Failed to get commit for rosdistro index file: index url")
	84	debug(tokens)
	85	return index_url
	86	owner = tokens[0]
	87	repo = tokens[1]
	88	branch = tokens[2]
	89	gh = get_github_interface(quiet=True)
	90	if gh is None:
	91	# Failed to get it with auth, try without auth (may fail)
	92	gh = Github(username=None, auth=None)
	93	try:
	94	data = gh.get_branch(owner, repo, branch)
	95	except GithubException:
	96	debug(traceback.format_exc())
	97	debug("Failed to get commit for rosdistro index file: api")
	98	return index_url
	99	_rosdistro_index_commit = data.get('commit', {}).get('sha', None)
	100	if _rosdistro_index_commit is not None:
	101	info("ROS Distro index file associate with commit '{0}'"
	102	.format(_rosdistro_index_commit))
	103	# Also mutate the index_url to use the commit (rather than the moving branch name)
	104	base_info = get_gh_info(index_url)
	105	base_branch = base_info['branch']
	106	rosdistro_index_commit = _rosdistro_index_commit # Copy global into local for substitution
	107	middle = "{org}/{repo}".format(**base_info)
	108	index_url = index_url.replace("{pr.netloc}/{middle}/{base_branch}/".format(**locals()),
	109	"{pr.netloc}/{middle}/{rosdistro_index_commit}/".format(**locals()))
	110	info("New ROS Distro index url: '{0}'".format(index_url))
	111	_rosdistro_index_original_branch = base_branch
	112	else:
	113	debug("Failed to get commit for rosdistro index file: json")
	114	return index_url
	115
	116
	117	def get_index():
	118	global _rosdistro_index
	119	if _rosdistro_index is None:
	120	_rosdistro_index = rosdistro.get_index(get_index_url())
	121	if _rosdistro_index.version == 1:
	122	error("This version of bloom does not support rosdistro version "
	123	"'{0}', please use an older version of bloom."
	124	.format(_rosdistro_index.version), exit=True)
	125	if _rosdistro_index.version > 4:
	126	error("This version of bloom does not support rosdistro version "
	127	"'{0}', please update bloom.".format(_rosdistro_index.version), exit=True)
	128	return _rosdistro_index
	129
	130
	131	def list_distributions():
	132	return sorted(get_index().distributions.keys())
	133
	134
	135	def get_distribution_type(distro):
	136	return get_index().distributions[distro].get('distribution_type')
	137
	138
	139	def get_most_recent(thing_name, repository, reference_distro):
	140	reference_distro_type = get_distribution_type(reference_distro)
	141	distros_with_entry = {}
	142	get_things = {
	143	'release': lambda r: None if r.release_repository is None else r.release_repository,
	144	'doc': lambda r: None if r.doc_repository is None else r.doc_repository,
	145	'source': lambda r: None if r.source_repository is None else r.source_repository,
	146	}
	147	get_thing = get_things[thing_name]
	148	for distro in list_distributions():
	149	# skip distros with a different type if the information is available
	150	if reference_distro_type is not None:
	151	if get_distribution_type(distro) != reference_distro_type:
	152	continue
	153	distro_file = get_distribution_file(distro)
	154	if repository in distro_file.repositories:
	155	thing = get_thing(distro_file.repositories[repository])
	156	if thing is not None:
	157	distros_with_entry[distro] = thing
	158	# Choose the alphabetical last distro which contained a release of this repository
	159	default_distro = (sorted(distros_with_entry.keys()) or [None])[-1]
	160	default_thing = distros_with_entry.get(default_distro, None)
	161	return default_distro, default_thing
	162
	163
	164	def get_distribution_file(distro):
	165	global _rosdistro_distribution_files
	166	if distro not in _rosdistro_distribution_files:
	167	# REP 143, get list of distribution files and take the last one
	168	files = rosdistro.get_distribution_files(get_index(), distro)
	169	if not files:
	170	error("No distribution files listed for distribution '{0}'."
	171	.format(distro), exit=True)
	172	_rosdistro_distribution_files[distro] = files[-1]
	173	return _rosdistro_distribution_files[distro]
	174
	175
	176	def get_rosdistro_index_commit():
	177	return _rosdistro_index_commit
	178
	179
	180	def get_rosdistro_index_original_branch():
	181	return _rosdistro_index_original_branch

+2

-1

docs/conf.py less more

13	13	from __future__ import print_function
14	14
15	15	import os
	16	import sys
16	17
17	18	# If extensions (or modules to document with autodoc) are in another directory,
18	19	# add these directories to sys.path here. If the directory is relative to the

60	61	# The full version, including alpha/beta/rc tags.
61	62	setup_py = os.path.join(os.path.dirname(__file__), '..', 'setup.py')
62	63	import subprocess
63		release = subprocess.check_output('/usr/bin/env python ' + setup_py + ' --version', shell=True).strip()
	64	release = subprocess.check_output(sys.executable + ' ' + setup_py + ' --version', shell=True).strip().decode('utf-8')
64	65	print('Using release version: {0}'.format(release))
65	66
66	67	# The language for content autogenerated by Sphinx. Refer to documentation

+1

-1

setup.py less more

21	21
22	22	setup(
23	23	name='bloom',
24		version='0.7.2',
	24	version='0.8.0',
25	25	packages=find_packages(exclude=['test', 'test.*']),
26	26	package_data={
27	27	'bloom.generators.debian': [

+2

-2

stdeb.cfg less more

3	3	Conflicts: python3-bloom
4	4	Conflicts3: python-bloom
5	5	Copyright-File: LICENSE.txt
6		Suite: oneiric precise quantal raring saucy trusty utopic vivid wily xenial yakkety zesty artful bionic wheezy jessie stretch buster
7		X-Python3-Version: >= 3.2
	6	Suite: trusty utopic vivid wily xenial yakkety zesty artful bionic jessie stretch buster
	7	X-Python3-Version: >= 3.4

+8

-0

test/__init__.py less more

4	4	scripts = os.path.join(os.path.dirname(__file__), '..', 'scripts')
5	5	scripts = os.path.abspath(scripts)
6	6	os.environ['PATH'] = scripts + ':' + os.environ['PATH']
	7
	8	user_email = 'test@example.com'
	9	user_name = 'Test User'
	10
	11	os.environ.setdefault('GIT_AUTHOR_NAME', user_name)
	12	os.environ.setdefault('GIT_AUTHOR_EMAIL', user_email)
	13	os.environ.setdefault('GIT_COMMITTER_NAME', user_name)
	14	os.environ.setdefault('GIT_COMMITTER_EMAIL', user_email)

+4

-4

test/system_tests/test_catkin_release.py less more

131	131	"no patches/release/melodic/foo branch"
132	132	# was the release tag created?
133	133	ret, out, err = user('git tag', return_io=True)
134		expected = 'release/melodic/foo/' + version + '-0'
	134	expected = 'release/melodic/foo/' + version + '-1'
135	135	assert out.count(expected) == 1, \
136	136	"no release tag created, expected: '{0}'".format(expected)
137	137

265	265	assert branch_exists('patches/release/melodic/' + pkg), \
266	266	"no patches/release/melodic/" + pkg + " branch"
267	267	# Did the release tag get created?
268		assert out.count('release/melodic/' + pkg + '/0.1.0-0') == 1, \
	268	assert out.count('release/melodic/' + pkg + '/0.1.0-1') == 1, \
269	269	"no release tag created for " + pkg
270	270	# Is there a package.xml in the top level?
271	271	with inbranch('release/melodic/' + pkg):

298	298	assert branch_exists('patches/release/melodic/' + pkg), \
299	299	"no patches/release/melodic/" + pkg + " branch"
300	300	# Did the release tag get created?
301		assert out.count('release/melodic/' + pkg + '/0.1.0-0') == 1, \
	301	assert out.count('release/melodic/' + pkg + '/0.1.0-1') == 1, \
302	302	"no release tag created for " + pkg
303	303	# Is there a package.xml in the top level?
304	304	with inbranch('release/melodic/' + pkg):

328	328	assert branch_exists(patches_branch), \
329	329	"no " + patches_branch + " branch"
330	330	# Did the debian tag get created?
331		tag = 'debian/ros-melodic-' + pkg_san + '_0.1.0-0_' + distro
	331	tag = 'debian/ros-melodic-' + pkg_san + '_0.1.0-1_' + distro
332	332	assert out.count(tag) == 1, \
333	333	"no '" + tag + "'' tag created for '" + pkg + "': `\n" + \
334	334	out + "\n`"

+5

-0

test/unit_tests/test_packages.py less more

0	0	import os
1	1
2	2	from ..utils.common import AssertRaisesContext
	3	from ..utils.common import in_temporary_directory
3	4	from ..utils.common import redirected_stdio
	5	from ..utils.common import user
4	6
5	7	from bloom.packages import get_package_data
6	8
7	9	test_data_dir = os.path.join(os.path.dirname(__file__), 'test_packages_data')
8	10
9	11
	12	@in_temporary_directory
10	13	def test_get_package_data_fails_on_uppercase():
	14	user('git init .')
	15
11	16	with AssertRaisesContext(SystemExit, "Invalid package names, aborting."):
12	17	with redirected_stdio():
13	18	get_package_data(directory=test_data_dir)