Commit c105af43547d11f3dd86708059c281d5fbb5177b - pyosmium

+18

-0

.github/actions/install-dependencies/action.yml less more

	0	name: 'Install pyosmium dependencies'
	1
	2	runs:
	3	using: "composite"
	4
	5	steps:
	6	- name: Install pip dependencies
	7	run: \|
	8	python -m pip install --upgrade pip
	9	pip install nose shapely setuptools
	10	shell: bash
	11
	12	- name: Install package dependencies
	13	run: \|
	14	git clone --quiet --depth 1 https://github.com/osmcode/libosmium.git contrib/libosmium
	15	git clone --quiet --depth 1 https://github.com/mapbox/protozero.git contrib/protozero
	16	git clone --quiet --depth 1 https://github.com/pybind/pybind11.git contrib/pybind11
	17	shell: bash

+141

-0

.github/workflows/ci.yml less more

	0	name: CI
	1
	2	on: [ push, pull_request ]
	3
	4	jobs:
	5	build-default:
	6	runs-on: ubuntu-20.04
	7
	8	strategy:
	9	matrix:
	10	python-version: [3.6, 3.7, 3.8]
	11
	12	steps:
	13	- uses: actions/checkout@v2
	14
	15	- name: Set up Python ${{ matrix.python-version }}
	16	uses: actions/setup-python@v2
	17	with:
	18	python-version: ${{ matrix.python-version }}
	19
	20	- name: Install packages
	21	run: sudo apt-get install -y -qq libboost-dev libexpat1-dev zlib1g-dev libbz2-dev libproj-dev libgeos-dev
	22
	23	- uses: ./.github/actions/install-dependencies
	24
	25	- name: Build package
	26	run: python setup.py build
	27	shell: bash
	28
	29	- name: Run tests
	30	run: python run_tests.py
	31	shell: bash
	32	working-directory: test
	33
	34
	35	build-ubuntu-1604:
	36	runs-on: ubuntu-16.04
	37
	38	strategy:
	39	matrix:
	40	compiler: [gcc, clang]
	41
	42	steps:
	43	- uses: actions/checkout@v2
	44
	45	- uses: actions/setup-python@v2
	46	with:
	47	python-version: 3.5
	48
	49	- name: Install packages
	50	run: sudo apt-get install -y -qq libboost-dev libexpat1-dev zlib1g-dev libbz2-dev libproj-dev libgeos-dev
	51
	52	- uses: ./.github/actions/install-dependencies
	53
	54	- name: Build package
	55	run: python setup.py build
	56	shell: bash
	57	env:
	58	CC: gcc-5
	59	CXX: g++-5
	60	if: ${{ matrix.compiler == 'gcc' }}
	61
	62	- name: Build package
	63	run: python setup.py build
	64	shell: bash
	65	env:
	66	CC: clang-6.0
	67	CXX: clang++-6.0
	68	if: ${{ matrix.compiler == 'clang' }}
	69
	70	- name: Run tests
	71	run: python run_tests.py
	72	shell: bash
	73	working-directory: test
	74
	75	build-ubuntu-2004:
	76	runs-on: ubuntu-20.04
	77
	78	strategy:
	79	matrix:
	80	compiler: [gcc, clang]
	81
	82	steps:
	83	- uses: actions/checkout@v2
	84
	85	- uses: actions/setup-python@v2
	86	with:
	87	python-version: 3.9
	88
	89	- name: Install packages
	90	run: sudo apt-get install -y -qq libboost-dev libexpat1-dev zlib1g-dev libbz2-dev libproj-dev libgeos-dev
	91
	92	- uses: ./.github/actions/install-dependencies
	93
	94	- name: Build package
	95	run: python setup.py build
	96	shell: bash
	97	env:
	98	CC: gcc-10
	99	CXX: g++-10
	100	if: ${{ matrix.compiler == 'gcc' }}
	101
	102	- name: Build package
	103	run: python setup.py build
	104	shell: bash
	105	env:
	106	CC: clang-10
	107	CXX: clang++-10
	108	if: ${{ matrix.compiler == 'clang' }}
	109
	110	- name: Run tests
	111	run: python run_tests.py
	112	shell: bash
	113	working-directory: test
	114
	115	build-macos:
	116	runs-on: macos-latest
	117
	118	steps:
	119	- uses: actions/checkout@v2
	120
	121	- uses: actions/setup-python@v2
	122	with:
	123	python-version: 3
	124
	125	- name: Install packages
	126	run: brew install boost geos
	127	shell: bash
	128
	129	- uses: ./.github/actions/install-dependencies
	130
	131	- name: Build package
	132	run: python setup.py build
	133	shell: bash
	134
	135	- name: Run tests
	136	run: python run_tests.py
	137	shell: bash
	138	working-directory: test
	139
	140

+0

-84

~~.travis.yml~~ less more

0		#-----------------------------------------------------------------------------
1		#
2		# Configuration for continuous integration service at travis-ci.org
3		#
4		#-----------------------------------------------------------------------------
5
6
7		matrix:
8		include:
9		- os: linux
10		compiler: "clang-3.4"
11		dist: xenial
12		language: cpp
13		- os: linux
14		compiler: "clang-7"
15		dist: bionic
16		language: cpp
17		- os: linux
18		compiler: "gcc-5"
19		dist: xenial
20		language: cpp
21		- os: linux
22		compiler: "gcc-8"
23		dist: bionic
24		language: cpp
25		- os: linux
26		compiler: gcc
27		python: 3.4
28		dist: xenial
29		language: python
30		- os: linux
31		compiler: gcc
32		python: 3.5
33		dist: bionic
34		language: python
35		- os: linux
36		compiler: gcc
37		python: 3.6
38		dist: bionic
39		language: python
40		- os: linux
41		compiler: gcc
42		python: 3.7
43		dist: bionic
44		language: python
45		- os: linux
46		compiler: gcc
47		python: 3.8
48		dist: bionic
49		language: python
50		- os: osx
51		osx_image: xcode7
52		compiler: clang
53		language: cpp
54		- os: osx
55		osx_image: xcode10.1
56		compiler: clang
57		language: cpp
58
59		# http://docs.travis-ci.com/user/apt/
60		addons:
61		apt:
62		packages:
63		- libboost-dev
64		- python3
65		- python3-dev
66		- python3-pip
67
68		install:
69		- git clone --quiet --depth 1 https://github.com/osmcode/libosmium.git contrib/libosmium
70		- git clone --quiet --depth 1 https://github.com/mapbox/protozero.git contrib/protozero
71		- git clone --quiet --depth 1 https://github.com/pybind/pybind11.git contrib/pybind11
72		- if [ "$TRAVIS_OS_NAME" = 'osx' ]; then
73		pip3 install --user -q nose mock shapely setuptools;
74		else
75		pip3 install -q nose mock shapely setuptools;
76		fi
77
78		script:
79		- python3 --version
80		- python3 setup.py build
81		- cd test
82		- python3 run_tests.py
83

+14

-0

CHANGELOG.md less more

2	2
3	3	All notable changes to this project will be documented in this file.
4	4	This project adheres to [Semantic Versioning](http://semver.org/).
	5
	6	## [3.1.0] - 2020-11-03
	7
	8	### Added
	9
	10	### Changed
	11
	12	- improved help messages for update tools
	13	- update to pybind11 2.6
	14
	15	### Fixed
	16
	17	- pyosmium-up-to-date: fix missing flag when applying changes to history files
	18	- smaller linting and documentation issues
5	19
6	20	## [3.0.1] - 2020-07-25
7	21

+14

-2

CMakeLists.txt less more

0	0	cmake_minimum_required(VERSION 2.8.12)
1	1	project(pyosmium)
	2	set(PACKAGE_VERSION 3.0.1)
2	3
3	4	list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
4	5

6	7	include_directories(SYSTEM ${OSMIUM_INCLUDE_DIRS} ${PROTOZERO_INCLUDE_DIR})
7	8	include_directories(${CMAKE_CURRENT_SOURCE_DIR}/lib)
8	9
	10	if(NOT "${CMAKE_CXX_STANDARD}")
	11	if(MSVC)
	12	set(CMAKE_CXX_STANDARD 14)
	13	else()
	14	set(CMAKE_CXX_STANDARD 11)
	15	endif()
	16	endif()
	17
	18	# required for pybind11 < 2.6
9	19	if(MSVC)
10		set(PYBIND11_CPP_STANDARD /std:c++14)
	20	set(PYBIND11_CPP_STANDARD /std=c++${CMAKE_CXX_STANDARD})
11	21	else()
12		set(PYBIND11_CPP_STANDARD -std=c++11)
	22	set(PYBIND11_CPP_STANDARD -std=c++${CMAKE_CXX_STANDARD})
13	23	endif()
	24
	25	message(STATUS "Building in C++${CMAKE_CXX_STANDARD} mode")
14	26
15	27	if(PYBIND11_PREFIX)
16	28	add_subdirectory(${PYBIND11_PREFIX} contrib/pybind11)

+1

-1

README.md less more

3	3	library, a library for working with OpenStreetMap data in a fast and flexible
4	4	manner.
5	5
6		[![Travis Build Status](https://api.travis-ci.org/osmcode/pyosmium.svg)](http://travis-ci.org/osmcode/pyosmium)
	6	[![Github Actions Build Status](https://github.com/osmcode/pyosmium/workflows/CI/badge.svg)](https://github.com/osmcode/pyosmium/actions?query=workflow%3ACI)
7	7	[![Appeveyor Build status](https://ci.appveyor.com/api/projects/status/ch3gwxucycytako4/branch/master?svg=true)](https://ci.appveyor.com/project/lonvia/pyosmium/branch/master)
8	8
9	9	## Installation

+5

-4

doc/updating_osm_data.rst less more

32	32	for planet-wide updates. There are change files for
33	33	minutely, hourly and daily intervals available.
34	34
35		* `Geofabrik <http://download.geofabrik.de>`_ offers daily change files
	35	* `Geofabrik <https://download.geofabrik.de>`_ offers daily change files
36	36	for all its updates. See the extract page for a link to the replication URL.
37	37	Note that change files go only about 3 months back. Older files are deleted.
38	38
39		* `openstreetmap.fr <http://download.geofabrik.de>`_ offers minutely change
40		files for all its extracts.
	39	* download.openstreetmap.fr offers
	40	`minutely change files <https://download.openstreetmap.fr/replication/>`_
	41	for all its `extracts <https://download.openstreetmap.fr/extracts/>`_.
41	42
42		For other services also check out the list of services on the
	43	For other services also check out the list of providers on the
43	44	`OSM wiki <https://wiki.openstreetmap.org/wiki/Planet.osm>`_.
44	45
45	46	Updating a planet or extract

+1

-1

examples/amenity_list.py less more

12	12
13	13	class AmenityListHandler(o.SimpleHandler):
14	14
15		def print_amenity(amenity, tags, lon, lat):
	15	def print_amenity(self, tags, lon, lat):
16	16	name = tags.get('name', '')
17	17	print("%f %f %-15s %s" % (lon, lat, tags['amenity'], name))
18	18

+2

-3

examples/pub_names.py less more

6	6	class NamesHandler(osmium.SimpleHandler):
7	7
8	8	def output_pubs(self, tags):
9		if tags.get('amenity') == 'pub':
10		if 'name' in tags:
11		print(tags['name'])
	9	if tags.get('amenity') == 'pub' and 'name' in tags:
	10	print(tags['name'])
12	11
13	12	def node(self, n):
14	13	self.output_pubs(n.tags)

+1

-2

examples/use_nodecache.py less more

8	8
9	9	def way(self, w):
10	10	for n in w.nodes:
11		n.lat, n.lon # throws an exception if the coordinates are missing
12	11	loc = idx.get(n.ref)
13		print("%d %s" %(w.id, len(w.nodes)))
	12	print("%d %s" % (w.id, len(w.nodes)))
14	13
15	14	if len(sys.argv) != 3:
16	15	print("Usage: python create_nodecache.py <osm file> <node cache>")

+1

-1

lib/io.cc less more

58	58
59	59	py::class_<osmium::io::Writer>(m, "Writer",
60	60	"Class for writing OSM data to a file. This class just encapsulates an "
61		"OSM file,. Have a look `osmium.SimpleWriter` for a high-level interface "
	61	"OSM file. Have a look `osmium.SimpleWriter` for a high-level interface "
62	62	"for writing out data.")
63	63	.def(py::init<std::string>())
64	64	.def(py::init<osmium::io::File>())

+4

-5

lib/merge_input_reader.cc less more

52	52	class MergeInputReader
53	53	{
54	54	public:
55		void apply(BaseHandler& handler, std::string const &idx = "",
56		bool simplify = true)
	55	void apply(BaseHandler& handler, std::string const &idx, bool simplify)
57	56	{
58	57	if (idx.empty())
59	58	apply_without_location(handler, simplify);

62	61	}
63	62
64	63	void apply_to_reader(osmium::io::Reader &reader, osmium::io::Writer &writer,
65		bool with_history = true)
	64	bool with_history)
66	65	{
67	66	auto input = osmium::io::make_input_iterator_range<osmium::OSMObject>(reader);
68	67	if (with_history) {

109	108	}
110	109
111	110	private:
112		void apply_without_location(BaseHandler& handler, bool simplify = true)
	111	void apply_without_location(BaseHandler& handler, bool simplify)
113	112	{
114	113	if (simplify) {
115	114	objects.sort(osmium::object_order_type_id_reverse_version());

132	131	}
133	132
134	133	void apply_with_location(BaseHandler& handler, std::string const &idx,
135		bool simplify = true)
	134	bool simplify)
136	135	{
137	136	using Index_fab =
138	137	osmium::index::MapFactory<osmium::unsigned_object_id_type, osmium::Location>;

+6

-3

setup.py less more

104	104	if 'BOOST_PREFIX' in env:
105	105	cmake_args += ['-DBOOST_ROOT={}'.format(env['BOOST_PREFIX'])]
106	106
	107	if 'CMAKE_CXX_STANDARD' in env:
	108	cmake_args += ['-DCMAKE_CXX_STANDARD={}'.format(env['CMAKE_CXX_STANDARD'])]
	109
107	110	if not os.path.exists(self.build_temp):
108	111	os.makedirs(self.build_temp)
109	112	subprocess.check_call(['cmake', ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env)

111	114
112	115	versions = get_versions()
113	116
114		if sys.version_info < (3,0):
115		raise RuntimeError("Python 3.3 or larger required.")
	117	if sys.version_info < (3,4):
	118	raise RuntimeError("Python 3.4 or larger required.")
116	119
117	120	with open('README.rst', 'r') as descfile:
118	121	long_description = descfile.read()

127	130	maintainer='Sarah Hoffmann',
128	131	maintainer_email='lonvia@denofr.de',
129	132	download_url='https://github.com/osmcode/pyosmium',
130		url='http://osmcode.org/pyosmium',
	133	url='https://osmcode.org/pyosmium',
131	134	keywords=["OSM", "OpenStreetMap", "Osmium"],
132	135	license='BSD',
133	136	scripts=['tools/pyosmium-get-changes', 'tools/pyosmium-up-to-date'],

+0

-1

src/osmium/helper.py less more

18	18	__HandlerWithCallbacks.area = staticmethod(area)
19	19
20	20	return __HandlerWithCallbacks()
21

+57

-44

src/osmium/osm/__init__.py less more

	0	import osmium.osm.mutable
0	1	from ._osm import *
1		import osmium.osm.mutable
2	2
3	3	def create_mutable_node(node, **args):
4	4	""" Create a mutable node replacing the properties given in the

25	25	Way.replace = create_mutable_way
26	26	Relation.replace = create_mutable_relation
27	27
28		Location.__repr__ = lambda l : 'osmium.osm.Location(x=%r, y=%r)' \
29		% (l.x, l.y) \
	28	def _make_repr(attr_list):
	29	fmt_string = 'osmium.osm.{0}('\
	30	+ ', '.join(['{0}={{1.{0}!r}}'.format(x) for x in attr_list])\
	31	+ ')'
	32
	33	return lambda o: fmt_string.format(o.__class__.__name__, o)
	34
	35	def _list_repr(obj):
	36	return 'osmium.osm.{}([{}])'.format(obj.__class__.__name__,
	37	', '.join(map(repr, obj)))
	38
	39	def _list_elipse(obj):
	40	objects = ','.join(map(str, obj))
	41	if len(objects) > 50:
	42	objects = objects[:47] + '...'
	43	return objects
	44
	45	Location.__repr__ = lambda l: 'osmium.osm.Location(x={0.x!r}, y={0.y!r})'.format(l) \
30	46	if l.valid() else 'osmium.osm.Location()'
31		Location.__str__ = lambda l : '%f/%f' % (l.lon_without_check(), l.lat_without_check()) \
32		if l.valid() else 'invalid'
	47	Location.__str__ = lambda l: '{:f}/{:f}'.format(l.lon_without_check(),
	48	l.lat_without_check()) \
	49	if l.valid() else 'invalid'
33	50
34		Box.__repr__ = lambda b : 'osmium.osm.Box(bottom_left=%r, top_right=%r)' \
35		% (b.bottom_left, b.top_right)
36		Box.__str__ = lambda b : '(%s %s)' % (b.bottom_left, b.top_right)
	51	Box.__repr__ = _make_repr(['bottom_left', 'top_right'])
	52	Box.__str__ = lambda b: '({0.bottom_left!s} {0.top_right!s})'.format(b)
37	53
38		Tag.__repr__ = lambda t : 'osmium.osm.Tag(k=%r, v=%r)' % (t.k, t.v)
	54	Tag.__repr__ = _make_repr(['k', 'v'])
	55	Tag.__str__ = lambda t: '{0.k}={0.v}'.format(t)
39	56
40		Tag.__str__ = lambda t : '%s=%s' % (t.k, t.v)
	57	TagList.__repr__ = lambda t: "osmium.osm.TagList({%s})" \
	58	% ', '.join(["%r: %r" % (i.k, i.v) for i in t])
	59	TagList.__str__ = lambda t: '{' + _list_elipse(t) + '}'
41	60
42		TagList.__repr__ = lambda t : "osmium.osm.TagList({%s})" \
43		% ",".join(["%r=%r" % (i.k, i.v) for i in t])
44		TagList.__str__ = lambda t : "{%s}" % ",".join([str(i) for i in t])
	61	NodeRef.__repr__ = _make_repr(['ref', 'location'])
	62	NodeRef.__str__ = lambda n: '{0.ref:d}@{0.location!s}'.format(n) \
	63	if n.location.valid() else str(n.ref)
45	64
46		NodeRef.__repr__ = lambda n : 'osmium.osm.NodeRef(ref=%r, location=%r)' % (n.ref, n.location)
47		NodeRef.__str__ = lambda n : '%s@%s' % (n.ref, n.location) if n.location.valid() \
48		else str(n.ref)
	65	NodeRefList.__repr__ = _list_repr
	66	NodeRefList.__str__ = lambda o: '[' + _list_elipse(o) + ']'
49	67
50		NodeRefList.__repr__ = lambda t : "%s([%s])" % (t.__class__.__name__,
51		",".join([repr(i) for i in t]))
52		NodeRefList.__str__ = lambda t : "[%s]" % ",".join([str(i) for i in t])
	68	RelationMember.__repr__ = _make_repr(['ref', 'type', 'role'])
	69	RelationMember.__str__ = lambda r: ('{0.type}{0.ref:d}@{0.role}' \
	70	if r.role else '{0.type}{0.ref:d}').format(r)
53	71
54		RelationMember.__repr__ = lambda r : 'osmium.osm.RelationMember(ref=%r, type=%r, role=%r)' \
55		% (r.ref, r.type, r.role)
56		RelationMember.__str__ = lambda r : '%s%d@%s' % (r.type, r.ref, r.role) \
57		if r.role else '%s%d' % (r.type, r.ref)
	72	RelationMemberList.__repr__ = _list_repr
	73	RelationMemberList.__str__ = lambda o: '[' + _list_elipse(o) + ']'
58	74
59		RelationMemberList.__repr__ = lambda t : "osmium.osm.RelationMemberList([%s])" \
60		% ",".join([repr(i) for i in t])
61		RelationMemberList.__str__ = lambda t : "[%s]" % ",".join([str(i) for i in t])
	75	OSMObject.__repr__ = _make_repr(['id', 'deleted', 'visible', 'version', 'changeset',
	76	'uid', 'timestamp', 'user', 'tags'])
62	77
63		OSMObject.__repr__ = lambda o : '%s(id=%r, deleted=%r, visible=%r, version=%r, changeset=%r, uid=%r, timestamp=%r, user=%r, tags=%r)'% (o.__class__.__name__, o.id, o.deleted, o.visible, o.version, o.changeset, o.uid, o.timestamp, o.user, o.tags)
	78	Node.__repr__ = _make_repr(['id', 'deleted', 'visible', 'version', 'changeset',
	79	'uid', 'timestamp', 'user', 'tags', 'location'])
	80	Node.__str__ = lambda n: 'n{0.id:d}: location={0.location!s} tags={0.tags!s}'\
	81	.format(n)
64	82
65		def _str_ellipse(s, length=50):
66		s = str(s)
67		return s if len(s) <= length else (s[:length - 4] + '...' + s[-1])
	83	Way.__repr__ = _make_repr(['id', 'deleted', 'visible', 'version', 'changeset',
	84	'uid', 'timestamp', 'user', 'tags', 'nodes'])
	85	Way.__str__ = lambda o: 'w{0.id:d}: nodes={0.nodes!s} tags={0.tags!s}' \
	86	.format(o)
68	87
69		Node.__repr__ = lambda o : '%s(id=%r, deleted=%r, visible=%r, version=%r, changeset=%r, uid=%r, timestamp=%r, user=%r, tags=%r, location=%r)'% (o.__class__.__name__, o.id, o.deleted, o.visible, o.version, o.changeset, o.uid, o.timestamp, o.user, o.tags, o.location)
70		Node.__str__ = lambda n : 'n%d: location=%s tags=%s' \
71		% (n.id, n.location, _str_ellipse(n.tags))
	88	Relation.__repr__ = _make_repr(['id', 'deleted', 'visible', 'version', 'changeset',
	89	'uid', 'timestamp', 'user', 'tags', 'members'])
	90	Relation.__str__ = lambda o: 'r{0.id:d}: members={0.members!s}, tags={0.tags!s}' \
	91	.format(o)
72	92
73		Way.__repr__ = lambda o : '%s(id=%r, deleted=%r, visible=%r, version=%r, changeset=%r, uid=%r, timestamp=%r, user=%r, tags=%r, nodes=%r)'% (o.__class__.__name__, o.id, o.deleted, o.visible, o.version, o.changeset, o.uid, o.timestamp, o.user, o.tags, o.nodes)
74		Way.__str__ = lambda o : 'w%d: nodes=%s tags=%s' \
75		% (o.id, _str_ellipse(o.nodes), _str_ellipse(o.tags))
76
77		Relation.__repr__ = lambda o : '%s(id=%r, deleted=%r, visible=%r, version=%r, changeset=%r, uid=%r, timestamp=%r, user=%r, tags=%r, members=%r)'% (o.__class__.__name__, o.id, o.deleted, o.visible, o.version, o.changeset, o.uid, o.timestamp, o.user, o.tags, o.members)
78		Relation.__str__ = lambda o : 'r%d: members=%s, tags=%s' \
79		% (o.id, _str_ellipse(o.members), _str_ellipse(o.tags))
80
81		Changeset.__repr__ = lambda o : '%s(id=%r, uid=%r, created_at=%r, closed_at=%r, open=%r, num_changes=%r, bounds=%r, user=%r, tags=%s)' %(o.__class__.__name__, o.id, o.uid, o.created_at, o.closed_at, o.open, o.num_changes, o.bounds, o.user, o.tags)
82		Changeset.__str__ = lambda o : 'c%d: closed_at=%s, bounds=%s, tags=%s' \
83		% (o.id, o.closed_at, o.bounds, _str_ellipse(o.tags))
	93	Changeset.__repr__ = _make_repr(['id', 'uid', 'created_at', 'closed_at', 'open',
	94	'num_changes', 'bounds', 'user', 'tags'])
	95	Changeset.__str__ = lambda o: 'c{0.id:d}: closed_at={0.closed_at!s}, bounds={0.bounds!s}, tags={0.tags!s}' \
	96	.format(o)

+1

-3

src/osmium/osm/mutable.py less more

9	9	"""
10	10
11	11	def __init__(self, base=None, id=None, version=None, visible=None, changeset=None,
12		timestamp=None, uid=None, tags=None, user=None):
	12	timestamp=None, uid=None, tags=None, user=None):
13	13	if base is None:
14	14	self.id = id
15	15	self.version = version

72	72	self.members = members
73	73	else:
74	74	self.members = members if members is not None else base.members
75
76

+22

-26

src/osmium/replication/server.py less more

0	0	""" Helper functions to communicate with replication servers.
1	1	"""
2	2
3		try:
4		import urllib.request as urlrequest
5		except ImportError:
6		import urllib2 as urlrequest
7		try:
8		import urllib.error as urlerror
9		except ImportError:
10		import urllib2 as urlerror
	3	import urllib.request as urlrequest
11	4	import datetime as dt
12	5	from collections import namedtuple
13	6	from math import ceil

17	10
18	11	import logging
19	12
20		log = logging.getLogger('pyosmium')
21		log.addHandler(logging.NullHandler())
	13	LOG = logging.getLogger('pyosmium')
	14	LOG.addHandler(logging.NullHandler())
22	15
23	16	OsmosisState = namedtuple('OsmosisState', ['sequence', 'timestamp'])
24	17	DownloadResult = namedtuple('DownloadResult', ['id', 'reader', 'newest'])
25	18
26		class ReplicationServer(object):
	19	class ReplicationServer:
27	20	""" Represents a server that publishes replication data. Replication
28	21	change files allow to keep local OSM data up-to-date without downloading
29	22	the full dataset again.

87	80	try:
88	81	diffdata = self.get_diff_block(current_id)
89	82	except:
90		log.debug("Error during diff download. Bailing out.")
	83	LOG.debug("Error during diff download. Bailing out.")
91	84	diffdata = ''
92	85	if len(diffdata) == 0:
93	86	if start_id == current_id:

95	88	break
96	89
97	90	left_size -= rd.add_buffer(diffdata, self.diff_type)
98		log.debug("Downloaded change %d. (%d kB available in download buffer)"
99		% (current_id, left_size / 1024))
	91	LOG.debug("Downloaded change %d. (%d kB available in download buffer)",
	92	current_id, left_size / 1024)
100	93	current_id += 1
101	94
102	95	return DownloadResult(current_id - 1, rd, newest.sequence)

136	129	return diffs.id
137	130
138	131	def apply_diffs_to_file(self, infile, outfile, start_id, max_size=1024,
139		set_replication_header=True, extra_headers={},
	132	set_replication_header=True, extra_headers=None,
140	133	outformat=None):
141	134	""" Download diffs starting with sequence id `start_id`, merge them
142	135	with the data from the OSM file named `infile` and write the result

177	170	h.set("osmosis_replication_sequence_number", str(diffs.id))
178	171	info = self.get_state_info(diffs.id)
179	172	h.set("osmosis_replication_timestamp", info.timestamp.strftime("%Y-%m-%dT%H:%M:%SZ"))
180		for k,v in extra_headers.items():
181		h.set(k, v)
	173	if extra_headers is not None:
	174	for k, v in extra_headers.items():
	175	h.set(k, v)
182	176
183	177	if outformat is None:
184	178	of = oio.File(outfile)
185	179	else:
186	180	of = oio.File(outfile, outformat)
187	181
	182	of.has_multiple_object_versions = has_history
188	183	writer = oio.Writer(of, h)
189	184
190		log.debug("Merging changes into OSM file.")
	185	LOG.debug("Merging changes into OSM file.")
191	186
192	187	diffs.reader.apply_to_reader(reader, writer, has_history)
193	188

219	214	lower = None
220	215	lowerid = 0
221	216	while lower is None:
222		log.debug("Trying with Id %s" % lowerid)
	217	LOG.debug("Trying with Id %s", lowerid)
223	218	lower = self.get_state_info(lowerid)
224	219
225	220	if lower is not None and lower.timestamp >= timestamp:

286	281	returns `None`. `retries` sets the number of times the download
287	282	is retried when pyosmium detects a truncated state file.
288	283	"""
289		for retry in range(retries + 1):
	284	for _ in range(retries + 1):
290	285	try:
291	286	response = self.open_url(self.make_request(self.get_state_url(seq)))
292	287	except Exception as err:
293		log.debug("Loading state info {} failed with: {}".format(seq, str(err)))
	288	LOG.debug("Loading state info %s failed with: %s", seq, str(err))
294	289	return None
295	290
296	291	ts = None

340	335	if seq is None:
341	336	return self.baseurl + '/state.txt'
342	337
343		return '%s/%03i/%03i/%03i.state.txt' % (self.baseurl,
344		seq / 1000000, (seq % 1000000) / 1000, seq % 1000)
	338	return '%s/%03i/%03i/%03i.state.txt' % \
	339	(self.baseurl, seq / 1000000, (seq % 1000000) / 1000, seq % 1000)
345	340
346	341
347	342	def get_diff_url(self, seq):
348	343	""" Returns the URL to the diff file for the given sequence id.
349	344	"""
350		return '%s/%03i/%03i/%03i.%s' % (self.baseurl,
351		seq / 1000000, (seq % 1000000) / 1000, seq % 1000,
352		self.diff_type)
	345	return '%s/%03i/%03i/%03i.%s' % \
	346	(self.baseurl,
	347	seq / 1000000, (seq % 1000000) / 1000, seq % 1000,
	348	self.diff_type)

+9

-9

src/osmium/replication/utils.py less more

5	5	from osmium.io import Reader as oreader
6	6	from osmium.osm import NOTHING
7	7
8		log = logging.getLogger('pyosmium')
	8	LOG = logging.getLogger('pyosmium')
9	9
10	10	ReplicationHeader = namedtuple('ReplicationHeader',
11		['url', 'sequence', 'timestamp'])
	11	['url', 'sequence', 'timestamp'])
12	12
13	13	def get_replication_header(fname):
14	14	""" Scans the given file for an Osmosis replication header. It returns

27	27	url = h.get("osmosis_replication_base_url")
28	28
29	29	if url or ts:
30		log.debug("Replication information found in OSM file header.")
	30	LOG.debug("Replication information found in OSM file header.")
31	31
32	32	if url:
33		log.debug("Replication URL: %s" % url)
	33	LOG.debug("Replication URL: %s", url)
34	34	# the sequence ID is only considered valid, if an URL is given
35	35	seq = h.get("osmosis_replication_sequence_number")
36	36	if seq:
37		log.debug("Replication sequence: %s" % seq)
	37	LOG.debug("Replication sequence: %s", seq)
38	38	try:
39	39	seq = int(seq)
40	40	if seq < 0:
41		log.warning("Sequence id '%d' in OSM file header is negative. Ignored." % seq)
	41	LOG.warning("Sequence id '%d' in OSM file header is negative. Ignored.", seq)
42	42	seq = None
43	43	except ValueError:
44		log.warning("Sequence id '%s' in OSM file header is not a number.Ignored" % seq)
	44	LOG.warning("Sequence id '%s' in OSM file header is not a number. Ignored.", seq)
45	45	seq = None
46	46	else:
47	47	seq = None

50	50	seq = None
51	51
52	52	if ts:
53		log.debug("Replication timestamp: %s" % ts)
	53	LOG.debug("Replication timestamp: %s", ts)
54	54	try:
55	55	ts = dt.datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ")
56	56	ts = ts.replace(tzinfo=dt.timezone.utc)
57	57
58	58	except ValueError:
59		log.warning("Date in OSM file header is not in ISO8601 format (e.g. 2015-12-24T08:08Z). Ignored")
	59	LOG.warning("Date in OSM file header is not in ISO8601 format (e.g. 2015-12-24T08:08Z). Ignored.")
60	60	ts = None
61	61	else:
62	62	ts = None

+3

-3

src/osmium/version.py less more

2	2	"""
3	3
4	4	# the major version
5		pyosmium_major = '3.0'
	5	pyosmium_major = '3.1'
6	6	# current release (Pip version)
7		pyosmium_release = '3.0.1'
	7	pyosmium_release = '3.1.0'
8	8
9	9	# libosmium version shipped with the Pip release
10	10	libosmium_version = '2.15.6'
11	11	# protozero version shipped with the Pip release
12	12	protozero_version = '1.7.0'
13	13	# pybind11 version shipped with the Pip release
14		pybind11_version = '2.5.0'
	14	pybind11_version = '2.6.0'

+20

-20

test/test_replication.py less more

30	30	self.side_effect = [BytesIO(dedent(s).encode()) for s in files]
31	31
32	32	def test_get_state_url():
33		svr = rserv.ReplicationServer("http://text.org")
	33	svr = rserv.ReplicationServer("https://text.org")
34	34
35	35	data = [
36		(None, 'http://text.org/state.txt'),
37		(1, 'http://text.org/000/000/001.state.txt'),
38		(999, 'http://text.org/000/000/999.state.txt'),
39		(1000, 'http://text.org/000/001/000.state.txt'),
40		(573923, 'http://text.org/000/573/923.state.txt'),
41		(3290012, 'http://text.org/003/290/012.state.txt'),
	36	(None, 'https://text.org/state.txt'),
	37	(1, 'https://text.org/000/000/001.state.txt'),
	38	(999, 'https://text.org/000/000/999.state.txt'),
	39	(1000, 'https://text.org/000/001/000.state.txt'),
	40	(573923, 'https://text.org/000/573/923.state.txt'),
	41	(3290012, 'https://text.org/003/290/012.state.txt'),
42	42	]
43	43
44	44	for i, o in data:

68	68	txnMax=1219304113
69	69	txnActiveList=1219303583,1219304054,1219304104""")
70	70
71		res = rserv.ReplicationServer("http://test.io").get_state_info()
	71	res = rserv.ReplicationServer("https://test.io").get_state_info()
72	72
73	73	assert_is_not_none(res)
74	74	assert_equals(res.timestamp, mkdate(2017, 8, 26, 11, 4, 2))

88	88	sequenceNumber=2594669
89	89	timestamp=2017-08-26T11\:04\:02Z"""))
90	90
91		res = rserv.ReplicationServer("http://test.io").get_state_info()
	91	res = rserv.ReplicationServer("https://test.io").get_state_info()
92	92
93	93	assert_is_not_none(res)
94	94	assert_equals(res.timestamp, mkdate(2017, 8, 26, 11, 4, 2))

109	109	sequenceNumber=2594669
110	110	timestamp=2017-08-26T11\:04\:02Z"""))
111	111
112		res = rserv.ReplicationServer("http://test.io").get_state_info()
	112	res = rserv.ReplicationServer("https://test.io").get_state_info()
113	113
114	114	assert_is_not_none(res)
115	115	assert_equals(res.timestamp, mkdate(2017, 8, 26, 11, 4, 2))

130	130	sequenceNumber=2594669
131	131	timestamp=2017-08-26T11\:04\:02Z"""))
132	132
133		res = rserv.ReplicationServer("http://test.io").get_state_info()
	133	res = rserv.ReplicationServer("https://test.io").get_state_info()
134	134
135	135	assert_is_not_none(res)
136	136	assert_equals(res.timestamp, mkdate(2017, 8, 26, 11, 4, 2))

161	161	sequenceNumber=2594669
162	162	timestamp=2017-08-26T11\:04\:02Z"""))
163	163
164		res = rserv.ReplicationServer("http://test.io").get_state_info()
	164	res = rserv.ReplicationServer("https://test.io").get_state_info()
165	165
166	166	assert_is_none(res)
167	167

173	173	def test_get_state_server_timeout(mock):
174	174	mock.side_effect = URLError(reason='Mock')
175	175
176		svr = rserv.ReplicationServer("http://test.io")
	176	svr = rserv.ReplicationServer("https://test.io")
177	177	assert_is_none(svr.get_state_info())
178	178
179	179	@patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock)

186	186	w1
187	187	r1
188	188	"""))
189		svr = rserv.ReplicationServer("http://test.io", "opl")
	189	svr = rserv.ReplicationServer("https://test.io", "opl")
190	190
191	191	h = CountingHandler()
192	192	assert_equals(100, svr.apply_diffs(h, 100, 10000))

204	204	w1
205	205	r1
206	206	"""))
207		svr = rserv.ReplicationServer("http://test.io", "opl")
	207	svr = rserv.ReplicationServer("https://test.io", "opl")
208	208
209	209	h = CountingHandler()
210	210	assert_equals(100, svr.apply_diffs(h, 100, 10000, simplify=False))

221	221	w1
222	222	r1
223	223	"""))
224		svr = rserv.ReplicationServer("http://test.io", "opl")
	224	svr = rserv.ReplicationServer("https://test.io", "opl")
225	225
226	226	h = CountingHandler()
227	227	assert_equals(100, svr.apply_diffs(h, 100, 10000, simplify=True))

236	236	n1 x10.0 y23.0
237	237	w1 Nn1,n2
238	238	"""))
239		svr = rserv.ReplicationServer("http://test.io", "opl")
	239	svr = rserv.ReplicationServer("https://test.io", "opl")
240	240
241	241	class Handler(CountingHandler):
242	242	def way(self, w):

266	266	w1
267	267	r1
268	268	"""))
269		svr = rserv.ReplicationServer("http://test.io", "opl")
	269	svr = rserv.ReplicationServer("https://test.io", "opl")
270	270
271	271	h = CountingHandler()
272	272

287	287	w1
288	288	r1
289	289	"""))
290		svr = rserv.ReplicationServer("http://test.io", "opl")
	290	svr = rserv.ReplicationServer("https://test.io", "opl")
291	291
292	292	h = CountingHandler()
293	293	diffs = svr.collect_diffs(100, 100000)

305	305	n1 x10.0 y23.0
306	306	w1 Nn1,n2
307	307	"""))
308		svr = rserv.ReplicationServer("http://test.io", "opl")
	308	svr = rserv.ReplicationServer("https://test.io", "opl")
309	309
310	310	class Handler(CountingHandler):
311	311	def way(self, w):

+7

-6

tools/pyosmium-get-changes less more

26	26
27	27	from argparse import ArgumentParser, RawDescriptionHelpFormatter, ArgumentTypeError
28	28	import datetime as dt
29		from sys import version_info as python_version
30	29	import socket
31	30	from osmium.replication import server as rserv
32	31	from osmium.replication import newest_change_from_file
33	32	from osmium.replication.utils import get_replication_header
	33	from osmium.version import pyosmium_release
34	34	from osmium import SimpleHandler, WriteHandler
35	35
36	36	try:

84	84	def from_date(datestr):
85	85	try:
86	86	date = dt.datetime.strptime(datestr, "%Y-%m-%dT%H:%M:%SZ")
87		if python_version >= (3,0):
88		date = date.replace(tzinfo=dt.timezone.utc)
	87	date = date.replace(tzinfo=dt.timezone.utc)
89	88	except ValueError:
90	89	raise ArgumentTypeError("Date needs to be in ISO8601 format (e.g. 2015-12-24T08:08:08Z).")
91	90

132	131	usage=None if from_main else 'pyosmium-get-changes [options]',
133	132	formatter_class=RawDescriptionHelpFormatter)
134	133	parser.add_argument('-v', dest='loglevel', action='count', default=0,
135		help='Increase verbosity')
	134	help='Increase verbosity (can be used multiple times)')
136	135	parser.add_argument('-o', '--outfile', dest='outfile',
137	136	help=h("""Name of diff output file. If omitted, only the
138	137	sequence ID will be printed where updates would start."""))

165	164	ignore potential replication information in the
166	165	header and search for the newest OSM object."""))
167	166	parser.add_argument('-d', '--no-deduplicate', action='store_false', dest='simplify',
168		help='Do not deduplicate and sort diffs.')
	167	help='Do not deduplicate diffs.')
169	168	parser.add_argument('--socket-timeout', dest='socket_timeout', type=int, default=60,
170	169	help='Set timeout for file downloads.')
	170	parser.add_argument('--version', action='version', version='pyosmium ' + pyosmium_release)
171	171
172	172	return parser
173	173
174	174
175	175	def main(args):
176	176	logging.basicConfig(stream=sys.stderr,
177		format='%(levelname)s: %(message)s')
	177	format='%(asctime)s %(levelname)s: %(message)s',
	178	datefmt='%Y-%m-%d %H:%M:%S')
178	179
179	180	options = get_arg_parser(from_main=True).parse_args(args)
180	181

+27

-17

tools/pyosmium-up-to-date less more

14	14	subsequent calls to pyosmium-up-to-date will continue the updates from the same
15	15	server exactly where they have left of.
16	16
	17	This program can update normal OSM data files as well as OSM history files.
	18	It detects automatically on what type of file it is called.
	19
17	20	The program returns 0, if updates have been successfully applied up to
18	21	the newest data. It returns 1, if some updates have been applied but
19	22	there is still data available on the server (either because the size

34	37
35	38	from argparse import ArgumentParser, RawDescriptionHelpFormatter
36	39	import datetime as dt
37		from sys import version_info as python_version
38	40	from osmium.replication import server as rserv
39	41	from osmium.replication.utils import get_replication_header
40	42	from osmium.replication import newest_change_from_file

93	95
94	96	if not options.force_update:
95	97	cmpdate = dt.datetime.utcnow() - dt.timedelta(days=90)
96		if python_version >= (3,0):
97		cmpdate = cmpdate.replace(tzinfo=dt.timezone.utc)
	98	cmpdate = cmpdate.replace(tzinfo=dt.timezone.utc)
98	99	if ts < cmpdate:
99	100	log.error(
100	101	"""The OSM file is more than 3 months old. You should download a

115	116	else:
116	117	ofname = outfile
117	118
118		extra_headers = { 'generator' : 'pyosmium-up-to-date/' + pyosmium_release }
119		outseqs = svr.apply_diffs_to_file(infile, ofname, startseq,
120		max_size=options.outsize*1024,
121		extra_headers=extra_headers,
122		outformat=options.outformat)
123
124		if outseqs is None:
125		log.info("No new updates found.")
126		return 3
127
128		if outfile is None:
129		os.rename(ofname, infile)
	119	try:
	120	extra_headers = { 'generator' : 'pyosmium-up-to-date/' + pyosmium_release }
	121	outseqs = svr.apply_diffs_to_file(infile, ofname, startseq,
	122	max_size=options.outsize*1024,
	123	extra_headers=extra_headers,
	124	outformat=options.outformat)
	125
	126	if outseqs is None:
	127	log.info("No new updates found.")
	128	return 3
	129
	130	if outfile is None:
	131	os.rename(ofname, infile)
	132	finally:
	133	if outfile is None:
	134	try:
	135	os.remove(ofname)
	136	except FileNotFoundError:
	137	pass
130	138
131	139	log.info("Downloaded until %d. Server has data available until %d." % outseqs)
132	140

176	184	usage=None if from_main else 'pyosmium-up-to-date [options] <osm file>',
177	185	formatter_class=RawDescriptionHelpFormatter)
178	186	parser.add_argument('-v', dest='loglevel', action='count', default=0,
179		help='Increase verbosity')
	187	help='Increase verbosity (can be used multiple times)')
180	188	parser.add_argument('infile', metavar='<osm file>', help="OSM file to update")
181	189	parser.add_argument('-o', '--outfile', dest='outfile',
182	190	help=h("""Name output of file. If missing, the input file

208	216	received cookies will be written to.""")
209	217	parser.add_argument('--socket-timeout', dest='socket_timeout', type=int, default=60,
210	218	help='Set timeout for file downloads.')
	219	parser.add_argument('--version', action='version', version='pyosmium ' + pyosmium_release)
211	220
212	221	return parser
213	222

216	225
217	226	if __name__ == '__main__':
218	227	logging.basicConfig(stream=sys.stderr,
219		format='%(asctime)s %(levelname)s: %(message)s')
	228	format='%(asctime)s %(levelname)s: %(message)s',
	229	datefmt='%Y-%m-%d %H:%M:%S')
220	230
221	231	options = get_arg_parser(from_main=True).parse_args()
222	232	log.setLevel(max(3 - options.loglevel, 0) * 10)