Commit 68d8d257532ede021d91b69e2def5527b443b318 - mapnik

Merge tag 'upstream/3.0.8+ds' Upstream version 3.0.8+ds Bas Couwenberg 8 years ago

50 changed file(s) with 2612 addition(s) and 1230 deletion(s). Raw diff Collapse all Expand all

-1

.travis.yml less more

71	71	./configure;
72	72	fi
73	73	- make
74		- make test
	74	- make test \|\| TEST_RESULT=$?
75	75	- if [[ ${COVERAGE} == true ]]; then
76	76	./mason_packages/.link/bin/cpp-coveralls --build-root . --gcov-options '\-lp' --exclude mason_packages --exclude .sconf_temp --exclude benchmark --exclude deps --exclude scons --exclude test --exclude demo --exclude docs --exclude fonts --exclude utils > /dev/null;
77	77	fi
78	78	- if [[ ${COVERAGE} != true ]]; then
79	79	make bench;
80	80	fi
	81	- if [[ ${TEST_RESULT} != 0 ]]; then exit $TEST_RESULT ; fi;
81	82	- if [[ ${MASON_PUBLISH} == true ]]; then
82	83	./mason_latest.sh build;
83	84	./mason_latest.sh link;

+17

-0

CHANGELOG.md less more

4	4	Developers: Please commit along with changes.
5	5
6	6	For a complete change history, see the git log.
	7
	8	## 3.0.8
	9
	10	Released: October 23, 2015
	11
	12	(Packaged from 2d15567)
	13
	14	#### Summary
	15
	16	- Renamed `SHAPE_MEMORY_MAPPED_FILE` define to `MAPNIK_MEMORY_MAPPED_FILE`. Pass `./configure MEMORY_MAPPED_FILE=True\|False` to request
	17	support for memory mapped files across Mapnik plugins (currently shape, csv, and geojson).
	18	- Unified `mapnik-index` utility supporing GeoJSON and CSV formats
	19	- Increased unit test coverage for GeoJSON and CSV plugins
	20	- shape.input - refactor to support *.shx and improve handling various bogus shapefiles
	21	- geojson.input - make JSON parser stricter + support single Feature/Geometry as well as FeatureCollection
	22	- maintain 'FT_LOAD_NO_HINTING' + support >= harfbuzz 1.0.5
	23	- geojson.input - implement on-disk-index support
7	24
8	25	## 3.0.7
9	26

-3

SConstruct less more

388	388	EnumVariable('PLUGIN_LINKING', "Set plugin linking with libmapnik", 'shared', ['shared','static']),
389	389
390	390	# Other variables
391		BoolVariable('SHAPE_MEMORY_MAPPED_FILE', 'Utilize memory-mapped files in Shapefile Plugin (higher memory usage, better performance)', 'True'),
	391	BoolVariable('MEMORY_MAPPED_FILE', 'Utilize memory-mapped files in Shapefile Plugin (higher memory usage, better performance)', 'True'),
392	392	('SYSTEM_FONTS','Provide location for python bindings to register fonts (if provided then the bundled DejaVu fonts are not installed)',''),
393	393	('LIB_DIR_NAME','Name to use for the subfolder beside libmapnik where fonts and plugins are installed','mapnik'),
394	394	PathVariable('PYTHON','Full path to Python executable used to build bindings', sys.executable),

1208	1208	thread_suffix = ''
1209	1209	env.Append(LIBS = 'pthread')
1210	1210
1211		if env['SHAPE_MEMORY_MAPPED_FILE']:
1212		env.Append(CPPDEFINES = '-DSHAPE_MEMORY_MAPPED_FILE')
	1211	if env['MEMORY_MAPPED_FILE']:
	1212	env.Append(CPPDEFINES = '-DMAPNIK_MEMORY_MAPPED_FILE')
1213	1213
1214	1214	# allow for mac osx /usr/lib/libicucore.dylib compatibility
1215	1215	# requires custom supplied headers since Apple does not include them

-4

demo/viewer/mainwindow.cpp less more

195	195	mapnik::auto_cpu_timer t(std::clog, "loading map took: ");
196	196	mapnik::load_map(*map,filename.toStdString());
197	197	}
198		//catch (mapnik::config_error & ex)
199		//{
200		// std::cout << ex.what() << "\n";
201		//}
	198	catch (std::exception const& ex)
	199	{
	200	std::cout << ex.what() << "\n";
	201	}
202	202	catch (...)
203	203	{
204	204	std::cerr << "Exception caught in load_map\n";

-1

demo/viewer/mapwidget.cpp less more

510	510	//{
511	511	// std::cerr << ex.what() << std::endl;
512	512	//}
513		catch (const std::exception & ex)
	513	catch (std::exception const& ex)
514	514	{
515	515	std::cerr << "exception: " << ex.what() << std::endl;
516	516	}

-15

include/mapnik/attribute_descriptor.hpp less more

47	47	type_(type),
48	48	size_(size),
49	49	precision_(precision),
50		primary_key_(primary_key) {}
	50	primary_key_(primary_key) {}
51	51
52	52	attribute_descriptor(attribute_descriptor const& other)
53	53	: name_(other.name_),

56	56	precision_(other.precision_),
57	57	primary_key_(other.primary_key_) {}
58	58
59		attribute_descriptor& operator=(attribute_descriptor const& other)
	59	attribute_descriptor& operator=(attribute_descriptor rhs)
60	60	{
61		if (this == &other)
62		{
63		return *this;
64		}
65		else
66		{
67		name_=other.name_;
68		type_=other.type_;
69		size_=other.size_;
70		precision_=other.precision_;
71		primary_key_=other.primary_key_;
72		return *this;
73		}
	61	using std::swap;
	62	std::swap(name_, rhs.name_);
	63	std::swap(type_, rhs.type_);
	64	std::swap(size_, rhs.size_);
	65	std::swap(precision_, rhs.precision_);
	66	std::swap(primary_key_, rhs.primary_key_);
	67	return *this;
74	68	}
75	69
76	70	std::string const& get_name() const

-2

include/mapnik/feature.hpp less more

108	108	raster_() {}
109	109
110	110	inline mapnik::value_integer id() const { return id_;}
111
112	111	inline void set_id(mapnik::value_integer id) { id_ = id;}
113
114	112	template <typename T>
115	113	inline void put(context_type::key_type const& key, T const& val)
116	114	{

+13

-7

include/mapnik/feature_layer_desc.hpp less more

29	29	// stl
30	30	#include <iosfwd>
31	31	#include <vector>
	32	#include <algorithm>
32	33
33	34	namespace mapnik
34	35	{

39	40	layer_descriptor(std::string const& name, std::string const& encoding)
40	41	: name_(name),
41	42	encoding_(encoding),
42		desc_ar_(),
	43	descriptors_(),
43	44	extra_params_() {}
44	45
45	46	layer_descriptor(layer_descriptor const& other)
46	47	: name_(other.name_),
47	48	encoding_(other.encoding_),
48		desc_ar_(other.desc_ar_),
	49	descriptors_(other.descriptors_),
49	50	extra_params_(other.extra_params_) {}
50	51
51	52	void set_name(std::string const& name)

70	71
71	72	void add_descriptor(attribute_descriptor const& desc)
72	73	{
73		desc_ar_.push_back(desc);
	74	descriptors_.push_back(desc);
74	75	}
75	76
76	77	std::vector<attribute_descriptor> const& get_descriptors() const
77	78	{
78		return desc_ar_;
	79	return descriptors_;
79	80	}
80	81
81	82	std::vector<attribute_descriptor>& get_descriptors()
82	83	{
83		return desc_ar_;
	84	return descriptors_;
84	85	}
85	86
86	87	parameters const& get_extra_parameters() const

92	93	{
93	94	return extra_params_;
94	95	}
95
	96	bool has_name(std::string const& name) const
	97	{
	98	auto result = std::find_if(std::begin(descriptors_), std::end(descriptors_),
	99	[&name](attribute_descriptor const& desc) { return name == desc.get_name();});
	100	return result != std::end(descriptors_);
	101	}
96	102	private:
97	103	std::string name_;
98	104	std::string encoding_;
99		std::vector<attribute_descriptor> desc_ar_;
	105	std::vector<attribute_descriptor> descriptors_;
100	106	parameters extra_params_;
101	107	};
102	108

+22

-1

include/mapnik/json/feature_collection_grammar.hpp less more

65	65	feature_collection_grammar(mapnik::transcoder const& tr);
66	66	// grammars
67	67	feature_grammar<Iterator,FeatureType> feature_g;
68		geometry_grammar<Iterator> geometry_g;
	68	//geometry_grammar<Iterator> geometry_g;
69	69	// rules
70	70	qi::rule<Iterator, void(context_ptr const&, std::size_t&, FeatureCallback&), space_type> start; // START
71	71	qi::rule<Iterator, void(context_ptr const&, std::size_t&, FeatureCallback&), space_type> feature_collection;
72	72	qi::rule<Iterator, space_type> type;
73	73	qi::rule<Iterator, void(context_ptr const&, std::size_t&, FeatureCallback&), space_type> features;
	74	qi::rule<Iterator, qi::locals<feature_ptr,int>, void(context_ptr const& ctx, std::size_t, FeatureCallback&), space_type> feature;
	75	//qi::rule<Iterator, qi::locals<feature_ptr,int>, void(context_ptr const& ctx, std::size_t, FeatureCallback&), space_type> feature_from_geometry;
	76	// phoenix functions
	77	//phoenix::function<json::set_geometry_impl> set_geometry;
	78	phoenix::function<apply_feature_callback> on_feature;
	79	};
	80
	81	template <typename Iterator, typename FeatureType, typename FeatureCallback = default_feature_callback>
	82	struct feature_grammar_callback :
	83	qi::grammar<Iterator, void(context_ptr const&, std::size_t&, FeatureCallback &), space_type>
	84	{
	85	feature_grammar_callback(mapnik::transcoder const& tr);
	86	// grammars
	87	feature_grammar<Iterator, FeatureType> feature_g;
	88	geometry_grammar<Iterator> geometry_g;
	89	// rules
	90	qi::rule<Iterator, void(context_ptr const&, std::size_t&, FeatureCallback&), space_type> start; // START
	91	//qi::rule<Iterator, void(context_ptr const&, std::size_t&, FeatureCallback&), space_type> feature_collection;
	92	//qi::rule<Iterator, space_type> type;
	93	//qi::rule<Iterator, void(context_ptr const&, std::size_t&, FeatureCallback&), space_type> features;
74	94	qi::rule<Iterator, qi::locals<feature_ptr,int>, void(context_ptr const& ctx, std::size_t, FeatureCallback&), space_type> feature;
75	95	qi::rule<Iterator, qi::locals<feature_ptr,int>, void(context_ptr const& ctx, std::size_t, FeatureCallback&), space_type> feature_from_geometry;
76	96	// phoenix functions

78	98	phoenix::function<apply_feature_callback> on_feature;
79	99	};
80	100
	101
81	102	}}
82	103
83	104	#endif // MAPNIK_FEATURE_COLLECTION_GRAMMAR_HPP

+58

-5

include/mapnik/json/feature_collection_grammar_impl.hpp less more

37	37	{
38	38	qi::lit_type lit;
39	39	qi::eps_type eps;
40		qi::_1_type _1;
	40	//qi::_1_type _1;
41	41	qi::_2_type _2;
42	42	qi::_3_type _3;
43	43	qi::_4_type _4;

49	49	using phoenix::new_;
50	50	using phoenix::val;
51	51
52		start = feature_collection(_r1, _r2, _r3) \| feature_from_geometry(_r1, _r2, _r3) \| feature(_r1, _r2, _r3)
	52	start = /feature_from_geometry(_r1, _r2, _r3) \| feature(_r1, _r2, _r3) \| /feature_collection(_r1, _r2, _r3)
53	53	;
54	54
55	55	feature_collection = lit('{') >> (type \| features(_r1, _r2, _r3) \| feature_g.json_.key_value) % lit(',') >> lit('}')

69	69	>> feature_g(*_a)[on_feature(_r3,_a)]
70	70	;
71	71
	72	//feature_from_geometry =
	73	// eps[_a = phoenix::construct<mapnik::feature_ptr>(new_<mapnik::feature_impl>(_r1, _r2))]
	74	// >> geometry_g[set_geometry(*_a, _1)] [on_feature(_r3, _a)]
	75	// ;
	76
	77	start.name("start");
	78	type.name("type");
	79	features.name("features");
	80	feature.name("feature");
	81	//feature_from_geometry.name("feature-from-geometry");
	82	feature_g.name("feature-grammar");
	83	//geometry_g.name("geometry-grammar");
	84
	85	qi::on_error<qi::fail>
	86	(
	87	start
	88	, std::clog
	89	<< phoenix::val("Error parsing GeoJSON ")
	90	<< _4
	91	<< phoenix::val(" here: \"")
	92	<< construct<std::string>(_3, _2)
	93	<< phoenix::val('\"')
	94	<< std::endl
	95	);
	96	}
	97
	98	//
	99
	100
	101	template <typename Iterator, typename FeatureType, typename FeatureCallback>
	102	feature_grammar_callback<Iterator,FeatureType, FeatureCallback>::feature_grammar_callback(mapnik::transcoder const& tr)
	103	: feature_grammar_callback::base_type(start,"start"),
	104	feature_g(tr)
	105	{
	106	qi::lit_type lit;
	107	qi::eps_type eps;
	108	qi::_1_type _1;
	109	qi::_2_type _2;
	110	qi::_3_type _3;
	111	qi::_4_type _4;
	112	qi::_a_type _a;
	113	qi::_r1_type _r1;
	114	qi::_r2_type _r2;
	115	qi::_r3_type _r3;
	116	using phoenix::construct;
	117	using phoenix::new_;
	118	using phoenix::val;
	119
	120	start = feature_from_geometry(_r1, _r2, _r3) \| feature(_r1, _r2, _r3)
	121	;
	122
	123	feature = eps[_a = phoenix::construct<mapnik::feature_ptr>(new_<mapnik::feature_impl>(_r1, _r2))]
	124	>> feature_g(*_a)[on_feature(_r3,_a)]
	125	;
	126
72	127	feature_from_geometry =
73	128	eps[_a = phoenix::construct<mapnik::feature_ptr>(new_<mapnik::feature_impl>(_r1, _r2))]
74	129	>> geometry_g[set_geometry(*_a, _1)] [on_feature(_r3, _a)]
75	130	;
76	131
77	132	start.name("start");
78		type.name("type");
79		features.name("features");
80	133	feature.name("feature");
81	134	feature_from_geometry.name("feature-from-geometry");
82	135	feature_g.name("feature-grammar");

84	137
85	138	qi::on_error<qi::fail>
86	139	(
87		feature_collection
	140	start
88	141	, std::clog
89	142	<< phoenix::val("Error parsing GeoJSON ")
90	143	<< _4

-0

include/mapnik/text/harfbuzz_shaper.hpp less more

100	100	hb_buffer_set_direction(buffer.get(), (text_item.dir == UBIDI_RTL)?HB_DIRECTION_RTL:HB_DIRECTION_LTR);
101	101	hb_buffer_set_script(buffer.get(), _icu_script_to_script(text_item.script));
102	102	hb_font_t *font(hb_ft_font_create(face->get_face(), nullptr));
	103	// https://github.com/mapnik/test-data-visual/pull/25
	104	#if HB_VERSION_MAJOR > 0
	105	#if HB_VERSION_ATLEAST(1, 0 , 5)
	106	hb_ft_font_set_load_flags(font,FT_LOAD_DEFAULT \| FT_LOAD_NO_HINTING);
	107	#endif
	108	#endif
103	109	hb_shape(font, buffer.get(), ff_settings.get_features(), ff_count);
104	110	hb_font_destroy(font);
105	111

-1

include/mapnik/version.hpp less more

24	24
25	25	#define MAPNIK_MAJOR_VERSION 3
26	26	#define MAPNIK_MINOR_VERSION 0
27		#define MAPNIK_PATCH_VERSION 7
	27	#define MAPNIK_PATCH_VERSION 8
28	28
29	29	#define MAPNIK_VERSION (MAPNIK_MAJOR_VERSION100000) + (MAPNIK_MINOR_VERSION100) + (MAPNIK_PATCH_VERSION)
30	30

-1

localize.sh less more

6	6	export LD_LIBRARY_PATH="${CURRENT_DIR}/src/":${LD_LIBRARY_PATH}
7	7	fi
8	8
9		export PATH=$(pwd)/utils/nik2img/:${PATH}
	9	export PATH=$(pwd)/utils/mapnik-render/:${PATH}
	10	export PATH=$(pwd)/utils/mapnik-index/:${PATH}
10	11	export PATH=$(pwd)/utils/mapnik-config/:${PATH}
11	12
12	13	# mapnik-settings.env is an optional file to store

+17

-26

plugins/input/csv/csv_datasource.cpp less more

40	40	#include <mapnik/util/fs.hpp>
41	41	#include <mapnik/util/spatial_index.hpp>
42	42	#include <mapnik/geom_util.hpp>
43		#ifdef CSV_MEMORY_MAPPED_FILE
	43	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
44	44	#pragma GCC diagnostic push
45	45	#pragma GCC diagnostic ignored "-Wshadow"
46	46	#pragma GCC diagnostic ignored "-Wsign-conversion"

124	124	}
125	125	else
126	126	{
127		#if defined (CSV_MEMORY_MAPPED_FILE)
	127	#if defined (MAPNIK_MEMORY_MAPPED_FILE)
128	128	using file_source_type = boost::interprocess::ibufferstream;
129	129	file_source_type in;
130	130	mapnik::mapped_region_ptr mapped_region;

194	194	<< "' quote: '" << quote_ << "'";
195	195	stream.seekg(0, std::ios::beg);
196	196
197		int line_number = 1;
	197	int line_number = 0;
198	198	if (!manual_headers_.empty())
199	199	{
200	200	std::size_t index = 0;

229	229	s << "CSV Plugin: expected a column header at line ";
230	230	s << line_number << ", column " << index;
231	231	s << " - ensure this row contains valid header fields: '";
232		s << csv_line << "'\n";
	232	s << csv_line;
233	233	throw mapnik::datasource_exception(s.str());
234	234	}
235	235	else

260	260	}
261	261	}
262	262
263		if (locator_.type == detail::geometry_column_locator::UNKNOWN)
264		{
265		throw mapnik::datasource_exception("CSV Plugin: could not detect column headers with the name of wkt, geojson, x/y, or "
266		"latitude/longitude - this is required for reading geometry data");
	263	std::size_t num_headers = headers_.size();
	264	if (!detail::valid(locator_, num_headers))
	265	{
	266	std::string str("CSV Plugin: could not detect column(s) with the name(s) of wkt, geojson, x/y, or ");
	267	str += "latitude/longitude in:\n";
	268	str += csv_line;
	269	str += "\n - this is required for reading geometry data";
	270	throw mapnik::datasource_exception(str);
267	271	}
268	272
269	273	mapnik::value_integer feature_count = 0;
270	274	bool extent_started = false;
271	275
272		std::size_t num_headers = headers_.size();
273	276	std::for_each(headers_.begin(), headers_.end(),
274	277	[ & ](std::string const& header){ ctx_->push(header); });
275	278

293	296	std::vector<item_type> boxes;
294	297	while (is_first_row \|\| csv_utils::getline_csv(stream, csv_line, newline, quote_))
295	298	{
296
297		if ((row_limit_ > 0) && (line_number++ > row_limit_))
	299	++line_number;
	300	if ((row_limit_ > 0) && (line_number > row_limit_))
298	301	{
299	302	MAPNIK_LOG_DEBUG(csv) << "csv_datasource: row limit hit, exiting at feature: " << feature_count;
300	303	break;

325	328	std::ostringstream s;
326	329	s << "CSV Plugin: # of columns("
327	330	<< num_fields << ") > # of headers("
328		<< num_headers << ") parsed for row " << line_number << "\n";
	331	<< num_headers << ") parsed for row " << line_number;
329	332	throw mapnik::datasource_exception(s.str());
330	333	}
331	334

352	355	for (std::size_t i = 0; i < num_headers; ++i)
353	356	{
354	357	std::string const& header = headers_.at(i);
355		if (beg == end) // there are more headers than column values for this row
356		{
357		// add an empty string here to represent a missing value
358		// not using null type here since nulls are not a csv thing
359		if (feature_count == 1)
360		{
361		desc_.add_descriptor(mapnik::attribute_descriptor(header, mapnik::String));
362		}
363		// continue here instead of break so that all missing values are
364		// encoded consistenly as empty strings
365		continue;
366		}
367	358	std::string value = mapnik::util::trim_copy(*beg++);
368	359	int value_length = value.length();
369	360	if (locator_.index == i && (locator_.type == detail::geometry_column_locator::WKT

430	421	std::ostringstream s;
431	422	s << "CSV Plugin: expected geometry column: could not parse row "
432	423	<< line_number << " "
433		<< values[locator_.index] << "'";
	424	<< values.at(locator_.index) << "'";
434	425	throw mapnik::datasource_exception(s.str());
435	426	}
436	427	}

439	430	if (strict_) throw ex;
440	431	else
441	432	{
442		MAPNIK_LOG_ERROR(csv) << ex.what();
	433	MAPNIK_LOG_ERROR(csv) << ex.what() << " at line: " << line_number;
443	434	}
444	435	}
445	436	catch (std::exception const& ex)

-3

plugins/input/csv/csv_featureset.cpp less more

33	33	csv_featureset::csv_featureset(std::string const& filename, detail::geometry_column_locator const& locator, char separator, char quote,
34	34	std::vector<std::string> const& headers, mapnik::context_ptr const& ctx, array_type && index_array)
35	35	:
36		#if defined(CSV_MEMORY_MAPPED_FILE)
	36	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
37	37	//
38	38	#elif defined( _WINDOWS)
39	39	file_(_wfopen(mapnik::utf8_to_utf16(filename).c_str(), L"rb"), std::fclose),

50	50	locator_(locator),
51	51	tr_("utf8")
52	52	{
53		#if defined (CSV_MEMORY_MAPPED_FILE)
	53	#if defined (MAPNIK_MEMORY_MAPPED_FILE)
54	54	boost::optional<mapnik::mapped_region_ptr> memory =
55	55	mapnik::mapped_memory_cache::instance().find(filename, true);
56	56	if (memory)

89	89	csv_datasource::item_type const& item = *index_itr_++;
90	90	std::size_t file_offset = item.second.first;
91	91	std::size_t size = item.second.second;
92		#if defined(CSV_MEMORY_MAPPED_FILE)
	92	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
93	93	char const* start = (char const*)mapped_region_->get_address() + file_offset;
94	94	char const* end = start + size;
95	95	#else

-2

plugins/input/csv/csv_featureset.hpp less more

29	29	#include <deque>
30	30	#include <cstdio>
31	31
32		#ifdef CSV_MEMORY_MAPPED_FILE
	32	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
33	33	#pragma GCC diagnostic push
34	34	#pragma GCC diagnostic ignored "-Wshadow"
35	35	#pragma GCC diagnostic ignored "-Wsign-conversion"

55	55	mapnik::feature_ptr next();
56	56	private:
57	57	mapnik::feature_ptr parse_feature(char const* beg, char const* end);
58		#if defined (CSV_MEMORY_MAPPED_FILE)
	58	#if defined (MAPNIK_MEMORY_MAPPED_FILE)
59	59	using file_source_type = boost::interprocess::ibufferstream;
60	60	mapnik::mapped_region_ptr mapped_region_;
61	61	#else

-3

plugins/input/csv/csv_index_featureset.cpp less more

47	47	ctx_(ctx),
48	48	locator_(locator),
49	49	tr_("utf8")
50		#if defined(CSV_MEMORY_MAPPED_FILE)
	50	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
51	51	//
52	52	#elif defined( _WINDOWS)
53	53	,file_(_wfopen(mapnik::utf8_to_utf16(filename).c_str(), L"rb"), std::fclose)

56	56	#endif
57	57
58	58	{
59		#if defined (CSV_MEMORY_MAPPED_FILE)
	59	#if defined (MAPNIK_MEMORY_MAPPED_FILE)
60	60	boost::optional<mapnik::mapped_region_ptr> memory =
61	61	mapnik::mapped_memory_cache::instance().find(filename, true);
62	62	if (memory)

111	111	while( itr_ != positions_.end())
112	112	{
113	113	auto pos = *itr_++;
114		#if defined(CSV_MEMORY_MAPPED_FILE)
	114	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
115	115	char const* start = (char const*)mapped_region_->get_address() + pos.first;
116	116	char const* end = start + pos.second;
117	117	#else

-2

plugins/input/csv/csv_index_featureset.hpp less more

28	28	#include "csv_utils.hpp"
29	29	#include "csv_datasource.hpp"
30	30
31		#ifdef CSV_MEMORY_MAPPED_FILE
	31	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
32	32	#pragma GCC diagnostic push
33	33	#pragma GCC diagnostic ignored "-Wshadow"
34	34	#pragma GCC diagnostic ignored "-Wsign-conversion"

62	62	mapnik::value_integer feature_id_ = 0;
63	63	detail::geometry_column_locator const& locator_;
64	64	mapnik::transcoder tr_;
65		#if defined (CSV_MEMORY_MAPPED_FILE)
	65	#if defined (MAPNIK_MEMORY_MAPPED_FILE)
66	66	using file_source_type = boost::interprocess::ibufferstream;
67	67	mapnik::mapped_region_ptr mapped_region_;
68	68	#else

+26

-17

plugins/input/csv/csv_utils.hpp less more

31	31	#include <mapnik/util/conversions.hpp>
32	32	#include <mapnik/csv/csv_grammar.hpp>
33	33	#include <mapnik/util/trim.hpp>
	34	#include <mapnik/datasource.hpp>
34	35	// boost
35	36	#pragma GCC diagnostic push
36	37	#pragma GCC diagnostic ignored "-Wunused-parameter"

43	44	#include <cstdio>
44	45	#include <algorithm>
45	46
46		#ifndef _WINDOWS
47		#define CSV_MEMORY_MAPPED_FILE
48		#endif
49
50	47	namespace csv_utils
51	48	{
52	49

60	57	if (num_columns > 0) values.reserve(num_columns);
61	58	if (!boost::spirit::qi::phrase_parse(start, end, (line_g)(separator, quote), skipper, values))
62	59	{
63		throw std::runtime_error("Failed to parse CSV line:\n" + std::string(start, end));
	60	throw mapnik::datasource_exception("Failed to parse CSV line:\n" + std::string(start, end));
64	61	}
65	62	return values;
66	63	}

250	247	}
251	248	}
252	249
	250	static inline bool valid(geometry_column_locator const& locator, std::size_t max_size)
	251	{
	252	if (locator.type == geometry_column_locator::UNKNOWN) return false;
	253	if (locator.index >= max_size) return false;
	254	if (locator.type == geometry_column_locator::LON_LAT && locator.index2 >= max_size) return false;
	255	return true;
	256	}
	257
253	258	static inline mapnik::geometry::geometry<double> extract_geometry(std::vector<std::string> const& row, geometry_column_locator const& locator)
254	259	{
255	260	mapnik::geometry::geometry<double> geom;
256	261	if (locator.type == geometry_column_locator::WKT)
257	262	{
258		if (mapnik::from_wkt(row[locator.index], geom))
	263	auto wkt_value = row.at(locator.index);
	264	if (mapnik::from_wkt(wkt_value, geom))
259	265	{
260	266	// correct orientations ..
261	267	mapnik::geometry::correct(geom);
262	268	}
263	269	else
264	270	{
265		throw std::runtime_error("Failed to parse WKT:" + row[locator.index]);
	271	throw mapnik::datasource_exception("Failed to parse WKT: '" + wkt_value + "'");
266	272	}
267	273	}
268	274	else if (locator.type == geometry_column_locator::GEOJSON)
269	275	{
270	276
271		if (!mapnik::json::from_geojson(row[locator.index], geom))
272		{
273		throw std::runtime_error("Failed to parse GeoJSON:" + row[locator.index]);
	277	auto json_value = row.at(locator.index);
	278	if (!mapnik::json::from_geojson(json_value, geom))
	279	{
	280	throw mapnik::datasource_exception("Failed to parse GeoJSON: '" + json_value + "'");
274	281	}
275	282	}
276	283	else if (locator.type == geometry_column_locator::LON_LAT)
277	284	{
278	285	double x, y;
279		if (!mapnik::util::string2double(row[locator.index],x))
280		{
281		throw std::runtime_error("Failed to parse Longitude(Easting):" + row[locator.index]);
282		}
283		if (!mapnik::util::string2double(row[locator.index2],y))
284		{
285		throw std::runtime_error("Failed to parse Latitude(Northing):" + row[locator.index2]);
	286	auto long_value = row.at(locator.index);
	287	auto lat_value = row.at(locator.index2);
	288	if (!mapnik::util::string2double(long_value,x))
	289	{
	290	throw mapnik::datasource_exception("Failed to parse Longitude: '" + long_value + "'");
	291	}
	292	if (!mapnik::util::string2double(lat_value,y))
	293	{
	294	throw mapnik::datasource_exception("Failed to parse Latitude: '" + lat_value + "'");
286	295	}
287	296	geom = mapnik::geometry::point<double>(x,y);
288	297	}

-1

plugins/input/geojson/build.py less more

41	41	"""
42	42	%(PLUGIN_NAME)s_datasource.cpp
43	43	%(PLUGIN_NAME)s_featureset.cpp
44		large_%(PLUGIN_NAME)s_featureset.cpp
	44	%(PLUGIN_NAME)s_index_featureset.cpp
	45	%(PLUGIN_NAME)s_memory_index_featureset.cpp
	46
45	47	""" % locals()
46	48	)
47	49

+220

-45

plugins/input/geojson/geojson_datasource.cpp less more

21	21
22	22	#include "geojson_datasource.hpp"
23	23	#include "geojson_featureset.hpp"
24		#include "large_geojson_featureset.hpp"
	24	#include "geojson_index_featureset.hpp"
	25	#include "geojson_memory_index_featureset.hpp"
25	26	#include <fstream>
26	27	#include <algorithm>
27	28

56	57	#include <mapnik/geometry_adapters.hpp>
57	58	#include <mapnik/json/feature_collection_grammar.hpp>
58	59	#include <mapnik/json/extract_bounding_box_grammar_impl.hpp>
59
60		#if defined(SHAPE_MEMORY_MAPPED_FILE)
	60	#include <mapnik/util/fs.hpp>
	61	#include <mapnik/util/spatial_index.hpp>
	62	#include <mapnik/geom_util.hpp>
	63
	64	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
61	65	#pragma GCC diagnostic push
62	66	#pragma GCC diagnostic ignored "-Wshadow"
63	67	#pragma GCC diagnostic ignored "-Wsign-conversion"

135	139	filename_ = base + "/" + file;
136	140	else
137	141	filename_ = *file;
138		}
	142	has_disk_index_ = mapnik::util::exists(filename_ + ".index");
	143	}
	144
139	145	if (!inline_string_.empty())
140	146	{
141	147	char const* start = inline_string_.c_str();
142	148	char const* end = start + inline_string_.size();
143	149	parse_geojson(start, end);
144	150	}
	151	else if (has_disk_index_)
	152	{
	153	initialise_disk_index(filename_);
	154	}
145	155	else
146	156	{
147	157	cache_features_ = *params.get<mapnik::boolean_type>("cache_features", true);
148		#if !defined(SHAPE_MEMORY_MAPPED_FILE)
	158	#if !defined(MAPNIK_MEMORY_MAPPED_FILE)
149	159	mapnik::util::file file(filename_);
150	160	if (!file.open())
151	161	{

191	201	using base_iterator_type = char const*;
192	202	const mapnik::transcoder geojson_datasource_static_tr("utf8");
193	203	const mapnik::json::feature_collection_grammar<base_iterator_type,mapnik::feature_impl> geojson_datasource_static_fc_grammar(geojson_datasource_static_tr);
	204	const mapnik::json::feature_grammar_callback<base_iterator_type,mapnik::feature_impl> geojson_datasource_static_feature_callback_grammar(geojson_datasource_static_tr);
194	205	const mapnik::json::feature_grammar<base_iterator_type, mapnik::feature_impl> geojson_datasource_static_feature_grammar(geojson_datasource_static_tr);
195	206	const mapnik::json::extract_bounding_box_grammar<base_iterator_type> geojson_datasource_static_bbox_grammar;
	207	}
	208
	209	void geojson_datasource::initialise_disk_index(std::string const& filename)
	210	{
	211	// read extent
	212	using value_type = std::pair<std::size_t, std::size_t>;
	213	std::ifstream index(filename_ + ".index", std::ios::binary);
	214	if (!index) throw mapnik::datasource_exception("GeoJSON Plugin: could not open: '" + filename_ + ".index'");
	215	extent_ = mapnik::util::spatial_index<value_type,
	216	mapnik::filter_in_box,
	217	std::ifstream>::bounding_box(index);
	218	mapnik::filter_in_box filter(extent_);
	219	std::vector<value_type> positions;
	220	mapnik::util::spatial_index<value_type,
	221	mapnik::filter_in_box,
	222	std::ifstream>::query_first_n(filter, index, positions, 5);
	223
	224	mapnik::util::file file(filename_);
	225	if (!file.open()) throw mapnik::datasource_exception("GeoJSON Plugin: could not open: '" + filename_ + "'");
	226
	227	for (auto const& pos : positions)
	228	{
	229	std::fseek(file.get(), pos.first, SEEK_SET);
	230	std::vector<char> record;
	231	record.resize(pos.second);
	232	std::fread(record.data(), pos.second, 1, file.get());
	233	auto const* start = record.data();
	234	auto const* end = start + record.size();
	235	mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
	236	mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx,1));
	237	using namespace boost::spirit;
	238	standard::space_type space;
	239	if (!boost::spirit::qi::phrase_parse(start, end,
	240	(geojson_datasource_static_feature_grammar)(boost::phoenix::ref(*feature)), space)
	241	\|\| start != end)
	242	{
	243	throw std::runtime_error("Failed to parse geojson feature");
	244	}
	245	for ( auto const& kv : *feature)
	246	{
	247	auto const& name = std::get<0>(kv);
	248	if (!desc_.has_name(name))
	249	{
	250	desc_.add_descriptor(mapnik::attribute_descriptor(name,
	251	mapnik::util::apply_visitor(attr_value_converter(),
	252	std::get<1>(kv))));
	253	}
	254	}
	255	}
196	256	}
197	257
198	258	template <typename Iterator>

203	263	Iterator itr = start;
204	264	if (!boost::spirit::qi::phrase_parse(itr, end, (geojson_datasource_static_bbox_grammar)(boost::phoenix::ref(boxes)) , space))
205	265	{
206		throw mapnik::datasource_exception("GeoJSON Plugin: could not parse: '" + filename_ + "'");
207		}
208		// bulk insert initialise r-tree
209		tree_ = std::make_unique<spatial_index_type>(boxes);
210		// calculate total extent
211		for (auto const& item : boxes)
212		{
213		auto const& box = std::get<0>(item);
214		auto const& geometry_index = std::get<1>(item);
215		if (!extent_.valid())
216		{
217		extent_ = box;
218		// parse first feature to extract attributes schema.
219		// NOTE: this doesn't yield correct answer for geoJSON in general, just an indication
220		Iterator itr2 = start + geometry_index.first;
221		Iterator end2 = itr2 + geometry_index.second;
222		mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
223		mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx,1));
224		if (!boost::spirit::qi::phrase_parse(itr2, end2, (geojson_datasource_static_feature_grammar)(boost::phoenix::ref(*feature)), space))
225		{
226		throw std::runtime_error("Failed to parse geojson feature");
227		}
228		for ( auto const& kv : *feature)
229		{
230		desc_.add_descriptor(mapnik::attribute_descriptor(std::get<0>(kv),
231		mapnik::util::apply_visitor(attr_value_converter(),
232		std::get<1>(kv))));
233		}
234		}
235		else
236		{
237		extent_.expand_to_include(box);
	266	cache_features_ = true; // force caching single feature
	267	itr = start; // reset iteraror
	268	// try parsing as single Feature or single Geometry JSON
	269	mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
	270	std::size_t start_id = 1;
	271	mapnik::json::default_feature_callback callback(features_);
	272	bool result = boost::spirit::qi::phrase_parse(itr, end, (geojson_datasource_static_feature_callback_grammar)
	273	(boost::phoenix::ref(ctx),boost::phoenix::ref(start_id), boost::phoenix::ref(callback)),
	274	space);
	275	if (!result \|\| itr != end)
	276	{
	277	if (!inline_string_.empty()) throw mapnik::datasource_exception("geojson_datasource: Failed parse GeoJSON file from in-memory string");
	278	else throw mapnik::datasource_exception("geojson_datasource: Failed parse GeoJSON file '" + filename_ + "'");
	279	}
	280
	281	using values_container = std::vector< std::pair<box_type, std::pair<std::size_t, std::size_t>>>;
	282	values_container values;
	283	values.reserve(features_.size());
	284
	285	std::size_t geometry_index = 0;
	286	for (mapnik::feature_ptr const& f : features_)
	287	{
	288	mapnik::box2d<double> box = f->envelope();
	289	if (box.valid())
	290	{
	291	if (geometry_index == 0)
	292	{
	293	extent_ = box;
	294	for ( auto const& kv : *f)
	295	{
	296	desc_.add_descriptor(mapnik::attribute_descriptor(std::get<0>(kv),
	297	mapnik::util::apply_visitor(attr_value_converter(),
	298	std::get<1>(kv))));
	299	}
	300	}
	301	else
	302	{
	303	extent_.expand_to_include(box);
	304	}
	305	values.emplace_back(box, std::make_pair(geometry_index,0));
	306	}
	307	++geometry_index;
	308	}
	309	// packing algorithm
	310	tree_ = std::make_unique<spatial_index_type>(values);
	311	}
	312	else
	313	{
	314	// bulk insert initialise r-tree
	315	tree_ = std::make_unique<spatial_index_type>(boxes);
	316	// calculate total extent
	317	for (auto const& item : boxes)
	318	{
	319	auto const& box = std::get<0>(item);
	320	auto const& geometry_index = std::get<1>(item);
	321	if (!extent_.valid())
	322	{
	323	extent_ = box;
	324	// parse first feature to extract attributes schema.
	325	// NOTE: this doesn't yield correct answer for geoJSON in general, just an indication
	326	Iterator itr2 = start + geometry_index.first;
	327	Iterator end2 = itr2 + geometry_index.second;
	328	mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
	329	mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx,-1)); // temp feature
	330	if (!boost::spirit::qi::phrase_parse(itr2, end2,
	331	(geojson_datasource_static_feature_grammar)(boost::phoenix::ref(*feature)), space)
	332	\|\| itr2 != end2)
	333	{
	334	throw std::runtime_error("Failed to parse geojson feature");
	335	}
	336	for ( auto const& kv : *feature)
	337	{
	338	desc_.add_descriptor(mapnik::attribute_descriptor(std::get<0>(kv),
	339	mapnik::util::apply_visitor(attr_value_converter(),
	340	std::get<1>(kv))));
	341	}
	342	}
	343	else
	344	{
	345	extent_.expand_to_include(box);
	346	}
238	347	}
239	348	}
240	349	}

247	356	std::size_t start_id = 1;
248	357
249	358	mapnik::json::default_feature_callback callback(features_);
250
251		bool result = boost::spirit::qi::phrase_parse(start, end, (geojson_datasource_static_fc_grammar)
	359	Iterator itr = start;
	360	bool result = boost::spirit::qi::phrase_parse(itr, end, (geojson_datasource_static_fc_grammar)
252	361	(boost::phoenix::ref(ctx),boost::phoenix::ref(start_id), boost::phoenix::ref(callback)),
253	362	space);
254		if (!result)
	363	if (!result \|\| itr != end)
255	364	{
256	365	if (!inline_string_.empty()) throw mapnik::datasource_exception("geojson_datasource: Failed parse GeoJSON file from in-memory string");
257	366	else throw mapnik::datasource_exception("geojson_datasource: Failed parse GeoJSON file '" + filename_ + "'");
	367	}
	368
	369	if (features_.size() == 0)
	370	{
	371	itr = start;
	372	// try parsing as single Feature or single Geometry JSON
	373	result = boost::spirit::qi::phrase_parse(itr, end, (geojson_datasource_static_feature_callback_grammar)
	374	(boost::phoenix::ref(ctx),boost::phoenix::ref(start_id), boost::phoenix::ref(callback)),
	375	space);
	376	if (!result \|\| itr != end)
	377	{
	378	if (!inline_string_.empty()) throw mapnik::datasource_exception("geojson_datasource: Failed parse GeoJSON file from in-memory string");
	379	else throw mapnik::datasource_exception("geojson_datasource: Failed parse GeoJSON file '" + filename_ + "'");
	380	}
258	381	}
259	382
260	383	using values_container = std::vector< std::pair<box_type, std::pair<std::size_t, std::size_t>>>;

316	439	{
317	440	boost::optional<mapnik::datasource_geometry_t> result;
318	441	int multi_type = 0;
319		if (cache_features_)
	442	if (has_disk_index_)
	443	{
	444	using value_type = std::pair<std::size_t, std::size_t>;
	445	std::ifstream index(filename_ + ".index", std::ios::binary);
	446	if (!index) throw mapnik::datasource_exception("GeoJSON Plugin: could not open: '" + filename_ + ".index'");
	447	mapnik::filter_in_box filter(extent_);
	448	std::vector<value_type> positions;
	449	mapnik::util::spatial_index<value_type,
	450	mapnik::filter_in_box,
	451	std::ifstream>::query_first_n(filter, index, positions, 5);
	452
	453	mapnik::util::file file(filename_);
	454
	455	if (!file.open()) throw mapnik::datasource_exception("GeoJSON Plugin: could not open: '" + filename_ + "'");
	456
	457	for (auto const& pos : positions)
	458	{
	459	std::fseek(file.get(), pos.first, SEEK_SET);
	460	std::vector<char> record;
	461	record.resize(pos.second);
	462	std::fread(record.data(), pos.second, 1, file.get());
	463	auto const* start = record.data();
	464	auto const* end = start + record.size();
	465	mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
	466	mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx, -1)); // temp feature
	467	using namespace boost::spirit;
	468	standard::space_type space;
	469	if (!boost::spirit::qi::phrase_parse(start, end,
	470	(geojson_datasource_static_feature_grammar)(boost::phoenix::ref(*feature)), space)
	471	\|\| start != end)
	472	{
	473	throw std::runtime_error("Failed to parse geojson feature");
	474	}
	475	result = mapnik::util::to_ds_type(feature->get_geometry());
	476	if (result)
	477	{
	478	int type = static_cast<int>(*result);
	479	if (multi_type > 0 && multi_type != type)
	480	{
	481	result.reset(mapnik::datasource_geometry_t::Collection);
	482	return result;
	483	}
	484	multi_type = type;
	485	}
	486	}
	487	}
	488	else if (cache_features_)
320	489	{
321	490	unsigned num_features = features_.size();
322	491	for (unsigned i = 0; i < num_features && i < 5; ++i)

361	530
362	531	using namespace boost::spirit;
363	532	standard::space_type space;
364		mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx,1));
	533	mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx, -1)); // temp feature
365	534	if (!qi::phrase_parse(start2, end2, (geojson_datasource_static_feature_grammar)(boost::phoenix::ref(*feature)), space))
366	535	{
367	536	throw std::runtime_error("Failed to parse geojson feature");

404	573	{
405	574	return item0.second.first < item1.second.first;
406	575	});
407		return std::make_shared<large_geojson_featureset>(filename_, std::move(index_array));
408		}
409		}
	576	return std::make_shared<geojson_memory_index_featureset>(filename_, std::move(index_array));
	577	}
	578	}
	579	else if (has_disk_index_)
	580	{
	581	mapnik::filter_in_box filter(q.get_bbox());
	582	return std::make_shared<geojson_index_featureset>(filename_, filter);
	583	}
	584
410	585	}
411	586	// otherwise return an empty featureset pointer
412	587	return mapnik::featureset_ptr();

-0

plugins/input/geojson/geojson_datasource.hpp less more

97	97	void parse_geojson(Iterator start, Iterator end);
98	98	template <typename Iterator>
99	99	void initialise_index(Iterator start, Iterator end);
	100	void initialise_disk_index(std::string const& filename);
100	101	private:
101	102	mapnik::datasource::datasource_t type_;
102	103	mapnik::layer_descriptor desc_;

106	107	std::vector<mapnik::feature_ptr> features_;
107	108	std::unique_ptr<spatial_index_type> tree_;
108	109	bool cache_features_ = true;
	110	bool has_disk_index_ = false;
109	111	};
110	112
111	113

+104

-0

plugins/input/geojson/geojson_index_featureset.cpp less more

	0	/*****************************************************************************
	1	*
	2	* This file is part of Mapnik (c++ mapping toolkit)
	3	*
	4	* Copyright (C) 2015 Artem Pavlenko
	5	*
	6	* This library is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU Lesser General Public
	8	* License as published by the Free Software Foundation; either
	9	* version 2.1 of the License, or (at your option) any later version.
	10	*
	11	* This library is distributed in the hope that it will be useful,
	12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	* Lesser General Public License for more details.
	15	*
	16	* You should have received a copy of the GNU Lesser General Public
	17	* License along with this library; if not, write to the Free Software
	18	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	19	*
	20	*****************************************************************************/
	21
	22	// mapnik
	23	#include "geojson_index_featureset.hpp"
	24	#include <mapnik/feature.hpp>
	25	#include <mapnik/feature_factory.hpp>
	26	#include <mapnik/json/geometry_grammar.hpp>
	27	#include <mapnik/json/feature_grammar.hpp>
	28	#include <mapnik/util/utf_conv_win.hpp>
	29	#include <mapnik/util/spatial_index.hpp>
	30	// stl
	31	#include <string>
	32	#include <vector>
	33	#include <fstream>
	34
	35	geojson_index_featureset::geojson_index_featureset(std::string const& filename, mapnik::filter_in_box const& filter)
	36	:
	37	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
	38	//
	39	#elif defined _WINDOWS
	40	file_(_wfopen(mapnik::utf8_to_utf16(filename).c_str(), L"rb"), std::fclose),
	41	#else
	42	file_(std::fopen(filename.c_str(),"rb"), std::fclose),
	43	#endif
	44	ctx_(std::make_shared<mapnik::context_type>())
	45	{
	46
	47	#if defined (MAPNIK_MEMORY_MAPPED_FILE)
	48	boost::optional<mapnik::mapped_region_ptr> memory =
	49	mapnik::mapped_memory_cache::instance().find(filename, true);
	50	if (memory)
	51	{
	52	mapped_region_ = *memory;
	53	}
	54	else
	55	{
	56	throw std::runtime_error("could not create file mapping for " + filename);
	57	}
	58	#else
	59	if (!file_) throw std::runtime_error("Can't open " + filename);
	60	#endif
	61	std::string indexname = filename + ".index";
	62	std::ifstream index(indexname.c_str(), std::ios::binary);
	63	if (!index) throw mapnik::datasource_exception("GeoJSON Plugin: can't open index file " + indexname);
	64	mapnik::util::spatial_index<value_type,
	65	mapnik::filter_in_box,
	66	std::ifstream>::query(filter, index, positions_);
	67
	68	std::sort(positions_.begin(), positions_.end(),
	69	[](value_type const& lhs, value_type const& rhs) { return lhs.first < rhs.first;});
	70	itr_ = positions_.begin();
	71	}
	72
	73	geojson_index_featureset::~geojson_index_featureset() {}
	74
	75	mapnik::feature_ptr geojson_index_featureset::next()
	76	{
	77	while( itr_ != positions_.end())
	78	{
	79	auto pos = *itr_++;
	80	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
	81	char const* start = (char const*)mapped_region_->get_address() + pos.first;
	82	char const* end = start + pos.second;
	83	#else
	84	std::fseek(file_.get(), pos.first, SEEK_SET);
	85	std::vector<char> record;
	86	record.resize(pos.second);
	87	std::fread(record.data(), pos.second, 1, file_.get());
	88	auto const* start = record.data();
	89	auto const* end = start + record.size();
	90	#endif
	91	static const mapnik::transcoder tr("utf8");
	92	static const mapnik::json::feature_grammar<char const*, mapnik::feature_impl> grammar(tr);
	93	using namespace boost::spirit;
	94	standard::space_type space;
	95	mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx_, feature_id_++));
	96	if (!qi::phrase_parse(start, end, (grammar)(boost::phoenix::ref(*feature)), space) \|\| start != end)
	97	{
	98	throw std::runtime_error("Failed to parse geojson feature");
	99	}
	100	return feature;
	101	}
	102	return mapnik::feature_ptr();
	103	}

+65

-0

plugins/input/geojson/geojson_index_featureset.hpp less more

	0	/*****************************************************************************
	1	*
	2	* This file is part of Mapnik (c++ mapping toolkit)
	3	*
	4	* Copyright (C) 2015 Artem Pavlenko
	5	*
	6	* This library is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU Lesser General Public
	8	* License as published by the Free Software Foundation; either
	9	* version 2.1 of the License, or (at your option) any later version.
	10	*
	11	* This library is distributed in the hope that it will be useful,
	12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	* Lesser General Public License for more details.
	15	*
	16	* You should have received a copy of the GNU Lesser General Public
	17	* License along with this library; if not, write to the Free Software
	18	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	19	*
	20	*****************************************************************************/
	21
	22	#ifndef GEOJSON_INDEX_FEATURESET_HPP
	23	#define GEOJSON_INDEX_FEATURESET_HPP
	24
	25	#include "geojson_datasource.hpp"
	26	#include <mapnik/feature.hpp>
	27	#include <mapnik/geom_util.hpp>
	28
	29	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
	30	#pragma GCC diagnostic push
	31	#pragma GCC diagnostic ignored "-Wshadow"
	32	#pragma GCC diagnostic ignored "-Wsign-conversion"
	33	#include <boost/interprocess/mapped_region.hpp>
	34	#include <boost/interprocess/streams/bufferstream.hpp>
	35	#pragma GCC diagnostic pop
	36	#include <mapnik/mapped_memory_cache.hpp>
	37	#endif
	38
	39	#include <deque>
	40	#include <cstdio>
	41
	42	class geojson_index_featureset : public mapnik::Featureset
	43	{
	44	using value_type = std::pair<std::size_t, std::size_t>;
	45	public:
	46	geojson_index_featureset(std::string const& filename, mapnik::filter_in_box const& filter);
	47	virtual ~geojson_index_featureset();
	48	mapnik::feature_ptr next();
	49
	50	private:
	51	#if defined (MAPNIK_MEMORY_MAPPED_FILE)
	52	using file_source_type = boost::interprocess::ibufferstream;
	53	mapnik::mapped_region_ptr mapped_region_;
	54	#else
	55	using file_ptr = std::unique_ptr<std::FILE, int ()(std::FILE )>;
	56	file_ptr file_;
	57	#endif
	58	mapnik::value_integer feature_id_ = 1;
	59	mapnik::context_ptr ctx_;
	60	std::vector<value_type> positions_;
	61	std::vector<value_type>::iterator itr_;
	62	};
	63
	64	#endif // GEOJSON_INDEX_FEATURESE_HPP

+80

-0

plugins/input/geojson/geojson_memory_index_featureset.cpp less more

	0	/*****************************************************************************
	1	*
	2	* This file is part of Mapnik (c++ mapping toolkit)
	3	*
	4	* Copyright (C) 2015 Artem Pavlenko
	5	*
	6	* This library is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU Lesser General Public
	8	* License as published by the Free Software Foundation; either
	9	* version 2.1 of the License, or (at your option) any later version.
	10	*
	11	* This library is distributed in the hope that it will be useful,
	12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	* Lesser General Public License for more details.
	15	*
	16	* You should have received a copy of the GNU Lesser General Public
	17	* License along with this library; if not, write to the Free Software
	18	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	19	*
	20	*****************************************************************************/
	21
	22	// mapnik
	23	#include <mapnik/feature.hpp>
	24	#include <mapnik/feature_factory.hpp>
	25	#include <mapnik/json/geometry_grammar.hpp>
	26	#include <mapnik/json/feature_grammar.hpp>
	27	#include <mapnik/util/utf_conv_win.hpp>
	28	// stl
	29	#include <string>
	30	#include <vector>
	31
	32	#include "geojson_memory_index_featureset.hpp"
	33
	34	geojson_memory_index_featureset::geojson_memory_index_featureset(std::string const& filename,
	35	array_type && index_array)
	36	:
	37	#ifdef _WINDOWS
	38	file_(_wfopen(mapnik::utf8_to_utf16(filename).c_str(), L"rb"), std::fclose),
	39	#else
	40	file_(std::fopen(filename.c_str(),"rb"), std::fclose),
	41	#endif
	42	index_array_(std::move(index_array)),
	43	index_itr_(index_array_.begin()),
	44	index_end_(index_array_.end()),
	45	ctx_(std::make_shared<mapnik::context_type>())
	46	{
	47	if (!file_) throw std::runtime_error("Can't open " + filename);
	48	}
	49
	50	geojson_memory_index_featureset::~geojson_memory_index_featureset() {}
	51
	52	mapnik::feature_ptr geojson_memory_index_featureset::next()
	53	{
	54	if (index_itr_ != index_end_)
	55	{
	56	geojson_datasource::item_type const& item = *index_itr_++;
	57	std::size_t file_offset = item.second.first;
	58	std::size_t size = item.second.second;
	59	std::fseek(file_.get(), file_offset, SEEK_SET);
	60	std::vector<char> json;
	61	json.resize(size);
	62	std::fread(json.data(), size, 1, file_.get());
	63
	64	using chr_iterator_type = char const*;
	65	chr_iterator_type start = json.data();
	66	chr_iterator_type end = start + json.size();
	67	static const mapnik::transcoder tr("utf8");
	68	static const mapnik::json::feature_grammar<chr_iterator_type,mapnik::feature_impl> grammar(tr);
	69	using namespace boost::spirit;
	70	standard::space_type space;
	71	mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx_, feature_id_++));
	72	if (!qi::phrase_parse(start, end, (grammar)(boost::phoenix::ref(*feature)), space) \|\| start != end)
	73	{
	74	throw std::runtime_error("Failed to parse geojson feature");
	75	}
	76	return feature;
	77	}
	78	return mapnik::feature_ptr();
	79	}

+52

-0

plugins/input/geojson/geojson_memory_index_featureset.hpp less more

	0	/*****************************************************************************
	1	*
	2	* This file is part of Mapnik (c++ mapping toolkit)
	3	*
	4	* Copyright (C) 2015 Artem Pavlenko
	5	*
	6	* This library is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU Lesser General Public
	8	* License as published by the Free Software Foundation; either
	9	* version 2.1 of the License, or (at your option) any later version.
	10	*
	11	* This library is distributed in the hope that it will be useful,
	12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	* Lesser General Public License for more details.
	15	*
	16	* You should have received a copy of the GNU Lesser General Public
	17	* License along with this library; if not, write to the Free Software
	18	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	19	*
	20	*****************************************************************************/
	21
	22	#ifndef GEOJSON_MEMORY_INDEX_FEATURESET_HPP
	23	#define GEOJSON_MEMORY_INDEX_FEATURESET_HPP
	24
	25	#include <mapnik/feature.hpp>
	26	#include "geojson_datasource.hpp"
	27
	28	#include <deque>
	29	#include <cstdio>
	30
	31	class geojson_memory_index_featureset : public mapnik::Featureset
	32	{
	33	public:
	34	using array_type = std::deque<geojson_datasource::item_type>;
	35	using file_ptr = std::unique_ptr<std::FILE, int ()(std::FILE )>;
	36
	37	geojson_memory_index_featureset(std::string const& filename,
	38	array_type && index_array);
	39	virtual ~geojson_memory_index_featureset();
	40	mapnik::feature_ptr next();
	41
	42	private:
	43	file_ptr file_;
	44	mapnik::value_integer feature_id_ = 1;
	45	const array_type index_array_;
	46	array_type::const_iterator index_itr_;
	47	array_type::const_iterator index_end_;
	48	mapnik::context_ptr ctx_;
	49	};
	50
	51	#endif // GEOJSON_MEMORY_INDEX_FEATURESET_HPP

-81

~~plugins/input/geojson/large_geojson_featureset.cpp~~ less more

0		/*****************************************************************************
1		*
2		* This file is part of Mapnik (c++ mapping toolkit)
3		*
4		* Copyright (C) 2015 Artem Pavlenko
5		*
6		* This library is free software; you can redistribute it and/or
7		* modify it under the terms of the GNU Lesser General Public
8		* License as published by the Free Software Foundation; either
9		* version 2.1 of the License, or (at your option) any later version.
10		*
11		* This library is distributed in the hope that it will be useful,
12		* but WITHOUT ANY WARRANTY; without even the implied warranty of
13		* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14		* Lesser General Public License for more details.
15		*
16		* You should have received a copy of the GNU Lesser General Public
17		* License along with this library; if not, write to the Free Software
18		* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
19		*
20		*****************************************************************************/
21
22		// mapnik
23		#include <mapnik/feature.hpp>
24		#include <mapnik/feature_factory.hpp>
25		#include <mapnik/json/geometry_grammar.hpp>
26		#include <mapnik/json/feature_grammar.hpp>
27		#include <mapnik/util/utf_conv_win.hpp>
28		// stl
29		#include <string>
30		#include <vector>
31
32		#include "large_geojson_featureset.hpp"
33
34		large_geojson_featureset::large_geojson_featureset(std::string const& filename,
35		array_type && index_array)
36		:
37		#ifdef _WINDOWS
38		file_(_wfopen(mapnik::utf8_to_utf16(filename).c_str(), L"rb"), std::fclose),
39		#else
40		file_(std::fopen(filename.c_str(),"rb"), std::fclose),
41		#endif
42		index_array_(std::move(index_array)),
43		index_itr_(index_array_.begin()),
44		index_end_(index_array_.end()),
45		ctx_(std::make_shared<mapnik::context_type>())
46		{
47		if (!file_) throw std::runtime_error("Can't open " + filename);
48		}
49
50		large_geojson_featureset::~large_geojson_featureset() {}
51
52		mapnik::feature_ptr large_geojson_featureset::next()
53		{
54		if (index_itr_ != index_end_)
55		{
56		geojson_datasource::item_type const& item = *index_itr_++;
57		std::size_t file_offset = item.second.first;
58		std::size_t size = item.second.second;
59		std::fseek(file_.get(), file_offset, SEEK_SET);
60		std::vector<char> json;
61		json.resize(size);
62		std::fread(json.data(), size, 1, file_.get());
63
64		using chr_iterator_type = char const*;
65		chr_iterator_type start = json.data();
66		chr_iterator_type end = start + json.size();
67
68		static const mapnik::transcoder tr("utf8");
69		static const mapnik::json::feature_grammar<chr_iterator_type,mapnik::feature_impl> grammar(tr);
70		using namespace boost::spirit;
71		standard::space_type space;
72		mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx_,1));
73		if (!qi::phrase_parse(start, end, (grammar)(boost::phoenix::ref(*feature)), space))
74		{
75		throw std::runtime_error("Failed to parse geojson feature");
76		}
77		return feature;
78		}
79		return mapnik::feature_ptr();
80		}

-52

~~plugins/input/geojson/large_geojson_featureset.hpp~~ less more

0		/*****************************************************************************
1		*
2		* This file is part of Mapnik (c++ mapping toolkit)
3		*
4		* Copyright (C) 2015 Artem Pavlenko
5		*
6		* This library is free software; you can redistribute it and/or
7		* modify it under the terms of the GNU Lesser General Public
8		* License as published by the Free Software Foundation; either
9		* version 2.1 of the License, or (at your option) any later version.
10		*
11		* This library is distributed in the hope that it will be useful,
12		* but WITHOUT ANY WARRANTY; without even the implied warranty of
13		* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14		* Lesser General Public License for more details.
15		*
16		* You should have received a copy of the GNU Lesser General Public
17		* License along with this library; if not, write to the Free Software
18		* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
19		*
20		*****************************************************************************/
21
22		#ifndef LARGE_GEOJSON_FEATURESET_HPP
23		#define LARGE_GEOJSON_FEATURESET_HPP
24
25		#include <mapnik/feature.hpp>
26		#include "geojson_datasource.hpp"
27
28		#include <deque>
29		#include <cstdio>
30
31		class large_geojson_featureset : public mapnik::Featureset
32		{
33		public:
34		using array_type = std::deque<geojson_datasource::item_type>;
35		using file_ptr = std::unique_ptr<std::FILE, int ()(std::FILE )>;
36
37		large_geojson_featureset(std::string const& filename,
38		array_type && index_array);
39		virtual ~large_geojson_featureset();
40		mapnik::feature_ptr next();
41
42		private:
43		file_ptr file_;
44
45		const array_type index_array_;
46		array_type::const_iterator index_itr_;
47		array_type::const_iterator index_end_;
48		mapnik::context_ptr ctx_;
49		};
50
51		#endif // LARGE_GEOJSON_FEATURESET_HPP

-2

plugins/input/ogr/ogr_index_featureset.cpp less more

32	32	#include <mapnik/geometry_correct.hpp>
33	33
34	34	// boost
35		#ifdef SHAPE_MEMORY_MAPPED_FILE
	35	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
36	36	#include <mapnik/mapped_memory_cache.hpp>
37	37	#pragma GCC diagnostic push
38	38	#pragma GCC diagnostic ignored "-Wshadow"

71	71	feature_envelope_()
72	72	{
73	73
74		#ifdef SHAPE_MEMORY_MAPPED_FILE
	74	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
75	75	boost::optional<mapnik::mapped_region_ptr> memory = mapnik::mapped_memory_cache::instance().find(index_file, true);
76	76	if (memory)
77	77	{

-4

plugins/input/shape/dbfile.cpp less more

35	35	#pragma GCC diagnostic ignored "-Wmissing-field-initializers"
36	36	#pragma GCC diagnostic ignored "-Wsign-conversion"
37	37	#include <boost/spirit/include/qi.hpp>
38		#ifdef SHAPE_MEMORY_MAPPED_FILE
	38	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
39	39	#include <boost/interprocess/mapped_region.hpp>
40	40	#include <mapnik/mapped_memory_cache.hpp>
41	41	#endif

57	57	:num_records_(0),
58	58	num_fields_(0),
59	59	record_length_(0),
60		#ifdef SHAPE_MEMORY_MAPPED_FILE
	60	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
61	61	file_(),
62	62	#elif defined(_WINDOWS)
63	63	file_(mapnik::utf8_to_utf16(file_name), std::ios::in \| std::ios::binary),

67	67	record_(0)
68	68	{
69	69
70		#ifdef SHAPE_MEMORY_MAPPED_FILE
	70	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
71	71	boost::optional<mapnik::mapped_region_ptr> memory = mapnik::mapped_memory_cache::instance().find(file_name,true);
72	72	if (memory)
73	73	{

94	94
95	95	bool dbf_file::is_open()
96	96	{
97		#ifdef SHAPE_MEMORY_MAPPED_FILE
	97	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
98	98	return (file_.buffer().second > 0);
99	99	#else
100	100	return file_.is_open();

-2

plugins/input/shape/dbfile.hpp less more

26	26	#include <mapnik/feature.hpp>
27	27	#include <mapnik/util/noncopyable.hpp>
28	28	#include <mapnik/unicode.hpp>
29		#ifdef SHAPE_MEMORY_MAPPED_FILE
	29	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
30	30	#include <mapnik/mapped_memory_cache.hpp>
31	31	#pragma GCC diagnostic push
32	32	#pragma GCC diagnostic ignored "-Wshadow"

58	58	int num_fields_;
59	59	std::size_t record_length_;
60	60	std::vector<field_descriptor> fields_;
61		#ifdef SHAPE_MEMORY_MAPPED_FILE
	61	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
62	62	boost::interprocess::ibufferstream file_;
63	63	mapnik::mapped_region_ptr mapped_region_;
64	64	#else

+21

-17

plugins/input/shape/shape_datasource.cpp less more

103	103	init(*shape_ref);
104	104	for (int i=0;i<shape_ref->dbf().num_fields();++i)
105	105	{
106		field_descriptor const& fd=shape_ref->dbf().descriptor(i);
	106	field_descriptor const& fd = shape_ref->dbf().descriptor(i);
107	107	std::string fld_name=fd.name_;
108	108	switch (fd.type_)
109	109	{

166	166	#endif
167	167
168	168	//first read header from *.shp
169		int file_code=shape.shp().read_xdr_integer();
170		if (file_code!=9994)
	169	shape_file::record_type header(100);
	170	shape.shp().read_record(header);
	171
	172	int file_code = header.read_xdr_integer();
	173	if (file_code != 9994)
171	174	{
172	175	std::ostringstream s;
173	176	s << "Shape Plugin: wrong file code " << file_code;
174	177	throw datasource_exception(s.str());
175	178	}
176
177		shape.shp().skip(5*4);
178		file_length_=shape.shp().read_xdr_integer();
179		int version=shape.shp().read_ndr_integer();
180
181		if (version!=1000)
	179	header.skip(5 * 4);
	180	file_length_ = header.read_xdr_integer();
	181	int version = header.read_ndr_integer();
	182
	183	if (version != 1000)
182	184	{
183	185	std::ostringstream s;
184	186	s << "Shape Plugin: nvalid version number " << version;
185	187	throw datasource_exception(s.str());
186	188	}
187	189
188		shape_type_ = static_cast<shape_io::shapeType>(shape.shp().read_ndr_integer());
	190	shape_type_ = static_cast<shape_io::shapeType>(header.read_ndr_integer());
189	191	if (shape_type_ == shape_io::shape_multipatch)
190	192	throw datasource_exception("Shape Plugin: shapefile multipatch type is not supported");
191	193
192		shape.shp().read_envelope(extent_);
	194	const double lox = header.read_double();
	195	const double loy = header.read_double();
	196	const double hix = header.read_double();
	197	const double hiy = header.read_double();
	198	extent_.init(lox, loy, hix, hiy);
193	199
194	200	#ifdef MAPNIK_LOG
195		const double zmin = shape.shp().read_double();
196		const double zmax = shape.shp().read_double();
197		const double mmin = shape.shp().read_double();
198		const double mmax = shape.shp().read_double();
	201	const double zmin = header.read_double();
	202	const double zmax = header.read_double();
	203	const double mmin = header.read_double();
	204	const double mmax = header.read_double();
199	205
200	206	MAPNIK_LOG_DEBUG(shape) << "shape_datasource: Z min/max=" << zmin << "," << zmax;
201	207	MAPNIK_LOG_DEBUG(shape) << "shape_datasource: M min/max=" << mmin << "," << mmax;

249	255	shape_name_,
250	256	q.property_names(),
251	257	desc_.get_encoding(),
252		file_length_,
253	258	row_limit_);
254	259	}
255	260	}

287	292	shape_name_,
288	293	names,
289	294	desc_.get_encoding(),
290		file_length_,
291	295	row_limit_);
292	296	}
293	297	}

+22

-16

plugins/input/shape/shape_featureset.cpp less more

21	21
22	22	// stl
23	23	#include <iostream>
24
	24	#include <cassert>
25	25	// mapnik
26	26	#include <mapnik/debug.hpp>
27	27	#include <mapnik/feature_factory.hpp>

40	40	std::string const& shape_name,
41	41	std::set<std::string> const& attribute_names,
42	42	std::string const& encoding,
43		long file_length,
44	43	int row_limit)
45	44	: filter_(filter),
46	45	shape_(shape_name, false),
47	46	query_ext_(),
48	47	feature_bbox_(),
49	48	tr_(new transcoder(encoding)),
50		file_length_(file_length),
	49	shx_file_length_(0),
51	50	row_limit_(row_limit),
52	51	count_(0),
53	52	ctx_(std::make_shared<mapnik::context_type>())
54	53	{
55		shape_.shp().skip(100);
56		setup_attributes(ctx_, attribute_names, shape_name, shape_,attr_ids_);
	54	if (!shape_.shx().is_open())
	55	{
	56	throw mapnik::datasource_exception("Shape Plugin: can't open '" + shape_name + ".shx' file");
	57	}
	58	shape_file::record_type shx_header(100);
	59	shape_.shx().read_record(shx_header);
	60	shx_header.skip(6 * 4);
	61	shx_file_length_ = shx_header.read_xdr_integer();
	62	setup_attributes(ctx_, attribute_names, shape_name, shape_, attr_ids_);
57	63	}
58	64
59	65	template <typename filterT>

64	70	return feature_ptr();
65	71	}
66	72
67
68		while (shape_.shp().pos() < std::streampos(file_length_ * 2))
	73	std::streampos position_limit = 2 * shx_file_length_ - 2 * sizeof(int);
	74	while (shape_.shx().is_good() && shape_.shx().pos() <= position_limit)
69	75	{
70		shape_.move_to(shape_.shp().pos());
71		shape_file::record_type record(shape_.reclength_ * 2);
	76	int offset = shape_.shx().read_xdr_integer();
	77	int record_length = shape_.shx().read_xdr_integer();
	78	shape_.move_to(2 * offset);
	79	mapnik::value_integer feature_id = shape_.id();
	80	assert(record_length == shape_.reclength_);
	81	shape_file::record_type record(record_length * 2);
72	82	shape_.shp().read_record(record);
73	83	int type = record.read_ndr_integer();
74	84
75	85	// skip null shapes
76	86	if (type == shape_io::shape_null) continue;
77	87
78		feature_ptr feature(feature_factory::create(ctx_, shape_.id_));
	88	feature_ptr feature(feature_factory::create(ctx_, feature_id));
79	89	switch (type)
80	90	{
81	91	case shape_io::shape_point:

130	140	return feature_ptr();
131	141	}
132	142
133		// FIXME: https://github.com/mapnik/mapnik/issues/1020
134		feature->set_id(shape_.id_);
135	143	if (attr_ids_.size())
136	144	{
137	145	shape_.dbf().move_to(shape_.id_);
138		std::vector<int>::const_iterator itr = attr_ids_.begin();
139		std::vector<int>::const_iterator end = attr_ids_.end();
140	146	try
141	147	{
142		for (; itr != end; ++itr)
	148	for (auto id : attr_ids_)
143	149	{
144		shape_.dbf().add_attribute(itr, tr_, *feature); //TODO optimize!!!
	150	shape_.dbf().add_attribute(id, tr_, feature); //TODO optimize!!!
145	151	}
146	152	}
147	153	catch (...)

-2

plugins/input/shape/shape_featureset.hpp less more

49	49	std::string const& shape_file,
50	50	std::set<std::string> const& attribute_names,
51	51	std::string const& encoding,
52		long file_length,
53	52	int row_limit);
54	53	virtual ~shape_featureset();
55	54	feature_ptr next();

60	59	box2d<double> query_ext_;
61	60	mutable box2d<double> feature_bbox_;
62	61	const std::unique_ptr<transcoder> tr_;
63		long file_length_;
	62	long shx_file_length_;
64	63	std::vector<int> attr_ids_;
65	64	mapnik::value_integer row_limit_;
66	65	mutable int count_;

-9

plugins/input/shape/shape_index_featureset.cpp less more

31	31	#pragma GCC diagnostic ignored "-Wunused-parameter"
32	32	#pragma GCC diagnostic ignored "-Wunused-local-typedef"
33	33	#include <boost/algorithm/string.hpp>
34		#ifdef SHAPE_MEMORY_MAPPED_FILE
	34	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
35	35	#include <boost/interprocess/streams/bufferstream.hpp>
36	36	#endif
37	37	#pragma GCC diagnostic pop

62	62	auto index = shape_ptr_->index();
63	63	if (index)
64	64	{
65		#ifdef SHAPE_MEMORY_MAPPED_FILE
	65	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
66	66	mapnik::util::spatial_index<int, filterT,boost::interprocess::ibufferstream>::query(filter, index->file(), offsets_);
67	67	#else
68	68	mapnik::util::spatial_index<int, filterT, std::ifstream>::query(filter, index->file(), offsets_);

84	84	while ( itr_ != offsets_.end())
85	85	{
86	86	shape_ptr_->move_to(*itr_++);
	87	mapnik::value_integer feature_id = shape_ptr_->id();
87	88	shape_file::record_type record(shape_ptr_->reclength_ * 2);
88	89	shape_ptr_->shp().read_record(record);
89	90	int type = record.read_ndr_integer();
90		feature_ptr feature(feature_factory::create(ctx_,shape_ptr_->id_));
	91	feature_ptr feature(feature_factory::create(ctx_, feature_id));
91	92
92	93	switch (type)
93	94	{

140	141	return feature_ptr();
141	142	}
142	143
143		// FIXME: https://github.com/mapnik/mapnik/issues/1020
144		feature->set_id(shape_ptr_->id_);
145	144	if (attr_ids_.size())
146	145	{
147	146	shape_ptr_->dbf().move_to(shape_ptr_->id_);
148		std::vector<int>::const_iterator itr = attr_ids_.begin();
149		std::vector<int>::const_iterator end = attr_ids_.end();
150	147	try
151	148	{
152		for (; itr!=end; ++itr)
	149	for (auto id : attr_ids_)
153	150	{
154		shape_ptr_->dbf().add_attribute(itr, tr_, *feature);
	151	shape_ptr_->dbf().add_attribute(id, tr_, feature);
155	152	}
156	153	}
157	154	catch (...)

+12

-1

plugins/input/shape/shape_io.cpp less more

30	30
31	31	using mapnik::datasource_exception;
32	32	const std::string shape_io::SHP = ".shp";
	33	const std::string shape_io::SHX = ".shx";
33	34	const std::string shape_io::DBF = ".dbf";
34	35	const std::string shape_io::INDEX = ".index";
35	36
36	37	shape_io::shape_io(std::string const& shape_name, bool open_index)
37	38	: type_(shape_null),
38	39	shp_(shape_name + SHP),
	40	shx_(shape_name + SHX),
39	41	dbf_(shape_name + DBF),
40	42	reclength_(0),
41	43	id_(0)
42	44	{
43	45	bool ok = (shp_.is_open() && dbf_.is_open());
44		if (! ok)
	46	if (!ok)
45	47	{
46	48	throw datasource_exception("Shape Plugin: cannot read shape file '" + shape_name + "'");
47	49	}

56	58	{
57	59	MAPNIK_LOG_WARN(shape) << "shape_io: Could not open index=" << shape_name << INDEX;
58	60	}
	61	}
	62	if (!index_ && !shx_.is_open())
	63	{
	64	throw datasource_exception("Shape Plugin: cannot read shape index file '" + shape_name + ".shx'");
59	65	}
60	66	}
61	67

71	77	shape_file& shape_io::shp()
72	78	{
73	79	return shp_;
	80	}
	81
	82	shape_file& shape_io::shx()
	83	{
	84	return shx_;
74	85	}
75	86
76	87	dbf_file& shape_io::dbf()

-2

plugins/input/shape/shape_io.hpp less more

60	60	~shape_io();
61	61
62	62	shape_file& shp();
	63	shape_file& shx();
63	64	dbf_file& dbf();
64	65
65	66	inline boost::optional<shape_file&> index()

73	74	return (index_ && index_->is_open());
74	75	}
75	76
	77	inline int id() const { return id_;}
76	78	void move_to(std::streampos pos);
77	79	static void read_bbox(shape_file::record_type & record, mapnik::box2d<double> & bbox);
78	80	static mapnik::geometry::geometry<double> read_polyline(shape_file::record_type & record);

80	82
81	83	shapeType type_;
82	84	shape_file shp_;
	85	shape_file shx_;
83	86	dbf_file dbf_;
84	87	std::unique_ptr<shape_file> index_;
85		unsigned reclength_;
86		unsigned id_;
	88	int reclength_;
	89	int id_;
87	90	box2d<double> cur_extent_;
88	91
89	92	static const std::string SHP;
	93	static const std::string SHX;
90	94	static const std::string DBF;
91	95	static const std::string INDEX;
92	96	};

+11

-6

plugins/input/shape/shapefile.hpp less more

32	32	#include <mapnik/global.hpp>
33	33	#include <mapnik/util/utf_conv_win.hpp>
34	34	#include <mapnik/box2d.hpp>
35		#ifdef SHAPE_MEMORY_MAPPED_FILE
	35	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
36	36	#pragma GCC diagnostic push
37	37	#pragma GCC diagnostic ignored "-Wshadow"
38	38	#pragma GCC diagnostic ignored "-Wsign-conversion"

140	140	{
141	141	public:
142	142
143		#ifdef SHAPE_MEMORY_MAPPED_FILE
	143	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
144	144	using file_source_type = boost::interprocess::ibufferstream;
145	145	using record_type = shape_record<MappedRecordTag>;
146	146	mapnik::mapped_region_ptr mapped_region_;

154	154	shape_file() {}
155	155
156	156	shape_file(std::string const& file_name) :
157		#ifdef SHAPE_MEMORY_MAPPED_FILE
	157	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
158	158	file_()
159	159	#elif defined (_WINDOWS)
160	160	file_(mapnik::utf8_to_utf16(file_name), std::ios::in \| std::ios::binary)

162	162	file_(file_name.c_str(), std::ios::in \| std::ios::binary)
163	163	#endif
164	164	{
165		#ifdef SHAPE_MEMORY_MAPPED_FILE
	165	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
166	166	boost::optional<mapnik::mapped_region_ptr> memory =
167	167	mapnik::mapped_memory_cache::instance().find(file_name,true);
168	168

187	187
188	188	inline bool is_open() const
189	189	{
190		#ifdef SHAPE_MEMORY_MAPPED_FILE
	190	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
191	191	return (file_.buffer().second > 0);
192	192	#else
193	193	return file_.is_open();

196	196
197	197	inline void read_record(record_type& rec)
198	198	{
199		#ifdef SHAPE_MEMORY_MAPPED_FILE
	199	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
200	200	rec.set_data(file_.buffer().first + file_.tellg());
201	201	file_.seekg(rec.size, std::ios::cur);
202	202	#else

258	258	{
259	259	return file_.eof();
260	260	}
	261
	262	inline bool is_good()
	263	{
	264	return file_.good();
	265	}
261	266	};
262	267
263	268	#endif // SHAPEFILE_HPP

-0

src/json/mapnik_json_feature_collection_grammar.cpp less more

25	25
26	26	using iterator_type = char const*;
27	27	template struct mapnik::json::feature_collection_grammar<iterator_type,mapnik::feature_impl, mapnik::json::default_feature_callback> ;
	28	template struct mapnik::json::feature_grammar_callback<iterator_type,mapnik::feature_impl, mapnik::json::default_feature_callback> ;

-1

src/mapped_memory_cache.cpp less more

19	19	*
20	20	*****************************************************************************/
21	21
22		#if defined(SHAPE_MEMORY_MAPPED_FILE)
	22	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
23	23
24	24	// mapnik
25	25	#include <mapnik/debug.hpp>

-676

~~test/standalone/csv_test.cpp~~ less more

0		#define CATCH_CONFIG_MAIN
1		#include "catch.hpp"
2
3		#include <mapnik/map.hpp>
4		#include <mapnik/datasource.hpp>
5		#include <mapnik/datasource_cache.hpp>
6		#include <mapnik/geometry.hpp>
7		#include <mapnik/geometry_types.hpp>
8		#include <mapnik/geometry_type.hpp>
9		#include <mapnik/expression.hpp>
10		#include <mapnik/expression_evaluator.hpp>
11		#include <mapnik/debug.hpp>
12		#include <mapnik/util/fs.hpp>
13		#include <boost/filesystem.hpp>
14		#include <boost/range/iterator_range_core.hpp>
15		#include <boost/format.hpp>
16		#include <boost/optional/optional_io.hpp>
17
18		#include <iostream>
19
20		namespace bfs = boost::filesystem;
21
22		namespace {
23		void add_csv_files(bfs::path dir, std::vector<bfs::path> &csv_files)
24		{
25		for (auto const &entry : boost::make_iterator_range(
26		bfs::directory_iterator(dir), bfs::directory_iterator()))
27		{
28		auto path = entry.path();
29		if (path.extension().native() == ".csv")
30		{
31		csv_files.emplace_back(path);
32		}
33		}
34		}
35
36		mapnik::datasource_ptr get_csv_ds(std::string const &file_name, bool strict = true)
37		{
38		mapnik::parameters params;
39		params["type"] = std::string("csv");
40		params["file"] = file_name;
41		params["strict"] = mapnik::value_bool(strict);
42		auto ds = mapnik::datasource_cache::instance().create(params);
43		// require a non-null pointer returned
44		REQUIRE(ds != nullptr);
45		return ds;
46		}
47
48		void require_field_names(std::vector<mapnik::attribute_descriptor> const &fields,
49		std::initializer_list<std::string> const &names)
50		{
51		REQUIRE(fields.size() == names.size());
52		auto itr_a = fields.begin();
53		auto const end_a = fields.end();
54		auto itr_b = names.begin();
55		for (; itr_a != end_a; ++itr_a, ++itr_b)
56		{
57		CHECK(itr_a->get_name() == *itr_b);
58		}
59		}
60
61		void require_field_types(std::vector<mapnik::attribute_descriptor> const &fields,
62		std::initializer_list<mapnik::eAttributeType> const &types) {
63		REQUIRE(fields.size() == types.size());
64		auto itr_a = fields.begin();
65		auto const end_a = fields.end();
66		auto itr_b = types.begin();
67		for (; itr_a != end_a; ++itr_a, ++itr_b) {
68		CHECK(itr_a->get_type() == *itr_b);
69		}
70		}
71
72		mapnik::featureset_ptr all_features(mapnik::datasource_ptr ds) {
73		auto fields = ds->get_descriptor().get_descriptors();
74		mapnik::query query(ds->envelope());
75		for (auto const &field : fields) {
76		query.add_property_name(field.get_name());
77		}
78		return ds->features(query);
79		}
80
81		std::size_t count_features(mapnik::featureset_ptr features) {
82		std::size_t count = 0;
83		while (features->next()) {
84		++count;
85		}
86		return count;
87		}
88
89		using attr = std::tuple<std::string, mapnik::value>;
90		void require_attributes(mapnik::feature_ptr feature,
91		std::initializer_list<attr> const &attrs) {
92		REQUIRE(bool(feature));
93		for (auto const &kv : attrs) {
94		REQUIRE(feature->has_key(std::get<0>(kv)));
95		CHECK(feature->get(std::get<0>(kv)) == std::get<1>(kv));
96		}
97		}
98
99		namespace detail {
100		struct feature_count {
101		template <typename T>
102		std::size_t operator()(T const &geom) const {
103		return mapnik::util::apply_visitor(*this, geom);
104		}
105
106		std::size_t operator()(mapnik::geometry::geometry_empty const &) const {
107		return 0;
108		}
109
110		template <typename T>
111		std::size_t operator()(mapnik::geometry::point<T> const &) const {
112		return 1;
113		}
114
115		template <typename T>
116		std::size_t operator()(mapnik::geometry::line_string<T> const &) const {
117		return 1;
118		}
119
120		template <typename T>
121		std::size_t operator()(mapnik::geometry::polygon<T> const &) const {
122		return 1;
123		}
124
125		template <typename T>
126		std::size_t operator()(mapnik::geometry::multi_point<T> const &mp) const {
127		return mp.size();
128		}
129
130		template <typename T>
131		std::size_t operator()(mapnik::geometry::multi_line_string<T> const &mls) const {
132		return mls.size();
133		}
134
135		template <typename T>
136		std::size_t operator()(mapnik::geometry::multi_polygon<T> const &mp) const {
137		return mp.size();
138		}
139
140		template <typename T>
141		std::size_t operator()(mapnik::geometry::geometry_collection<T> const &col) const {
142		std::size_t sum = 0;
143		for (auto const &geom : col) {
144		sum += operator()(geom);
145		}
146		return sum;
147		}
148		};
149		} // namespace detail
150
151		template <typename T>
152		std::size_t feature_count(mapnik::geometry::geometry<T> const &g) {
153		return detail::feature_count()(g);
154		}
155
156		void require_geometry(mapnik::feature_ptr feature,
157		std::size_t num_parts,
158		mapnik::geometry::geometry_types type) {
159		REQUIRE(bool(feature));
160		CHECK(mapnik::geometry::geometry_type(feature->get_geometry()) == type);
161		CHECK(feature_count(feature->get_geometry()) == num_parts);
162		}
163		} // anonymous namespace
164
165		static const std::string csv_plugin("./plugins/input/csv.input");
166
167		const bool registered = mapnik::datasource_cache::instance().register_datasources(csv_plugin);
168
169		TEST_CASE("csv") {
170
171		if (mapnik::util::exists(csv_plugin))
172		{
173		REQUIRE(registered);
174		// make the tests silent since we intentially test error conditions that are noisy
175		auto const severity = mapnik::logger::instance().get_severity();
176		mapnik::logger::instance().set_severity(mapnik::logger::none);
177
178		// check the CSV datasource is loaded
179		const std::vector<std::string> plugin_names =
180		mapnik::datasource_cache::instance().plugin_names();
181		const bool have_csv_plugin =
182		std::find(plugin_names.begin(), plugin_names.end(), "csv") != plugin_names.end();
183
184		SECTION("broken files") {
185		if (have_csv_plugin) {
186		std::vector<bfs::path> broken;
187		add_csv_files("test/data/csv/fails", broken);
188		add_csv_files("test/data/csv/warns", broken);
189		broken.emplace_back("test/data/csv/fails/does_not_exist.csv");
190
191		for (auto const &path : broken)
192		{
193		INFO(path);
194		REQUIRE_THROWS(get_csv_ds(path.native()));
195		}
196		}
197		} // END SECTION
198
199		SECTION("good files") {
200		if (have_csv_plugin) {
201		std::vector<bfs::path> good;
202		add_csv_files("test/data/csv", good);
203		add_csv_files("test/data/csv/warns", good);
204
205		for (auto const& path : good)
206		{
207		auto ds = get_csv_ds(path.native(), false);
208		// require a non-null pointer returned
209		REQUIRE(bool(ds));
210		}
211		}
212		} // END SECTION
213
214		SECTION("lon/lat detection")
215		{
216		for (auto const& lon_name : {std::string("lon"), std::string("lng")})
217		{
218		auto ds = get_csv_ds((boost::format("test/data/csv/%1%_lat.csv") % lon_name).str());
219		auto fields = ds->get_descriptor().get_descriptors();
220		require_field_names(fields, {lon_name, "lat"});
221		require_field_types(fields, {mapnik::Integer, mapnik::Integer});
222
223		CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
224
225		mapnik::query query(ds->envelope());
226		for (auto const &field : fields)
227		{
228		query.add_property_name(field.get_name());
229		}
230		auto features = ds->features(query);
231		auto feature = features->next();
232
233		require_attributes(feature, {
234		attr { lon_name, mapnik::value_integer(0) },
235		attr { "lat", mapnik::value_integer(0) }
236		});
237		}
238		} // END SECTION
239
240		SECTION("type detection") {
241		auto ds = get_csv_ds("test/data/csv/nypd.csv");
242		auto fields = ds->get_descriptor().get_descriptors();
243		require_field_names(fields, {"Precinct", "Phone", "Address", "City", "geo_longitude", "geo_latitude", "geo_accuracy"});
244		require_field_types(fields, {mapnik::String, mapnik::String, mapnik::String, mapnik::String, mapnik::Double, mapnik::Double, mapnik::String});
245
246		CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
247		CHECK(count_features(all_features(ds)) == 2);
248
249		auto feature = all_features(ds)->next();
250		require_attributes(feature, {
251		attr { "City", mapnik::value_unicode_string("New York, NY") }
252		, attr { "geo_accuracy", mapnik::value_unicode_string("house") }
253		, attr { "Phone", mapnik::value_unicode_string("(212) 334-0711") }
254		, attr { "Address", mapnik::value_unicode_string("19 Elizabeth Street") }
255		, attr { "Precinct", mapnik::value_unicode_string("5th Precinct") }
256		, attr { "geo_longitude", mapnik::value_integer(-70) }
257		, attr { "geo_latitude", mapnik::value_integer(40) }
258		});
259		} // END SECTION
260
261		SECTION("skipping blank rows") {
262		auto ds = get_csv_ds("test/data/csv/blank_rows.csv");
263		auto fields = ds->get_descriptor().get_descriptors();
264		require_field_names(fields, {"x", "y", "name"});
265		require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String});
266
267		CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
268		CHECK(count_features(all_features(ds)) == 2);
269		} // END SECTION
270
271		SECTION("empty rows") {
272		auto ds = get_csv_ds("test/data/csv/empty_rows.csv");
273		auto fields = ds->get_descriptor().get_descriptors();
274		require_field_names(fields, {"x", "y", "text", "date", "integer", "boolean", "float", "time", "datetime", "empty_column"});
275		require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String, mapnik::String, mapnik::Integer, mapnik::Boolean, mapnik::Double, mapnik::String, mapnik::String, mapnik::String});
276
277		CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
278		CHECK(count_features(all_features(ds)) == 4);
279
280		auto featureset = all_features(ds);
281		auto feature = featureset->next();
282		require_attributes(feature, {
283		attr { "x", mapnik::value_integer(0) }
284		, attr { "empty_column", mapnik::value_unicode_string("") }
285		, attr { "text", mapnik::value_unicode_string("a b") }
286		, attr { "float", mapnik::value_double(1.0) }
287		, attr { "datetime", mapnik::value_unicode_string("1971-01-01T04:14:00") }
288		, attr { "y", mapnik::value_integer(0) }
289		, attr { "boolean", mapnik::value_bool(true) }
290		, attr { "time", mapnik::value_unicode_string("04:14:00") }
291		, attr { "date", mapnik::value_unicode_string("1971-01-01") }
292		, attr { "integer", mapnik::value_integer(40) }
293		});
294
295		while (bool(feature = featureset->next())) {
296		CHECK(feature->size() == 10);
297		CHECK(feature->get("empty_column") == mapnik::value_unicode_string(""));
298		}
299		} // END SECTION
300
301		SECTION("slashes") {
302		auto ds = get_csv_ds("test/data/csv/has_attributes_with_slashes.csv");
303		auto fields = ds->get_descriptor().get_descriptors();
304		require_field_names(fields, {"x", "y", "name"});
305		// NOTE: y column is integer, even though a double value is used below in the test?
306		require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String});
307
308		auto featureset = all_features(ds);
309		require_attributes(featureset->next(), {
310		attr{"x", 0}
311		, attr{"y", 0}
312		, attr{"name", mapnik::value_unicode_string("a/a") } });
313		require_attributes(featureset->next(), {
314		attr{"x", 1}
315		, attr{"y", 4}
316		, attr{"name", mapnik::value_unicode_string("b/b") } });
317		require_attributes(featureset->next(), {
318		attr{"x", 10}
319		, attr{"y", 2.5}
320		, attr{"name", mapnik::value_unicode_string("c/c") } });
321		} // END SECTION
322
323		SECTION("wkt field") {
324		using mapnik::geometry::geometry_types;
325
326		auto ds = get_csv_ds("test/data/csv/wkt.csv");
327		auto fields = ds->get_descriptor().get_descriptors();
328		require_field_names(fields, {"type"});
329		require_field_types(fields, {mapnik::String});
330
331		auto featureset = all_features(ds);
332		require_geometry(featureset->next(), 1, geometry_types::Point);
333		require_geometry(featureset->next(), 1, geometry_types::LineString);
334		require_geometry(featureset->next(), 1, geometry_types::Polygon);
335		require_geometry(featureset->next(), 1, geometry_types::Polygon);
336		require_geometry(featureset->next(), 4, geometry_types::MultiPoint);
337		require_geometry(featureset->next(), 2, geometry_types::MultiLineString);
338		require_geometry(featureset->next(), 2, geometry_types::MultiPolygon);
339		require_geometry(featureset->next(), 2, geometry_types::MultiPolygon);
340		} // END SECTION
341
342		SECTION("handling of missing header") {
343		// TODO: does this mean 'missing_header.csv' should be in the warnings
344		// subdirectory, since it doesn't work in strict mode?
345		auto ds = get_csv_ds("test/data/csv/missing_header.csv", false);
346		auto fields = ds->get_descriptor().get_descriptors();
347		require_field_names(fields, {"one", "two", "x", "y", "_4", "aftermissing"});
348		auto feature = all_features(ds)->next();
349		REQUIRE(feature);
350		REQUIRE(feature->has_key("_4"));
351		CHECK(feature->get("_4") == mapnik::value_unicode_string("missing"));
352		} // END SECTION
353
354		SECTION("handling of headers that are numbers") {
355		auto ds = get_csv_ds("test/data/csv/numbers_for_headers.csv");
356		auto fields = ds->get_descriptor().get_descriptors();
357		require_field_names(fields, {"x", "y", "1990", "1991", "1992"});
358		auto feature = all_features(ds)->next();
359		require_attributes(feature, {
360		attr{"x", 0}
361		, attr{"y", 0}
362		, attr{"1990", 1}
363		, attr{"1991", 2}
364		, attr{"1992", 3}
365		});
366		auto expression = mapnik::parse_expression("[1991]=2");
367		REQUIRE(bool(expression));
368		auto value = mapnik::util::apply_visitor(
369		mapnik::evaluate<mapnik::feature_impl, mapnik::value_type, mapnik::attributes>(
370		feature, mapnik::attributes()), expression);
371		CHECK(value == true);
372		} // END SECTION
373
374		SECTION("quoted numbers") {
375		using ustring = mapnik::value_unicode_string;
376
377		auto ds = get_csv_ds("test/data/csv/quoted_numbers.csv");
378		auto fields = ds->get_descriptor().get_descriptors();
379		require_field_names(fields, {"x", "y", "label"});
380		auto featureset = all_features(ds);
381
382		require_attributes(featureset->next(), {
383		attr{"x", 0}, attr{"y", 0}, attr{"label", ustring("0,0") } });
384		require_attributes(featureset->next(), {
385		attr{"x", 5}, attr{"y", 5}, attr{"label", ustring("5,5") } });
386		require_attributes(featureset->next(), {
387		attr{"x", 0}, attr{"y", 5}, attr{"label", ustring("0,5") } });
388		require_attributes(featureset->next(), {
389		attr{"x", 5}, attr{"y", 0}, attr{"label", ustring("5,0") } });
390		require_attributes(featureset->next(), {
391		attr{"x", 2.5}, attr{"y", 2.5}, attr{"label", ustring("2.5,2.5") } });
392
393		} // END SECTION
394
395		SECTION("reading newlines") {
396		for (auto const &platform : {std::string("windows"), std::string("mac")}) {
397		std::string file_name = (boost::format("test/data/csv/%1%_newlines.csv") % platform).str();
398		auto ds = get_csv_ds(file_name);
399		auto fields = ds->get_descriptor().get_descriptors();
400		require_field_names(fields, {"x", "y", "z"});
401		require_attributes(all_features(ds)->next(), {
402		attr{"x", 1}, attr{"y", 10}, attr{"z", 9999.9999} });
403		}
404		} // END SECTION
405
406		SECTION("mixed newlines") {
407		using ustring = mapnik::value_unicode_string;
408
409		for (auto const &file : {
410		std::string("test/data/csv/mac_newlines_with_unix_inline.csv")
411		, std::string("test/data/csv/mac_newlines_with_unix_inline_escaped.csv")
412		, std::string("test/data/csv/windows_newlines_with_unix_inline.csv")
413		, std::string("test/data/csv/windows_newlines_with_unix_inline_escaped.csv")
414		}) {
415		auto ds = get_csv_ds(file);
416		auto fields = ds->get_descriptor().get_descriptors();
417		require_field_names(fields, {"x", "y", "line"});
418		require_attributes(all_features(ds)->next(), {
419		attr{"x", 0}, attr{"y", 0}
420		, attr{"line", ustring("many\n lines\n of text\n with unix newlines")} });
421		}
422		} // END SECTION
423
424		SECTION("tabs") {
425		auto ds = get_csv_ds("test/data/csv/tabs_in_csv.csv");
426		auto fields = ds->get_descriptor().get_descriptors();
427		require_field_names(fields, {"x", "y", "z"});
428		require_attributes(all_features(ds)->next(), {
429		attr{"x", -122}, attr{"y", 48}, attr{"z", 0} });
430		} // END SECTION
431
432		SECTION("separators") {
433		using ustring = mapnik::value_unicode_string;
434
435		for (auto const &file : {
436		std::string("test/data/csv/pipe_delimiters.csv")
437		, std::string("test/data/csv/semicolon_delimiters.csv")
438		}) {
439		auto ds = get_csv_ds(file);
440		auto fields = ds->get_descriptor().get_descriptors();
441		require_field_names(fields, {"x", "y", "z"});
442		require_attributes(all_features(ds)->next(), {
443		attr{"x", 0}, attr{"y", 0}, attr{"z", ustring("hello")} });
444		}
445		} // END SECTION
446
447		SECTION("null and bool keywords are empty strings") {
448		using ustring = mapnik::value_unicode_string;
449
450		auto ds = get_csv_ds("test/data/csv/nulls_and_booleans_as_strings.csv");
451		auto fields = ds->get_descriptor().get_descriptors();
452		require_field_names(fields, {"x", "y", "null", "boolean"});
453		require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String, mapnik::Boolean});
454
455		auto featureset = all_features(ds);
456		require_attributes(featureset->next(), {
457		attr{"x", 0}, attr{"y", 0}, attr{"null", ustring("null")}, attr{"boolean", true}});
458		require_attributes(featureset->next(), {
459		attr{"x", 0}, attr{"y", 0}, attr{"null", ustring("")}, attr{"boolean", false}});
460		} // END SECTION
461
462		SECTION("nonexistent query fields throw") {
463		auto ds = get_csv_ds("test/data/csv/lon_lat.csv");
464		auto fields = ds->get_descriptor().get_descriptors();
465		require_field_names(fields, {"lon", "lat"});
466		require_field_types(fields, {mapnik::Integer, mapnik::Integer});
467
468		mapnik::query query(ds->envelope());
469		for (auto const &field : fields) {
470		query.add_property_name(field.get_name());
471		}
472		// also add an invalid one, triggering throw
473		query.add_property_name("bogus");
474
475		REQUIRE_THROWS(ds->features(query));
476		} // END SECTION
477
478		SECTION("leading zeros mean strings") {
479		using ustring = mapnik::value_unicode_string;
480
481		auto ds = get_csv_ds("test/data/csv/leading_zeros.csv");
482		auto fields = ds->get_descriptor().get_descriptors();
483		require_field_names(fields, {"x", "y", "fips"});
484		require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String});
485
486		auto featureset = all_features(ds);
487		require_attributes(featureset->next(), {
488		attr{"x", 0}, attr{"y", 0}, attr{"fips", ustring("001")}});
489		require_attributes(featureset->next(), {
490		attr{"x", 0}, attr{"y", 0}, attr{"fips", ustring("003")}});
491		require_attributes(featureset->next(), {
492		attr{"x", 0}, attr{"y", 0}, attr{"fips", ustring("005")}});
493		} // END SECTION
494
495		SECTION("advanced geometry detection") {
496		using row = std::pair<std::string, mapnik::datasource_geometry_t>;
497
498		for (row r : {
499		row{"point", mapnik::datasource_geometry_t::Point}
500		, row{"poly", mapnik::datasource_geometry_t::Polygon}
501		, row{"multi_poly", mapnik::datasource_geometry_t::Polygon}
502		, row{"line", mapnik::datasource_geometry_t::LineString}
503		}) {
504		std::string file_name = (boost::format("test/data/csv/%1%_wkt.csv") % r.first).str();
505		auto ds = get_csv_ds(file_name);
506		CHECK(ds->get_geometry_type() == r.second);
507		}
508		} // END SECTION
509
510		SECTION("creation of CSV from in-memory strings") {
511		using ustring = mapnik::value_unicode_string;
512
513		for (auto const &name : {std::string("Winthrop, WA"), std::string(u8"Qu\u00e9bec")}) {
514		std::string csv_string =
515		(boost::format(
516		"wkt,Name\n"
517		"\"POINT (120.15 48.47)\",\"%1%\"\n"
518		) % name).str();
519
520		mapnik::parameters params;
521		params["type"] = std::string("csv");
522		params["inline"] = csv_string;
523		auto ds = mapnik::datasource_cache::instance().create(params);
524		REQUIRE(bool(ds));
525
526		auto feature = all_features(ds)->next();
527		REQUIRE(bool(feature));
528		REQUIRE(feature->has_key("Name"));
529		CHECK(feature->get("Name") == ustring(name.c_str()));
530		}
531		} // END SECTION
532
533		SECTION("geojson quoting") {
534		using mapnik::geometry::geometry_types;
535
536		for (auto const &file : {
537		std::string("test/data/csv/geojson_double_quote_escape.csv")
538		, std::string("test/data/csv/geojson_single_quote.csv")
539		, std::string("test/data/csv/geojson_2x_double_quote_filebakery_style.csv")
540		}) {
541		auto ds = get_csv_ds(file);
542		auto fields = ds->get_descriptor().get_descriptors();
543		require_field_names(fields, {"type"});
544		require_field_types(fields, {mapnik::String});
545
546		auto featureset = all_features(ds);
547		require_geometry(featureset->next(), 1, geometry_types::Point);
548		require_geometry(featureset->next(), 1, geometry_types::LineString);
549		require_geometry(featureset->next(), 1, geometry_types::Polygon);
550		require_geometry(featureset->next(), 1, geometry_types::Polygon);
551		require_geometry(featureset->next(), 4, geometry_types::MultiPoint);
552		require_geometry(featureset->next(), 2, geometry_types::MultiLineString);
553		require_geometry(featureset->next(), 2, geometry_types::MultiPolygon);
554		require_geometry(featureset->next(), 2, geometry_types::MultiPolygon);
555		}
556		} // END SECTION
557
558		SECTION("fewer headers than rows throws") {
559		REQUIRE_THROWS(get_csv_ds("test/data/csv/more_column_values_than_headers.csv"));
560		} // END SECTION
561
562		SECTION("feature ID only incremented for valid rows") {
563		auto ds = get_csv_ds("test/data/csv/warns/feature_id_counting.csv", false);
564		auto fs = all_features(ds);
565
566		// first
567		auto feature = fs->next();
568		REQUIRE(bool(feature));
569		CHECK(feature->id() == 1);
570
571		// second, should have skipped bogus one
572		feature = fs->next();
573		REQUIRE(bool(feature));
574		CHECK(feature->id() == 2);
575
576		feature = fs->next();
577		CHECK(!feature);
578		} // END SECTION
579
580		SECTION("dynamically defining headers") {
581		using ustring = mapnik::value_unicode_string;
582		using row = std::pair<std::string, std::size_t>;
583
584		for (auto const &r : {
585		row{"test/data/csv/fails/needs_headers_two_lines.csv", 2},
586		row{"test/data/csv/fails/needs_headers_one_line.csv", 1},
587		row{"test/data/csv/fails/needs_headers_one_line_no_newline.csv", 1}})
588		{
589		mapnik::parameters params;
590		params["type"] = std::string("csv");
591		params["file"] = r.first;
592		params["headers"] = "x,y,name";
593		auto ds = mapnik::datasource_cache::instance().create(params);
594		REQUIRE(bool(ds));
595		auto fields = ds->get_descriptor().get_descriptors();
596		require_field_names(fields, {"x", "y", "name"});
597		require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String});
598		require_attributes(all_features(ds)->next(), {
599		attr{"x", 0}, attr{"y", 0}, attr{"name", ustring("data_name")} });
600		REQUIRE(count_features(all_features(ds)) == r.second);
601		CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
602		}
603		} // END SECTION
604
605		#pragma GCC diagnostic push
606		#pragma GCC diagnostic ignored "-Wlong-long"
607		SECTION("64bit int fields work") {
608		auto ds = get_csv_ds("test/data/csv/64bit_int.csv");
609		auto fields = ds->get_descriptor().get_descriptors();
610		require_field_names(fields, {"x", "y", "bigint"});
611		require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::Integer});
612
613		auto fs = all_features(ds);
614		auto feature = fs->next();
615		require_attributes(feature, {
616		attr{"x", 0}, attr{"y", 0}, attr{"bigint", 2147483648} });
617
618		feature = fs->next();
619		require_attributes(feature, {
620		attr{"x", 0}, attr{"y", 0}, attr{"bigint", 9223372036854775807ll} });
621		require_attributes(feature, {
622		attr{"x", 0}, attr{"y", 0}, attr{"bigint", 0x7FFFFFFFFFFFFFFFll} });
623		} // END SECTION
624		#pragma GCC diagnostic pop
625
626		SECTION("various number types") {
627		auto ds = get_csv_ds("test/data/csv/number_types.csv");
628		auto fields = ds->get_descriptor().get_descriptors();
629		require_field_names(fields, {"x", "y", "floats"});
630		require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::Double});
631		auto fs = all_features(ds);
632		for (double d : { .0, +.0, 1e-06, -1e-06, 0.000001, 1.234e+16, 1.234e+16 }) {
633		auto feature = fs->next();
634		REQUIRE(bool(feature));
635		CHECK(feature->get("floats").get<mapnik::value_double>() == Approx(d));
636		}
637		} // END SECTION
638
639		SECTION("manually supplied extent") {
640		std::string csv_string("wkt,Name\n");
641		mapnik::parameters params;
642		params["type"] = std::string("csv");
643		params["inline"] = csv_string;
644		params["extent"] = "-180,-90,180,90";
645		auto ds = mapnik::datasource_cache::instance().create(params);
646		REQUIRE(bool(ds));
647		auto box = ds->envelope();
648		CHECK(box.minx() == -180);
649		CHECK(box.miny() == -90);
650		CHECK(box.maxx() == 180);
651		CHECK(box.maxy() == 90);
652		} // END SECTION
653
654		SECTION("inline geojson") {
655		std::string csv_string = "geojson\n'{\"coordinates\":[-92.22568,38.59553],\"type\":\"Point\"}'";
656		mapnik::parameters params;
657		params["type"] = std::string("csv");
658		params["inline"] = csv_string;
659		params["quote"] = "'";
660		auto ds = mapnik::datasource_cache::instance().create(params);
661		REQUIRE(bool(ds));
662
663		auto fields = ds->get_descriptor().get_descriptors();
664		require_field_names(fields, {});
665
666		// TODO: this originally had the following comment:
667		// - re-enable after https://github.com/mapnik/mapnik/issues/2319 is fixed
668		// but that seems to have been merged and tested separately?
669		auto fs = all_features(ds);
670		auto feat = fs->next();
671		CHECK(feature_count(feat->get_geometry()) == 1);
672		} // END SECTION
673		mapnik::logger::instance().set_severity(severity);
674		}
675		} // END TEST CASE

+1173

-0

test/unit/datasource/csv.cpp less more

	0	/*****************************************************************************
	1	*
	2	* This file is part of Mapnik (c++ mapping toolkit)
	3	*
	4	* Copyright (C) 2015 Artem Pavlenko
	5	*
	6	* This library is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU Lesser General Public
	8	* License as published by the Free Software Foundation; either
	9	* version 2.1 of the License, or (at your option) any later version.
	10	*
	11	* This library is distributed in the hope that it will be useful,
	12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	* Lesser General Public License for more details.
	15	*
	16	* You should have received a copy of the GNU Lesser General Public
	17	* License along with this library; if not, write to the Free Software
	18	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	19	*
	20	*****************************************************************************/
	21
	22	#include "catch.hpp"
	23
	24	#include <mapnik/map.hpp>
	25	#include <mapnik/datasource.hpp>
	26	#include <mapnik/datasource_cache.hpp>
	27	#include <mapnik/geometry.hpp>
	28	#include <mapnik/geometry_types.hpp>
	29	#include <mapnik/geometry_type.hpp>
	30	#include <mapnik/expression.hpp>
	31	#include <mapnik/expression_evaluator.hpp>
	32	#include <mapnik/debug.hpp>
	33	#include <mapnik/util/fs.hpp>
	34	#include <boost/filesystem.hpp>
	35	#include <boost/range/iterator_range_core.hpp>
	36	#include <boost/format.hpp>
	37	#include <boost/optional/optional_io.hpp>
	38
	39	#include <iostream>
	40
	41	namespace bfs = boost::filesystem;
	42
	43	namespace {
	44	void add_csv_files(bfs::path dir, std::vector<bfs::path> &csv_files)
	45	{
	46	for (auto const &entry : boost::make_iterator_range(
	47	bfs::directory_iterator(dir), bfs::directory_iterator()))
	48	{
	49	auto path = entry.path();
	50	if (path.extension().native() == ".csv")
	51	{
	52	csv_files.emplace_back(path);
	53	}
	54	}
	55	}
	56
	57	mapnik::datasource_ptr get_csv_ds(std::string const &file_name, bool strict = true)
	58	{
	59	mapnik::parameters params;
	60	params["type"] = std::string("csv");
	61	params["file"] = file_name;
	62	params["strict"] = mapnik::value_bool(strict);
	63	auto ds = mapnik::datasource_cache::instance().create(params);
	64	// require a non-null pointer returned
	65	REQUIRE(ds != nullptr);
	66	return ds;
	67	}
	68
	69	void require_field_names(std::vector<mapnik::attribute_descriptor> const &fields,
	70	std::initializer_list<std::string> const &names)
	71	{
	72	REQUIRE(fields.size() == names.size());
	73	auto itr_a = fields.begin();
	74	auto const end_a = fields.end();
	75	auto itr_b = names.begin();
	76	for (; itr_a != end_a; ++itr_a, ++itr_b)
	77	{
	78	CHECK(itr_a->get_name() == *itr_b);
	79	}
	80	}
	81
	82	void require_field_types(std::vector<mapnik::attribute_descriptor> const &fields,
	83	std::initializer_list<mapnik::eAttributeType> const &types) {
	84	REQUIRE(fields.size() == types.size());
	85	auto itr_a = fields.begin();
	86	auto const end_a = fields.end();
	87	auto itr_b = types.begin();
	88	for (; itr_a != end_a; ++itr_a, ++itr_b) {
	89	CHECK(itr_a->get_type() == *itr_b);
	90	}
	91	}
	92
	93	mapnik::featureset_ptr all_features(mapnik::datasource_ptr ds) {
	94	auto fields = ds->get_descriptor().get_descriptors();
	95	mapnik::query query(ds->envelope());
	96	for (auto const &field : fields) {
	97	query.add_property_name(field.get_name());
	98	}
	99	return ds->features(query);
	100	}
	101
	102	std::size_t count_features(mapnik::featureset_ptr features) {
	103	std::size_t count = 0;
	104	while (features->next()) {
	105	++count;
	106	}
	107	return count;
	108	}
	109
	110	using attr = std::tuple<std::string, mapnik::value>;
	111	void require_attributes(mapnik::feature_ptr feature,
	112	std::initializer_list<attr> const &attrs) {
	113	REQUIRE(bool(feature));
	114	for (auto const &kv : attrs) {
	115	REQUIRE(feature->has_key(std::get<0>(kv)));
	116	CHECK(feature->get(std::get<0>(kv)) == std::get<1>(kv));
	117	}
	118	}
	119
	120	namespace detail {
	121	struct feature_count {
	122	template <typename T>
	123	std::size_t operator()(T const &geom) const {
	124	return mapnik::util::apply_visitor(*this, geom);
	125	}
	126
	127	std::size_t operator()(mapnik::geometry::geometry_empty const &) const {
	128	return 0;
	129	}
	130
	131	template <typename T>
	132	std::size_t operator()(mapnik::geometry::point<T> const &) const {
	133	return 1;
	134	}
	135
	136	template <typename T>
	137	std::size_t operator()(mapnik::geometry::line_string<T> const &) const {
	138	return 1;
	139	}
	140
	141	template <typename T>
	142	std::size_t operator()(mapnik::geometry::polygon<T> const &) const {
	143	return 1;
	144	}
	145
	146	template <typename T>
	147	std::size_t operator()(mapnik::geometry::multi_point<T> const &mp) const {
	148	return mp.size();
	149	}
	150
	151	template <typename T>
	152	std::size_t operator()(mapnik::geometry::multi_line_string<T> const &mls) const {
	153	return mls.size();
	154	}
	155
	156	template <typename T>
	157	std::size_t operator()(mapnik::geometry::multi_polygon<T> const &mp) const {
	158	return mp.size();
	159	}
	160
	161	template <typename T>
	162	std::size_t operator()(mapnik::geometry::geometry_collection<T> const &col) const {
	163	std::size_t sum = 0;
	164	for (auto const &geom : col) {
	165	sum += operator()(geom);
	166	}
	167	return sum;
	168	}
	169	};
	170	} // namespace detail
	171
	172	template <typename T>
	173	std::size_t feature_count(mapnik::geometry::geometry<T> const &g) {
	174	return detail::feature_count()(g);
	175	}
	176
	177	void require_geometry(mapnik::feature_ptr feature,
	178	std::size_t num_parts,
	179	mapnik::geometry::geometry_types type) {
	180	REQUIRE(bool(feature));
	181	CHECK(mapnik::geometry::geometry_type(feature->get_geometry()) == type);
	182	CHECK(feature_count(feature->get_geometry()) == num_parts);
	183	}
	184
	185	int create_disk_index(std::string const& filename, bool silent = true)
	186	{
	187	std::string cmd;
	188	if (std::getenv("DYLD_LIBRARY_PATH") != nullptr)
	189	{
	190	cmd += std::string("export DYLD_LIBRARY_PATH=") + std::getenv("DYLD_LIBRARY_PATH") + " && ";
	191	}
	192	cmd += "mapnik-index " + filename;
	193	if (silent)
	194	{
	195	#ifndef _WINDOWS
	196	cmd += " 2>/dev/null";
	197	#else
	198	cmd += " 2> nul";
	199	#endif
	200	}
	201	return std::system(cmd.c_str());
	202	}
	203
	204	} // anonymous namespace
	205
	206	static const std::string csv_plugin("./plugins/input/csv.input");
	207
	208	const bool registered = mapnik::datasource_cache::instance().register_datasources(csv_plugin);
	209
	210	TEST_CASE("csv") {
	211
	212	if (mapnik::util::exists(csv_plugin))
	213	{
	214	REQUIRE(registered);
	215	// make the tests silent since we intentially test error conditions that are noisy
	216	auto const severity = mapnik::logger::instance().get_severity();
	217	mapnik::logger::instance().set_severity(mapnik::logger::none);
	218
	219	// check the CSV datasource is loaded
	220	const std::vector<std::string> plugin_names =
	221	mapnik::datasource_cache::instance().plugin_names();
	222	const bool have_csv_plugin =
	223	std::find(plugin_names.begin(), plugin_names.end(), "csv") != plugin_names.end();
	224
	225	SECTION("CSV I/O errors")
	226	{
	227	std::string filename = "does_not_exist.csv";
	228	for (auto create_index : { true, false })
	229	{
	230	if (create_index)
	231	{
	232	int ret = create_disk_index(filename);
	233	int ret_posix = (ret >> 8) & 0x000000ff;
	234	INFO(ret);
	235	INFO(ret_posix);
	236	// index wont be created
	237	CHECK(!mapnik::util::exists(filename + ".index"));
	238	}
	239	mapnik::parameters params;
	240	params["type"] = "csv";
	241	params["file"] = filename;
	242	REQUIRE_THROWS(mapnik::datasource_cache::instance().create(params));
	243	}
	244	}
	245
	246	SECTION("broken files")
	247	{
	248	for (auto create_index : { false, true })
	249	{
	250	if (have_csv_plugin)
	251	{
	252	std::vector<bfs::path> broken;
	253	add_csv_files("test/data/csv/fails", broken);
	254	add_csv_files("test/data/csv/warns", broken);
	255	broken.emplace_back("test/data/csv/fails/does_not_exist.csv");
	256
	257	for (auto const& path : broken)
	258	{
	259	bool require_fail = true;
	260	if (create_index)
	261	{
	262	int ret = create_disk_index(path.native());
	263	int ret_posix = (ret >> 8) & 0x000000ff;
	264	INFO(ret);
	265	INFO(ret_posix);
	266	require_fail = (path.native() == "test/data/csv/warns/feature_id_counting.csv") ? false : true;
	267	if (!require_fail)
	268	{
	269	REQUIRE(mapnik::util::exists(path.native() + ".index"));
	270	}
	271	}
	272	INFO(path);
	273	if (require_fail)
	274	{
	275	REQUIRE_THROWS(get_csv_ds(path.native()));
	276	}
	277	else
	278	{
	279	CHECK(bool(get_csv_ds(path.native())));
	280	}
	281	if (mapnik::util::exists(path.native() + ".index"))
	282	{
	283	CHECK(mapnik::util::remove(path.native() + ".index"));
	284	}
	285	}
	286	}
	287	}
	288	} // END SECTION
	289
	290	SECTION("good files")
	291	{
	292	if (have_csv_plugin)
	293	{
	294	std::vector<bfs::path> good;
	295	add_csv_files("test/data/csv", good);
	296	add_csv_files("test/data/csv/warns", good);
	297
	298	for (auto const& path : good)
	299	{
	300	// cleanup in the case of a failed previous run
	301	if (mapnik::util::exists(path.native() + ".index"))
	302	{
	303	boost::filesystem::remove(path.native() + ".index");
	304	}
	305	for (auto create_index : { false, true })
	306	{
	307	if (create_index)
	308	{
	309	int ret = create_disk_index(path.native());
	310	int ret_posix = (ret >> 8) & 0x000000ff;
	311	INFO(ret);
	312	INFO(ret_posix);
	313	if (path.native() != "test/data/csv/more_headers_than_column_values.csv") // mapnik-index won't create *.index for 0 features
	314	{
	315	CHECK(mapnik::util::exists(path.native() + ".index"));
	316	}
	317	}
	318	auto ds = get_csv_ds(path.native(), false);
	319	// require a non-null pointer returned
	320	REQUIRE(bool(ds));
	321	if (mapnik::util::exists(path.native() + ".index"))
	322	{
	323	CHECK(mapnik::util::remove(path.native() + ".index"));
	324	}
	325	}
	326	}
	327	}
	328	} // END SECTION
	329
	330	SECTION("lon/lat detection")
	331	{
	332	for (auto create_index : { false, true })
	333	{
	334	for (auto const& lon_name : {std::string("lon"), std::string("lng")})
	335	{
	336	std::string filename = (boost::format("test/data/csv/%1%_lat.csv") % lon_name).str();
	337	// cleanup in the case of a failed previous run
	338	if (mapnik::util::exists(filename + ".index"))
	339	{
	340	boost::filesystem::remove(filename + ".index");
	341	}
	342	if (create_index)
	343	{
	344	int ret = create_disk_index(filename);
	345	int ret_posix = (ret >> 8) & 0x000000ff;
	346	INFO(ret);
	347	INFO(ret_posix);
	348	CHECK(mapnik::util::exists(filename + ".index"));
	349	}
	350	auto ds = get_csv_ds(filename);
	351	auto fields = ds->get_descriptor().get_descriptors();
	352	require_field_names(fields, {lon_name, "lat"});
	353	require_field_types(fields, {mapnik::Integer, mapnik::Integer});
	354
	355	CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
	356
	357	mapnik::query query(ds->envelope());
	358	for (auto const &field : fields)
	359	{
	360	query.add_property_name(field.get_name());
	361	}
	362	auto features = ds->features(query);
	363	auto feature = features->next();
	364
	365	require_attributes(feature, {
	366	attr { lon_name, mapnik::value_integer(0) },
	367	attr { "lat", mapnik::value_integer(0) }
	368	});
	369	if (mapnik::util::exists(filename + ".index"))
	370	{
	371	boost::filesystem::remove(filename + ".index");
	372	}
	373	}
	374	}
	375	} // END SECTION
	376
	377	SECTION("type detection")
	378	{
	379	for (auto create_index : { false, true })
	380	{
	381	std::string filename = "test/data/csv/nypd.csv";
	382	// cleanup in the case of a failed previous run
	383	if (mapnik::util::exists(filename + ".index"))
	384	{
	385	boost::filesystem::remove(filename + ".index");
	386	}
	387	if (create_index)
	388	{
	389	int ret = create_disk_index(filename);
	390	int ret_posix = (ret >> 8) & 0x000000ff;
	391	INFO(ret);
	392	INFO(ret_posix);
	393	CHECK(mapnik::util::exists(filename + ".index"));
	394	}
	395	auto ds = get_csv_ds(filename);
	396	auto fields = ds->get_descriptor().get_descriptors();
	397	require_field_names(fields, {"Precinct", "Phone", "Address", "City", "geo_longitude", "geo_latitude", "geo_accuracy"});
	398	require_field_types(fields, {mapnik::String, mapnik::String, mapnik::String, mapnik::String, mapnik::Double, mapnik::Double, mapnik::String});
	399
	400	CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
	401	CHECK(count_features(all_features(ds)) == 2);
	402
	403	auto feature = all_features(ds)->next();
	404	require_attributes(feature, {
	405	attr { "City", mapnik::value_unicode_string("New York, NY") }
	406	, attr { "geo_accuracy", mapnik::value_unicode_string("house") }
	407	, attr { "Phone", mapnik::value_unicode_string("(212) 334-0711") }
	408	, attr { "Address", mapnik::value_unicode_string("19 Elizabeth Street") }
	409	, attr { "Precinct", mapnik::value_unicode_string("5th Precinct") }
	410	, attr { "geo_longitude", mapnik::value_integer(-70) }
	411	, attr { "geo_latitude", mapnik::value_integer(40) }
	412	});
	413	if (mapnik::util::exists(filename + ".index"))
	414	{
	415	boost::filesystem::remove(filename + ".index");
	416	}
	417	}
	418	} // END SECTION
	419
	420	SECTION("skipping blank rows")
	421	{
	422	for (auto create_index : { false, true })
	423	{
	424	std::string filename = "test/data/csv/blank_rows.csv";
	425	// cleanup in the case of a failed previous run
	426	if (mapnik::util::exists(filename + ".index"))
	427	{
	428	boost::filesystem::remove(filename + ".index");
	429	}
	430	if (create_index)
	431	{
	432	int ret = create_disk_index(filename);
	433	int ret_posix = (ret >> 8) & 0x000000ff;
	434	INFO(ret);
	435	INFO(ret_posix);
	436	CHECK(mapnik::util::exists(filename + ".index"));
	437	}
	438	auto ds = get_csv_ds(filename);
	439	auto fields = ds->get_descriptor().get_descriptors();
	440	require_field_names(fields, {"x", "y", "name"});
	441	require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String});
	442	CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
	443	CHECK(count_features(all_features(ds)) == 2);
	444	if (mapnik::util::exists(filename + ".index"))
	445	{
	446	boost::filesystem::remove(filename + ".index");
	447	}
	448	}
	449	} // END SECTION
	450
	451	SECTION("empty rows")
	452	{
	453	for (auto create_index : { false, true })
	454	{
	455	std::string filename = "test/data/csv/empty_rows.csv";
	456	// cleanup in the case of a failed previous run
	457	if (mapnik::util::exists(filename + ".index"))
	458	{
	459	boost::filesystem::remove(filename + ".index");
	460	}
	461	if (create_index)
	462	{
	463	int ret = create_disk_index(filename);
	464	int ret_posix = (ret >> 8) & 0x000000ff;
	465	INFO(ret);
	466	INFO(ret_posix);
	467	CHECK(mapnik::util::exists(filename + ".index"));
	468	}
	469	auto ds = get_csv_ds(filename);
	470
	471	auto fields = ds->get_descriptor().get_descriptors();
	472	require_field_names(fields, {"x", "y", "text", "date", "integer", "boolean", "float", "time", "datetime", "empty_column"});
	473	require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String, mapnik::String,
	474	mapnik::Integer, mapnik::Boolean, mapnik::Double, mapnik::String, mapnik::String, mapnik::String});
	475	CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
	476	CHECK(count_features(all_features(ds)) == 4);
	477
	478	auto featureset = all_features(ds);
	479	auto feature = featureset->next();
	480	require_attributes(feature, {
	481	attr { "x", mapnik::value_integer(0) }
	482	, attr { "empty_column", mapnik::value_unicode_string("") }
	483	, attr { "text", mapnik::value_unicode_string("a b") }
	484	, attr { "float", mapnik::value_double(1.0) }
	485	, attr { "datetime", mapnik::value_unicode_string("1971-01-01T04:14:00") }
	486	, attr { "y", mapnik::value_integer(0) }
	487	, attr { "boolean", mapnik::value_bool(true) }
	488	, attr { "time", mapnik::value_unicode_string("04:14:00") }
	489	, attr { "date", mapnik::value_unicode_string("1971-01-01") }
	490	, attr { "integer", mapnik::value_integer(40) }
	491	});
	492
	493	while (bool(feature = featureset->next())) {
	494	CHECK(feature->size() == 10);
	495	CHECK(feature->get("empty_column") == mapnik::value_unicode_string(""));
	496	}
	497	if (mapnik::util::exists(filename + ".index"))
	498	{
	499	boost::filesystem::remove(filename + ".index");
	500	}
	501	}
	502	} // END SECTION
	503
	504	SECTION("slashes")
	505	{
	506	for (auto create_index : { false, true })
	507	{
	508	std::string filename = "test/data/csv/has_attributes_with_slashes.csv";
	509	// cleanup in the case of a failed previous run
	510	if (mapnik::util::exists(filename + ".index"))
	511	{
	512	boost::filesystem::remove(filename + ".index");
	513	}
	514	if (create_index)
	515	{
	516	int ret = create_disk_index(filename);
	517	int ret_posix = (ret >> 8) & 0x000000ff;
	518	INFO(ret);
	519	INFO(ret_posix);
	520	CHECK(mapnik::util::exists(filename + ".index"));
	521	}
	522	auto ds = get_csv_ds(filename);
	523	auto fields = ds->get_descriptor().get_descriptors();
	524	require_field_names(fields, {"x", "y", "name"});
	525	// NOTE: y column is integer, even though a double value is used below in the test?
	526	require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String});
	527
	528	auto featureset = all_features(ds);
	529	require_attributes(featureset->next(), {
	530	attr{"x", 0}
	531	, attr{"y", 0}
	532	, attr{"name", mapnik::value_unicode_string("a/a") } });
	533	require_attributes(featureset->next(), {
	534	attr{"x", 1}
	535	, attr{"y", 4}
	536	, attr{"name", mapnik::value_unicode_string("b/b") } });
	537	require_attributes(featureset->next(), {
	538	attr{"x", 10}
	539	, attr{"y", 2.5}
	540	, attr{"name", mapnik::value_unicode_string("c/c") } });
	541	if (mapnik::util::exists(filename + ".index"))
	542	{
	543	boost::filesystem::remove(filename + ".index");
	544	}
	545	}
	546	} // END SECTION
	547
	548	SECTION("wkt field")
	549	{
	550	for (auto create_index : { false, true })
	551	{
	552	std::string filename = "test/data/csv/wkt.csv";
	553	// cleanup in the case of a failed previous run
	554	if (mapnik::util::exists(filename + ".index"))
	555	{
	556	boost::filesystem::remove(filename + ".index");
	557	}
	558	if (create_index)
	559	{
	560	int ret = create_disk_index(filename);
	561	int ret_posix = (ret >> 8) & 0x000000ff;
	562	INFO(ret);
	563	INFO(ret_posix);
	564	CHECK(mapnik::util::exists(filename + ".index"));
	565	}
	566	using mapnik::geometry::geometry_types;
	567	auto ds = get_csv_ds(filename);
	568	auto fields = ds->get_descriptor().get_descriptors();
	569	require_field_names(fields, {"type"});
	570	require_field_types(fields, {mapnik::String});
	571
	572	auto featureset = all_features(ds);
	573	require_geometry(featureset->next(), 1, geometry_types::Point);
	574	require_geometry(featureset->next(), 1, geometry_types::LineString);
	575	require_geometry(featureset->next(), 1, geometry_types::Polygon);
	576	require_geometry(featureset->next(), 1, geometry_types::Polygon);
	577	require_geometry(featureset->next(), 4, geometry_types::MultiPoint);
	578	require_geometry(featureset->next(), 2, geometry_types::MultiLineString);
	579	require_geometry(featureset->next(), 2, geometry_types::MultiPolygon);
	580	require_geometry(featureset->next(), 2, geometry_types::MultiPolygon);
	581	if (mapnik::util::exists(filename + ".index"))
	582	{
	583	boost::filesystem::remove(filename + ".index");
	584	}
	585	}
	586	} // END SECTION
	587
	588	SECTION("handling of missing header")
	589	{
	590	for (auto create_index : { false, true })
	591	{
	592	std::string filename = "test/data/csv/missing_header.csv";
	593	// cleanup in the case of a failed previous run
	594	if (mapnik::util::exists(filename + ".index"))
	595	{
	596	boost::filesystem::remove(filename + ".index");
	597	}
	598	if (create_index)
	599	{
	600	int ret = create_disk_index(filename);
	601	int ret_posix = (ret >> 8) & 0x000000ff;
	602	INFO(ret);
	603	INFO(ret_posix);
	604	CHECK(mapnik::util::exists(filename + ".index"));
	605	}
	606	// TODO: does this mean 'missing_header.csv' should be in the warnings
	607	// subdirectory, since it doesn't work in strict mode?
	608	auto ds = get_csv_ds(filename, false);
	609	auto fields = ds->get_descriptor().get_descriptors();
	610	require_field_names(fields, {"one", "two", "x", "y", "_4", "aftermissing"});
	611	auto feature = all_features(ds)->next();
	612	REQUIRE(feature);
	613	REQUIRE(feature->has_key("_4"));
	614	CHECK(feature->get("_4") == mapnik::value_unicode_string("missing"));
	615	if (mapnik::util::exists(filename + ".index"))
	616	{
	617	boost::filesystem::remove(filename + ".index");
	618	}
	619	}
	620	} // END SECTION
	621
	622	SECTION("handling of headers that are numbers")
	623	{
	624	for (auto create_index : { false, true })
	625	{
	626	std::string filename = "test/data/csv/numbers_for_headers.csv";
	627	// cleanup in the case of a failed previous run
	628	if (mapnik::util::exists(filename + ".index"))
	629	{
	630	boost::filesystem::remove(filename + ".index");
	631	}
	632	if (create_index)
	633	{
	634	int ret = create_disk_index(filename);
	635	int ret_posix = (ret >> 8) & 0x000000ff;
	636	INFO(ret);
	637	INFO(ret_posix);
	638	CHECK(mapnik::util::exists(filename + ".index"));
	639	}
	640	auto ds = get_csv_ds(filename);
	641	auto fields = ds->get_descriptor().get_descriptors();
	642	require_field_names(fields, {"x", "y", "1990", "1991", "1992"});
	643	auto feature = all_features(ds)->next();
	644	require_attributes(feature, {
	645	attr{"x", 0}
	646	, attr{"y", 0}
	647	, attr{"1990", 1}
	648	, attr{"1991", 2}
	649	, attr{"1992", 3}
	650	});
	651	auto expression = mapnik::parse_expression("[1991]=2");
	652	REQUIRE(bool(expression));
	653	auto value = mapnik::util::apply_visitor(
	654	mapnik::evaluate<mapnik::feature_impl, mapnik::value_type, mapnik::attributes>(
	655	feature, mapnik::attributes()), expression);
	656	CHECK(value == true);
	657	if (mapnik::util::exists(filename + ".index"))
	658	{
	659	boost::filesystem::remove(filename + ".index");
	660	}
	661	}
	662	} // END SECTION
	663
	664	SECTION("quoted numbers")
	665	{
	666	using ustring = mapnik::value_unicode_string;
	667	for (auto create_index : { false, true })
	668	{
	669	std::string filename = "test/data/csv/quoted_numbers.csv";
	670	// cleanup in the case of a failed previous run
	671	if (mapnik::util::exists(filename + ".index"))
	672	{
	673	boost::filesystem::remove(filename + ".index");
	674	}
	675	if (create_index)
	676	{
	677	int ret = create_disk_index(filename);
	678	int ret_posix = (ret >> 8) & 0x000000ff;
	679	INFO(ret);
	680	INFO(ret_posix);
	681	CHECK(mapnik::util::exists(filename + ".index"));
	682	}
	683	auto ds = get_csv_ds(filename);
	684	auto fields = ds->get_descriptor().get_descriptors();
	685	require_field_names(fields, {"x", "y", "label"});
	686	auto featureset = all_features(ds);
	687
	688	require_attributes(featureset->next(), {
	689	attr{"x", 0}, attr{"y", 0}, attr{"label", ustring("0,0") } });
	690	require_attributes(featureset->next(), {
	691	attr{"x", 5}, attr{"y", 5}, attr{"label", ustring("5,5") } });
	692	require_attributes(featureset->next(), {
	693	attr{"x", 0}, attr{"y", 5}, attr{"label", ustring("0,5") } });
	694	require_attributes(featureset->next(), {
	695	attr{"x", 5}, attr{"y", 0}, attr{"label", ustring("5,0") } });
	696	require_attributes(featureset->next(), {
	697	attr{"x", 2.5}, attr{"y", 2.5}, attr{"label", ustring("2.5,2.5") } });
	698	if (mapnik::util::exists(filename + ".index"))
	699	{
	700	boost::filesystem::remove(filename + ".index");
	701	}
	702	}
	703	} // END SECTION
	704
	705	SECTION("reading newlines")
	706	{
	707	for (auto create_index : { false, true })
	708	{
	709	for (auto const& platform : {std::string("windows"), std::string("mac")})
	710	{
	711	std::string filename = (boost::format("test/data/csv/%1%_newlines.csv") % platform).str();
	712	// cleanup in the case of a failed previous run
	713	if (mapnik::util::exists(filename + ".index"))
	714	{
	715	boost::filesystem::remove(filename + ".index");
	716	}
	717	if (create_index)
	718	{
	719	int ret = create_disk_index(filename);
	720	int ret_posix = (ret >> 8) & 0x000000ff;
	721	INFO(ret);
	722	INFO(ret_posix);
	723	CHECK(mapnik::util::exists(filename + ".index"));
	724	}
	725	auto ds = get_csv_ds(filename);
	726	auto fields = ds->get_descriptor().get_descriptors();
	727	require_field_names(fields, {"x", "y", "z"});
	728	require_attributes(all_features(ds)->next(), {
	729	attr{"x", 1}, attr{"y", 10}, attr{"z", 9999.9999} });
	730	if (mapnik::util::exists(filename + ".index"))
	731	{
	732	boost::filesystem::remove(filename + ".index");
	733	}
	734	}
	735	}
	736	} // END SECTION
	737
	738	SECTION("mixed newlines")
	739	{
	740	using ustring = mapnik::value_unicode_string;
	741	for (auto create_index : { false, true })
	742	{
	743	for (auto const& filename : {
	744	std::string("test/data/csv/mac_newlines_with_unix_inline.csv")
	745	, std::string("test/data/csv/mac_newlines_with_unix_inline_escaped.csv")
	746	, std::string("test/data/csv/windows_newlines_with_unix_inline.csv")
	747	, std::string("test/data/csv/windows_newlines_with_unix_inline_escaped.csv")
	748	})
	749	{
	750	// cleanup in the case of a failed previous run
	751	if (mapnik::util::exists(filename + ".index"))
	752	{
	753	boost::filesystem::remove(filename + ".index");
	754	}
	755	if (create_index)
	756	{
	757	int ret = create_disk_index(filename);
	758	int ret_posix = (ret >> 8) & 0x000000ff;
	759	INFO(ret);
	760	INFO(ret_posix);
	761	CHECK(mapnik::util::exists(filename + ".index"));
	762	}
	763	auto ds = get_csv_ds(filename);
	764	auto fields = ds->get_descriptor().get_descriptors();
	765	require_field_names(fields, {"x", "y", "line"});
	766	require_attributes(all_features(ds)->next(), {
	767	attr{"x", 0}, attr{"y", 0}
	768	, attr{"line", ustring("many\n lines\n of text\n with unix newlines")} });
	769	if (mapnik::util::exists(filename + ".index"))
	770	{
	771	boost::filesystem::remove(filename + ".index");
	772	}
	773	}
	774	}
	775	} // END SECTION
	776
	777	SECTION("tabs")
	778	{
	779	for (auto create_index : { false, true })
	780	{
	781	std::string filename = "test/data/csv/tabs_in_csv.csv";
	782	if (mapnik::util::exists(filename + ".index"))
	783	{
	784	boost::filesystem::remove(filename + ".index");
	785	}
	786	if (create_index)
	787	{
	788	int ret = create_disk_index(filename);
	789	int ret_posix = (ret >> 8) & 0x000000ff;
	790	INFO(ret);
	791	INFO(ret_posix);
	792	CHECK(mapnik::util::exists(filename + ".index"));
	793	}
	794	auto ds = get_csv_ds(filename);
	795	auto fields = ds->get_descriptor().get_descriptors();
	796	require_field_names(fields, {"x", "y", "z"});
	797	require_attributes(all_features(ds)->next(), {
	798	attr{"x", -122}, attr{"y", 48}, attr{"z", 0} });
	799	if (mapnik::util::exists(filename + ".index"))
	800	{
	801	boost::filesystem::remove(filename + ".index");
	802	}
	803	}
	804	} // END SECTION
	805
	806	SECTION("separators")
	807	{
	808	using ustring = mapnik::value_unicode_string;
	809	for (auto const& filename : {
	810	std::string("test/data/csv/pipe_delimiters.csv")
	811	, std::string("test/data/csv/semicolon_delimiters.csv")
	812	})
	813	{
	814	for (auto create_index : { false, true })
	815	{
	816	// cleanup in the case of a failed previous run
	817	if (mapnik::util::exists(filename + ".index"))
	818	{
	819	boost::filesystem::remove(filename + ".index");
	820	}
	821	if (create_index)
	822	{
	823	int ret = create_disk_index(filename);
	824	int ret_posix = (ret >> 8) & 0x000000ff;
	825	INFO(ret);
	826	INFO(ret_posix);
	827	CHECK(mapnik::util::exists(filename + ".index"));
	828	}
	829	auto ds = get_csv_ds(filename);
	830	auto fields = ds->get_descriptor().get_descriptors();
	831	require_field_names(fields, {"x", "y", "z"});
	832	require_attributes(all_features(ds)->next(), {
	833	attr{"x", 0}, attr{"y", 0}, attr{"z", ustring("hello")} });
	834	if (mapnik::util::exists(filename + ".index"))
	835	{
	836	boost::filesystem::remove(filename + ".index");
	837	}
	838	}
	839	}
	840	} // END SECTION
	841
	842	SECTION("null and bool keywords are empty strings")
	843	{
	844	using ustring = mapnik::value_unicode_string;
	845	std::string filename = "test/data/csv/nulls_and_booleans_as_strings.csv";
	846	for (auto create_index : { false, true })
	847	{
	848	// cleanup in the case of a failed previous run
	849	if (mapnik::util::exists(filename + ".index"))
	850	{
	851	boost::filesystem::remove(filename + ".index");
	852	}
	853	if (create_index)
	854	{
	855	int ret = create_disk_index(filename);
	856	int ret_posix = (ret >> 8) & 0x000000ff;
	857	INFO(ret);
	858	INFO(ret_posix);
	859	CHECK(mapnik::util::exists(filename + ".index"));
	860	}
	861	auto ds = get_csv_ds(filename);
	862	auto fields = ds->get_descriptor().get_descriptors();
	863	require_field_names(fields, {"x", "y", "null", "boolean"});
	864	require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String, mapnik::Boolean});
	865
	866	auto featureset = all_features(ds);
	867	require_attributes(featureset->next(), {
	868	attr{"x", 0}, attr{"y", 0}, attr{"null", ustring("null")}, attr{"boolean", true}});
	869	require_attributes(featureset->next(), {
	870	attr{"x", 0}, attr{"y", 0}, attr{"null", ustring("")}, attr{"boolean", false}});
	871
	872	if (mapnik::util::exists(filename + ".index"))
	873	{
	874	boost::filesystem::remove(filename + ".index");
	875	}
	876	}
	877	} // END SECTION
	878
	879	SECTION("nonexistent query fields throw")
	880	{
	881	std::string filename = "test/data/csv/lon_lat.csv";
	882	for (auto create_index : { false, true })
	883	{
	884	// cleanup in the case of a failed previous run
	885	if (mapnik::util::exists(filename + ".index"))
	886	{
	887	boost::filesystem::remove(filename + ".index");
	888	}
	889	if (create_index)
	890	{
	891	int ret = create_disk_index(filename);
	892	int ret_posix = (ret >> 8) & 0x000000ff;
	893	INFO(ret);
	894	INFO(ret_posix);
	895	CHECK(mapnik::util::exists(filename + ".index"));
	896	}
	897	auto ds = get_csv_ds(filename);
	898	auto fields = ds->get_descriptor().get_descriptors();
	899	require_field_names(fields, {"lon", "lat"});
	900	require_field_types(fields, {mapnik::Integer, mapnik::Integer});
	901
	902	mapnik::query query(ds->envelope());
	903	for (auto const &field : fields)
	904	{
	905	query.add_property_name(field.get_name());
	906	}
	907	// also add an invalid one, triggering throw
	908	query.add_property_name("bogus");
	909	REQUIRE_THROWS(ds->features(query));
	910	if (mapnik::util::exists(filename + ".index"))
	911	{
	912	boost::filesystem::remove(filename + ".index");
	913	}
	914	}
	915	} // END SECTION
	916
	917	SECTION("leading zeros mean strings")
	918	{
	919	using ustring = mapnik::value_unicode_string;
	920	std::string filename = "test/data/csv/leading_zeros.csv";
	921	for (auto create_index : { false, true })
	922	{
	923	// cleanup in the case of a failed previous run
	924	if (mapnik::util::exists(filename + ".index"))
	925	{
	926	boost::filesystem::remove(filename + ".index");
	927	}
	928	if (create_index)
	929	{
	930	int ret = create_disk_index(filename);
	931	int ret_posix = (ret >> 8) & 0x000000ff;
	932	INFO(ret);
	933	INFO(ret_posix);
	934	CHECK(mapnik::util::exists(filename + ".index"));
	935	}
	936	auto ds = get_csv_ds(filename);
	937	auto fields = ds->get_descriptor().get_descriptors();
	938	require_field_names(fields, {"x", "y", "fips"});
	939	require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String});
	940
	941	auto featureset = all_features(ds);
	942	require_attributes(featureset->next(), {
	943	attr{"x", 0}, attr{"y", 0}, attr{"fips", ustring("001")}});
	944	require_attributes(featureset->next(), {
	945	attr{"x", 0}, attr{"y", 0}, attr{"fips", ustring("003")}});
	946	require_attributes(featureset->next(), {
	947	attr{"x", 0}, attr{"y", 0}, attr{"fips", ustring("005")}});
	948	if (mapnik::util::exists(filename + ".index"))
	949	{
	950	boost::filesystem::remove(filename + ".index");
	951	}
	952	}
	953	} // END SECTION
	954
	955	SECTION("advanced geometry detection")
	956	{
	957	using row = std::pair<std::string, mapnik::datasource_geometry_t>;
	958	for (row r : {
	959	row{"point", mapnik::datasource_geometry_t::Point}
	960	, row{"poly", mapnik::datasource_geometry_t::Polygon}
	961	, row{"multi_poly", mapnik::datasource_geometry_t::Polygon}
	962	, row{"line", mapnik::datasource_geometry_t::LineString}
	963	}) {
	964	std::string file_name = (boost::format("test/data/csv/%1%_wkt.csv") % r.first).str();
	965	auto ds = get_csv_ds(file_name);
	966	CHECK(ds->get_geometry_type() == r.second);
	967	}
	968	} // END SECTION
	969
	970	SECTION("creation of CSV from in-memory strings")
	971	{
	972	using ustring = mapnik::value_unicode_string;
	973
	974	for (auto const &name : {std::string("Winthrop, WA"), std::string(u8"Qu\u00e9bec")}) {
	975	std::string csv_string =
	976	(boost::format(
	977	"wkt,Name\n"
	978	"\"POINT (120.15 48.47)\",\"%1%\"\n"
	979	) % name).str();
	980
	981	mapnik::parameters params;
	982	params["type"] = std::string("csv");
	983	params["inline"] = csv_string;
	984	auto ds = mapnik::datasource_cache::instance().create(params);
	985	REQUIRE(bool(ds));
	986
	987	auto feature = all_features(ds)->next();
	988	REQUIRE(bool(feature));
	989	REQUIRE(feature->has_key("Name"));
	990	CHECK(feature->get("Name") == ustring(name.c_str()));
	991	}
	992	} // END SECTION
	993
	994	SECTION("creation of CSV from in-memory strings with bogus headers")
	995	{
	996	mapnik::parameters params;
	997	params["type"] = std::string("csv");
	998
	999	// should throw
	1000	params["inline"] = "latitude, longtitude, Name\n" // misspellt (!)
	1001	"120.15,48.47,Winhrop";
	1002	REQUIRE_THROWS(mapnik::datasource_cache::instance().create(params));
	1003
	1004	// should throw
	1005	params["strict"] = true;
	1006	params["inline"] = "latitude, longitude\n" // -- missing header
	1007	"120.15,48.47,Winhrop";
	1008	REQUIRE_THROWS(mapnik::datasource_cache::instance().create(params));
	1009
	1010	// should not throw
	1011	params["strict"] = false;
	1012	params["inline"] = "latitude, longitude,Name\n"
	1013	"0,0,Unknown, extra bogus field\n"
	1014	"120.15,48.47,Winhrop\n";
	1015	auto ds = mapnik::datasource_cache::instance().create(params);
	1016	REQUIRE(bool(ds));
	1017	REQUIRE(ds->envelope() == mapnik::box2d<double>(48.47,120.15,48.47,120.15));
	1018	auto feature = all_features(ds)->next();
	1019	REQUIRE(bool(feature));
	1020	REQUIRE(feature->has_key("Name"));
	1021
	1022	// should throw
	1023	params["strict"] = false;
	1024	params["inline"] = "x, Name\n" // -- missing required geometry header
	1025	"120.15,Winhrop";
	1026	REQUIRE_THROWS(mapnik::datasource_cache::instance().create(params));
	1027
	1028	} // END SECTION
	1029
	1030	SECTION("geojson quoting") {
	1031	using mapnik::geometry::geometry_types;
	1032
	1033	for (auto const &file : {
	1034	std::string("test/data/csv/geojson_double_quote_escape.csv")
	1035	, std::string("test/data/csv/geojson_single_quote.csv")
	1036	, std::string("test/data/csv/geojson_2x_double_quote_filebakery_style.csv")
	1037	}) {
	1038	auto ds = get_csv_ds(file);
	1039	auto fields = ds->get_descriptor().get_descriptors();
	1040	require_field_names(fields, {"type"});
	1041	require_field_types(fields, {mapnik::String});
	1042
	1043	auto featureset = all_features(ds);
	1044	require_geometry(featureset->next(), 1, geometry_types::Point);
	1045	require_geometry(featureset->next(), 1, geometry_types::LineString);
	1046	require_geometry(featureset->next(), 1, geometry_types::Polygon);
	1047	require_geometry(featureset->next(), 1, geometry_types::Polygon);
	1048	require_geometry(featureset->next(), 4, geometry_types::MultiPoint);
	1049	require_geometry(featureset->next(), 2, geometry_types::MultiLineString);
	1050	require_geometry(featureset->next(), 2, geometry_types::MultiPolygon);
	1051	require_geometry(featureset->next(), 2, geometry_types::MultiPolygon);
	1052	}
	1053	} // END SECTION
	1054
	1055	SECTION("fewer headers than rows throws") {
	1056	REQUIRE_THROWS(get_csv_ds("test/data/csv/more_column_values_than_headers.csv"));
	1057	} // END SECTION
	1058
	1059	SECTION("feature ID only incremented for valid rows") {
	1060	auto ds = get_csv_ds("test/data/csv/warns/feature_id_counting.csv", false);
	1061	auto fs = all_features(ds);
	1062
	1063	// first
	1064	auto feature = fs->next();
	1065	REQUIRE(bool(feature));
	1066	CHECK(feature->id() == 1);
	1067
	1068	// second, should have skipped bogus one
	1069	feature = fs->next();
	1070	REQUIRE(bool(feature));
	1071	CHECK(feature->id() == 2);
	1072
	1073	feature = fs->next();
	1074	CHECK(!feature);
	1075	} // END SECTION
	1076
	1077	SECTION("dynamically defining headers") {
	1078	using ustring = mapnik::value_unicode_string;
	1079	using row = std::pair<std::string, std::size_t>;
	1080
	1081	for (auto const &r : {
	1082	row{"test/data/csv/fails/needs_headers_two_lines.csv", 2},
	1083	row{"test/data/csv/fails/needs_headers_one_line.csv", 1},
	1084	row{"test/data/csv/fails/needs_headers_one_line_no_newline.csv", 1}})
	1085	{
	1086	mapnik::parameters params;
	1087	params["type"] = std::string("csv");
	1088	params["file"] = r.first;
	1089	params["headers"] = "x,y,name";
	1090	auto ds = mapnik::datasource_cache::instance().create(params);
	1091	REQUIRE(bool(ds));
	1092	auto fields = ds->get_descriptor().get_descriptors();
	1093	require_field_names(fields, {"x", "y", "name"});
	1094	require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String});
	1095	require_attributes(all_features(ds)->next(), {
	1096	attr{"x", 0}, attr{"y", 0}, attr{"name", ustring("data_name")} });
	1097	REQUIRE(count_features(all_features(ds)) == r.second);
	1098	CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
	1099	}
	1100	} // END SECTION
	1101
	1102	#pragma GCC diagnostic push
	1103	#pragma GCC diagnostic ignored "-Wlong-long"
	1104	SECTION("64bit int fields work") {
	1105	auto ds = get_csv_ds("test/data/csv/64bit_int.csv");
	1106	auto fields = ds->get_descriptor().get_descriptors();
	1107	require_field_names(fields, {"x", "y", "bigint"});
	1108	require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::Integer});
	1109
	1110	auto fs = all_features(ds);
	1111	auto feature = fs->next();
	1112	require_attributes(feature, {
	1113	attr{"x", 0}, attr{"y", 0}, attr{"bigint", 2147483648} });
	1114
	1115	feature = fs->next();
	1116	require_attributes(feature, {
	1117	attr{"x", 0}, attr{"y", 0}, attr{"bigint", 9223372036854775807ll} });
	1118	require_attributes(feature, {
	1119	attr{"x", 0}, attr{"y", 0}, attr{"bigint", 0x7FFFFFFFFFFFFFFFll} });
	1120	} // END SECTION
	1121	#pragma GCC diagnostic pop
	1122
	1123	SECTION("various number types") {
	1124	auto ds = get_csv_ds("test/data/csv/number_types.csv");
	1125	auto fields = ds->get_descriptor().get_descriptors();
	1126	require_field_names(fields, {"x", "y", "floats"});
	1127	require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::Double});
	1128	auto fs = all_features(ds);
	1129	for (double d : { .0, +.0, 1e-06, -1e-06, 0.000001, 1.234e+16, 1.234e+16 }) {
	1130	auto feature = fs->next();
	1131	REQUIRE(bool(feature));
	1132	CHECK(feature->get("floats").get<mapnik::value_double>() == Approx(d));
	1133	}
	1134	} // END SECTION
	1135
	1136	SECTION("manually supplied extent") {
	1137	std::string csv_string("wkt,Name\n");
	1138	mapnik::parameters params;
	1139	params["type"] = std::string("csv");
	1140	params["inline"] = csv_string;
	1141	params["extent"] = "-180,-90,180,90";
	1142	auto ds = mapnik::datasource_cache::instance().create(params);
	1143	REQUIRE(bool(ds));
	1144	auto box = ds->envelope();
	1145	CHECK(box.minx() == -180);
	1146	CHECK(box.miny() == -90);
	1147	CHECK(box.maxx() == 180);
	1148	CHECK(box.maxy() == 90);
	1149	} // END SECTION
	1150
	1151	SECTION("inline geojson") {
	1152	std::string csv_string = "geojson\n'{\"coordinates\":[-92.22568,38.59553],\"type\":\"Point\"}'";
	1153	mapnik::parameters params;
	1154	params["type"] = std::string("csv");
	1155	params["inline"] = csv_string;
	1156	params["quote"] = "'";
	1157	auto ds = mapnik::datasource_cache::instance().create(params);
	1158	REQUIRE(bool(ds));
	1159
	1160	auto fields = ds->get_descriptor().get_descriptors();
	1161	require_field_names(fields, {});
	1162
	1163	// TODO: this originally had the following comment:
	1164	// - re-enable after https://github.com/mapnik/mapnik/issues/2319 is fixed
	1165	// but that seems to have been merged and tested separately?
	1166	auto fs = all_features(ds);
	1167	auto feat = fs->next();
	1168	CHECK(feature_count(feat->get_geometry()) == 1);
	1169	} // END SECTION
	1170	mapnik::logger::instance().set_severity(severity);
	1171	}
	1172	} // END TEST CASE

+435

-75

test/unit/datasource/geojson.cpp less more

25	25	#include <mapnik/datasource.hpp>
26	26	#include <mapnik/datasource_cache.hpp>
27	27	#include <mapnik/geometry.hpp>
	28	#include <mapnik/geometry_type.hpp>
28	29	#include <mapnik/util/fs.hpp>
	30	#include <cstdlib>
	31
	32	#include <boost/filesystem/operations.hpp>
	33	#include <boost/optional/optional_io.hpp>
	34
	35	namespace {
	36
	37	std::pair<mapnik::datasource_ptr,mapnik::feature_ptr> fetch_first_feature(std::string const& filename, bool cache_features)
	38	{
	39	mapnik::parameters params;
	40	params["type"] = "geojson";
	41	params["file"] = filename;
	42	params["cache_features"] = cache_features;
	43	auto ds = mapnik::datasource_cache::instance().create(params);
	44	CHECK(ds->type() == mapnik::datasource::datasource_t::Vector);
	45	auto fields = ds->get_descriptor().get_descriptors();
	46	mapnik::query query(ds->envelope());
	47	for (auto const& field : fields)
	48	{
	49	query.add_property_name(field.get_name());
	50	}
	51	auto features = ds->features(query);
	52	auto feature = features->next();
	53	return std::make_pair(ds,feature);
	54	}
	55
	56	int create_disk_index(std::string const& filename, bool silent = true)
	57	{
	58	std::string cmd;
	59	if (std::getenv("DYLD_LIBRARY_PATH") != nullptr)
	60	{
	61	cmd += std::string("export DYLD_LIBRARY_PATH=") + std::getenv("DYLD_LIBRARY_PATH") + " && ";
	62	}
	63	cmd += "mapnik-index " + filename;
	64	if (silent)
	65	{
	66	#ifndef _WINDOWS
	67	cmd += " 2>/dev/null";
	68	#else
	69	cmd += " 2> nul";
	70	#endif
	71	}
	72	return std::system(cmd.c_str());
	73	}
	74
	75	}
29	76
30	77	TEST_CASE("geojson") {
31	78
32	79	std::string geojson_plugin("./plugins/input/geojson.input");
33	80	if (mapnik::util::exists(geojson_plugin))
34	81	{
35		SECTION("json feature cache-feature=\"true\"")
	82	SECTION("GeoJSON I/O errors")
	83	{
	84	std::string filename = "does_not_exist.geojson";
	85	for (auto create_index : { true, false })
	86	{
	87	if (create_index)
	88	{
	89	int ret = create_disk_index(filename);
	90	int ret_posix = (ret >> 8) & 0x000000ff;
	91	INFO(ret);
	92	INFO(ret_posix);
	93	// index wont be created
	94	CHECK(!mapnik::util::exists(filename + ".index"));
	95	}
	96
	97	for (auto cache_features : {true, false})
	98	{
	99	mapnik::parameters params;
	100	params["type"] = "geojson";
	101	params["file"] = filename;
	102	params["cache_features"] = cache_features;
	103	REQUIRE_THROWS(mapnik::datasource_cache::instance().create(params));
	104	}
	105	}
	106	}
	107
	108	SECTION("GeoJSON invalid Point")
	109	{
	110	for (auto cache_features : {true, false})
	111	{
	112	mapnik::parameters params;
	113	params["type"] = "geojson";
	114	params["file"] = "./test/data/json/point-invalid.json";
	115	params["cache_features"] = cache_features;
	116	REQUIRE_THROWS(mapnik::datasource_cache::instance().create(params));
	117	}
	118	}
	119
	120	SECTION("GeoJSON Point ")
	121	{
	122	for (auto cache_features : {true, false})
	123	{
	124	auto result = fetch_first_feature("./test/data/json/point.json", cache_features);
	125	auto feature = result.second;
	126	auto ds = result.first;
	127	CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
	128	auto const& geometry = feature->get_geometry();
	129	REQUIRE(mapnik::geometry::geometry_type(geometry) == mapnik::geometry::Point);
	130	auto const& pt = mapnik::util::get<mapnik::geometry::point<double> >(geometry);
	131	REQUIRE(pt.x == 100);
	132	REQUIRE(pt.y == 0);
	133	}
	134	}
	135
	136	SECTION("GeoJSON LineString")
	137	{
	138	for (auto cache_features : {true, false})
	139	{
	140	auto result = fetch_first_feature("./test/data/json/linestring.json", cache_features);
	141	auto feature = result.second;
	142	auto ds = result.first;
	143	CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::LineString);
	144	auto const& geometry = feature->get_geometry();
	145	REQUIRE(mapnik::geometry::geometry_type(geometry) == mapnik::geometry::LineString);
	146	auto const& line = mapnik::util::get<mapnik::geometry::line_string<double> >(geometry);
	147	REQUIRE(line.size() == 2);
	148	REQUIRE(mapnik::geometry::envelope(line) == mapnik::box2d<double>(100,0,101,1));
	149
	150	}
	151	}
	152
	153	SECTION("GeoJSON Polygon")
	154	{
	155	for (auto cache_features : {true, false})
	156	{
	157	auto result = fetch_first_feature("./test/data/json/polygon.json", cache_features);
	158	auto feature = result.second;
	159	auto ds = result.first;
	160	CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Polygon);
	161	auto const& geometry = feature->get_geometry();
	162	REQUIRE(mapnik::geometry::geometry_type(geometry) == mapnik::geometry::Polygon);
	163	auto const& poly = mapnik::util::get<mapnik::geometry::polygon<double> >(geometry);
	164	REQUIRE(poly.num_rings() == 2);
	165	REQUIRE(poly.exterior_ring.size() == 5);
	166	REQUIRE(poly.interior_rings.size() == 1);
	167	REQUIRE(poly.interior_rings[0].size() == 5);
	168	REQUIRE(mapnik::geometry::envelope(poly) == mapnik::box2d<double>(100,0,101,1));
	169
	170	}
	171	}
	172
	173	SECTION("GeoJSON MultiPoint")
	174	{
	175	for (auto cache_features : {true, false})
	176	{
	177	auto result = fetch_first_feature("./test/data/json/multipoint.json", cache_features);
	178	auto feature = result.second;
	179	auto ds = result.first;
	180	CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
	181	auto const& geometry = feature->get_geometry();
	182	REQUIRE(mapnik::geometry::geometry_type(geometry) == mapnik::geometry::MultiPoint);
	183	auto const& multi_pt = mapnik::util::get<mapnik::geometry::multi_point<double> >(geometry);
	184	REQUIRE(multi_pt.size() == 2);
	185	REQUIRE(mapnik::geometry::envelope(multi_pt) == mapnik::box2d<double>(100,0,101,1));
	186	}
	187	}
	188
	189	SECTION("GeoJSON MultiLineString")
	190	{
	191	for (auto cache_features : {true, false})
	192	{
	193	auto result = fetch_first_feature("./test/data/json/multilinestring.json", cache_features);
	194	auto feature = result.second;
	195	auto ds = result.first;
	196	CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::LineString);
	197	auto const& geometry = feature->get_geometry();
	198	REQUIRE(mapnik::geometry::geometry_type(geometry) == mapnik::geometry::MultiLineString);
	199	auto const& multi_line = mapnik::util::get<mapnik::geometry::multi_line_string<double> >(geometry);
	200	REQUIRE(multi_line.size() == 2);
	201	REQUIRE(multi_line[0].size() == 2);
	202	REQUIRE(multi_line[1].size() == 2);
	203	REQUIRE(mapnik::geometry::envelope(multi_line) == mapnik::box2d<double>(100,0,103,3));
	204
	205	}
	206	}
	207
	208	SECTION("GeoJSON MultiPolygon")
	209	{
	210	for (auto cache_features : {true, false})
	211	{
	212	auto result = fetch_first_feature("./test/data/json/multipolygon.json", cache_features);
	213	auto feature = result.second;
	214	auto ds = result.first;
	215	CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Polygon);
	216	// test
	217	auto const& geometry = feature->get_geometry();
	218	REQUIRE(mapnik::geometry::geometry_type(geometry) == mapnik::geometry::MultiPolygon);
	219	auto const& multi_poly = mapnik::util::get<mapnik::geometry::multi_polygon<double> >(geometry);
	220	REQUIRE(multi_poly.size() == 2);
	221	REQUIRE(multi_poly[0].num_rings() == 1);
	222	REQUIRE(multi_poly[1].num_rings() == 2);
	223	REQUIRE(mapnik::geometry::envelope(multi_poly) == mapnik::box2d<double>(100,0,103,3));
	224
	225	}
	226	}
	227
	228	SECTION("GeoJSON GeometryCollection")
	229	{
	230	std::string filename("./test/data/json/geometrycollection.json");
	231	for (auto create_index : { true, false })
	232	{
	233	if (create_index)
	234	{
	235	int ret = create_disk_index(filename);
	236	int ret_posix = (ret >> 8) & 0x000000ff;
	237	INFO(ret);
	238	INFO(ret_posix);
	239	// index will not exist because this is not a featurecollection
	240	CHECK(!mapnik::util::exists(filename + ".index"));
	241	}
	242
	243	for (auto cache_features : {true, false})
	244	{
	245	auto result = fetch_first_feature(filename, cache_features);
	246	auto feature = result.second;
	247	auto ds = result.first;
	248	CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Collection);
	249	// test
	250	auto const& geometry = feature->get_geometry();
	251	REQUIRE(mapnik::geometry::geometry_type(geometry) == mapnik::geometry::GeometryCollection);
	252	auto const& collection = mapnik::util::get<mapnik::geometry::geometry_collection<double> >(geometry);
	253	REQUIRE(collection.size() == 2);
	254	REQUIRE(mapnik::geometry::geometry_type(collection[0]) == mapnik::geometry::Point);
	255	REQUIRE(mapnik::geometry::geometry_type(collection[1]) == mapnik::geometry::LineString);
	256	REQUIRE(mapnik::geometry::envelope(collection) == mapnik::box2d<double>(100,0,102,1));
	257	}
	258	}
	259	}
	260
	261	SECTION("GeoJSON Feature")
36	262	{
37	263	// Create datasource
38	264	mapnik::parameters params;
39	265	params["type"] = "geojson";
40		params["file"] = "./test/data/json/feature.json";
41		params["cache-features"] = true;
42		auto ds = mapnik::datasource_cache::instance().create(params);
43		REQUIRE(bool(ds));
44		auto fields = ds->get_descriptor().get_descriptors();
45		mapnik::query query(ds->envelope());
46		for (auto const &field : fields)
47		{
48		query.add_property_name(field.get_name());
49		}
50		auto features = ds->features(query);
51		REQUIRE(features != nullptr);
52		auto feature = features->next();
53		REQUIRE(feature != nullptr);
54		}
55
56		SECTION("json feature cache-feature=\"false\"")
57		{
58		mapnik::parameters params;
59		params["type"] = "geojson";
60		params["file"] = "./test/data/json/feature.json";
61		params["cache-features"] = false;
62		auto ds = mapnik::datasource_cache::instance().create(params);
63		REQUIRE(bool(ds));
64		auto fields = ds->get_descriptor().get_descriptors();
65		mapnik::query query(ds->envelope());
66		for (auto const &field : fields)
67		{
68		query.add_property_name(field.get_name());
69		}
70		auto features = ds->features(query);
71		REQUIRE(features != nullptr);
72		auto feature = features->next();
73		REQUIRE(feature != nullptr);
74		}
75
76		SECTION("json extra properties cache-feature=\"true\"")
	266	std::string base("./test/data/json/");
	267	std::string file("feature.json");
	268	params["base"] = base;
	269	params["file"] = file;
	270	std::string filename = base + file;
	271	for (auto create_index : { true, false })
	272	{
	273	if (create_index)
	274	{
	275	int ret = create_disk_index(filename);
	276	int ret_posix = (ret >> 8) & 0x000000ff;
	277	INFO(ret);
	278	INFO(ret_posix);
	279	// index will not exist because this is not a featurecollection
	280	CHECK(!mapnik::util::exists(filename + ".index"));
	281	}
	282
	283	for (auto cache_features : {true, false})
	284	{
	285	params["cache_features"] = cache_features;
	286	auto ds = mapnik::datasource_cache::instance().create(params);
	287	REQUIRE(bool(ds));
	288	CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
	289	auto fields = ds->get_descriptor().get_descriptors();
	290	mapnik::query query(ds->envelope());
	291	for (auto const& field : fields)
	292	{
	293	query.add_property_name(field.get_name());
	294	}
	295	auto features = ds->features(query);
	296	auto features2 = ds->features_at_point(ds->envelope().center(),0);
	297	REQUIRE(features != nullptr);
	298	REQUIRE(features2 != nullptr);
	299	auto feature = features->next();
	300	auto feature2 = features2->next();
	301	REQUIRE(feature != nullptr);
	302	REQUIRE(feature2 != nullptr);
	303	CHECK(feature->id() == 1);
	304	CHECK(feature2->id() == 1);
	305	mapnik::value val = feature->get("name");
	306	CHECK(val.to_string() == "Dinagat Islands");
	307	mapnik::value val2 = feature2->get("name");
	308	CHECK(val2.to_string() == "Dinagat Islands");
	309	REQUIRE(features->next() == nullptr);
	310	REQUIRE(features2->next() == nullptr);
	311	}
	312	}
	313	}
	314
	315	SECTION("GeoJSON FeatureCollection")
	316	{
	317	std::string filename("./test/data/json/featurecollection.json");
	318
	319	// cleanup in the case of a failed previous run
	320	if (mapnik::util::exists(filename + ".index"))
	321	{
	322	boost::filesystem::remove(filename + ".index");
	323	}
	324
	325	for (auto create_index : { true, false })
	326	{
	327	if (create_index)
	328	{
	329	int ret = create_disk_index(filename);
	330	int ret_posix = (ret >> 8) & 0x000000ff;
	331	INFO(ret);
	332	INFO(ret_posix);
	333	CHECK(mapnik::util::exists(filename + ".index"));
	334	}
	335
	336	mapnik::parameters params;
	337	params["type"] = "geojson";
	338	params["file"] = filename;
	339
	340	for (auto cache_features : {true, false})
	341	{
	342	params["cache_features"] = cache_features;
	343
	344	auto ds = mapnik::datasource_cache::instance().create(params);
	345	CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Collection);
	346	auto fields = ds->get_descriptor().get_descriptors();
	347	mapnik::query query(ds->envelope());
	348	for (auto const& field : fields)
	349	{
	350	query.add_property_name(field.get_name());
	351	}
	352	auto features = ds->features(query);
	353	auto features2 = ds->features_at_point(ds->envelope().center(),10);
	354	auto bounding_box = ds->envelope();
	355	mapnik::box2d<double> bbox;
	356	mapnik::value_integer count = 0;
	357	while (true)
	358	{
	359	auto feature = features->next();
	360	auto feature2 = features2->next();
	361	if (!feature \|\| !feature2) break;
	362	if (!bbox.valid()) bbox = feature->envelope();
	363	else bbox.expand_to_include(feature->envelope());
	364	++count;
	365	REQUIRE(feature->id() == count);
	366	REQUIRE(feature2->id() == count);
	367	}
	368	REQUIRE(count == 3);
	369	REQUIRE(bounding_box == bbox);
	370	}
	371	if (mapnik::util::exists(filename + ".index"))
	372	{
	373	CHECK(mapnik::util::remove(filename + ".index"));
	374	}
	375	}
	376	}
	377
	378	SECTION("GeoJSON extra properties")
77	379	{
78	380	// Create datasource
79	381	mapnik::parameters params;
80	382	params["type"] = "geojson";
81		params["file"] = "./test/data/json/feature_collection_extra_properties.json";
82		params["cache-features"] = true;
83		auto ds = mapnik::datasource_cache::instance().create(params);
84		REQUIRE(bool(ds));
85		auto fields = ds->get_descriptor().get_descriptors();
86		mapnik::query query(ds->envelope());
87		for (auto const &field : fields)
88		{
89		query.add_property_name(field.get_name());
90		}
91		auto features = ds->features(query);
92		REQUIRE(features != nullptr);
93		auto feature = features->next();
94		REQUIRE(feature != nullptr);
95		REQUIRE(feature->envelope() == mapnik::box2d<double>(123,456,123,456));
96		}
97
98		SECTION("json extra properties cache-feature=\"false\"")
99		{
100		// Create datasource
	383	std::string filename("./test/data/json/feature_collection_extra_properties.json");
	384	params["file"] = filename;
	385
	386	// cleanup in the case of a failed previous run
	387	if (mapnik::util::exists(filename + ".index"))
	388	{
	389	boost::filesystem::remove(filename + ".index");
	390	}
	391
	392	for (auto create_index : { true, false })
	393	{
	394	if (create_index)
	395	{
	396	int ret = create_disk_index(filename);
	397	int ret_posix = (ret >> 8) & 0x000000ff;
	398	INFO(ret);
	399	INFO(ret_posix);
	400	CHECK(mapnik::util::exists(filename + ".index"));
	401	}
	402
	403	for (auto cache_features : {true, false})
	404	{
	405	params["cache_features"] = cache_features;
	406	auto ds = mapnik::datasource_cache::instance().create(params);
	407	CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
	408	REQUIRE(bool(ds));
	409	auto fields = ds->get_descriptor().get_descriptors();
	410	mapnik::query query(ds->envelope());
	411	for (auto const& field : fields)
	412	{
	413	query.add_property_name(field.get_name());
	414	}
	415	auto features = ds->features(query);
	416	REQUIRE(features != nullptr);
	417	auto feature = features->next();
	418	REQUIRE(feature != nullptr);
	419	REQUIRE(feature->envelope() == mapnik::box2d<double>(123,456,123,456));
	420	}
	421
	422	// cleanup
	423	if (create_index && mapnik::util::exists(filename + ".index"))
	424	{
	425	boost::filesystem::remove(filename + ".index");
	426	}
	427
	428	}
	429	}
	430
	431	SECTION("GeoJSON ensure input fully consumed and throw exception otherwise")
	432	{
101	433	mapnik::parameters params;
102	434	params["type"] = "geojson";
103		params["file"] = "./test/data/json/feature_collection_extra_properties.json";
104		params["cache-features"] = false;
105		auto ds = mapnik::datasource_cache::instance().create(params);
106		REQUIRE(bool(ds));
107		auto fields = ds->get_descriptor().get_descriptors();
108		mapnik::query query(ds->envelope());
109		for (auto const &field : fields)
110		{
111		query.add_property_name(field.get_name());
112		}
113		auto features = ds->features(query);
114		REQUIRE(features != nullptr);
115		auto feature = features->next();
116		REQUIRE(feature != nullptr);
117		REQUIRE(feature->envelope() == mapnik::box2d<double>(123,456,123,456));
118		}
119
	435	std::string filename("./test/data/json/points-malformed.geojson");
	436	params["file"] = filename; // mismatched parentheses
	437
	438	// cleanup in the case of a failed previous run
	439	if (mapnik::util::exists(filename + ".index"))
	440	{
	441	boost::filesystem::remove(filename + ".index");
	442	}
	443
	444	for (auto create_index : { true, false })
	445	{
	446	if (create_index)
	447	{
	448	CHECK(!mapnik::util::exists(filename + ".index"));
	449	int ret = create_disk_index(filename);
	450	int ret_posix = (ret >> 8) & 0x000000ff;
	451	INFO(ret);
	452	INFO(ret_posix);
	453	CHECK(mapnik::util::exists(filename + ".index"));
	454	}
	455
	456	for (auto cache_features : {true, false})
	457	{
	458	// unfortunately when using an index or not
	459	// caching features we use the bbox grammar
	460	// which is not strict (and would be a perf hit if it were strict).
	461	// So this is one known hole where invalid data may silently parse okay
	462	// refs https://github.com/mapnik/mapnik/issues/3125
	463	if (!create_index && cache_features == true)
	464	{
	465	std::stringstream msg;
	466	msg << "testcase: create index " << create_index << " cache_features " << cache_features;
	467	params["cache_features"] = cache_features;
	468	INFO(msg.str());
	469	CHECK_THROWS(mapnik::datasource_cache::instance().create(params));
	470	}
	471	}
	472
	473	// cleanup
	474	if (create_index && mapnik::util::exists(filename + ".index"))
	475	{
	476	boost::filesystem::remove(filename + ".index");
	477	}
	478	}
	479	}
120	480	}
121	481	}

-1

utils/mapnik-index/build.py less more

40	40	boost_program_options = 'boost_program_options%s' % env['BOOST_APPEND']
41	41	boost_system = 'boost_system%s' % env['BOOST_APPEND']
42	42	libraries = [env['MAPNIK_NAME'], boost_program_options, boost_system]
43		libraries.append(env['ICU_LIB_NAME'])
44	43	libraries.append('mapnik-json')
45	44	libraries.append('mapnik-wkt')
46	45

+27

-11

utils/mapnik-index/mapnik-index.cpp less more

132	132	return EXIT_FAILURE;
133	133	}
134	134
	135	std::vector<std::string> files_to_process;
	136
	137	for (auto const& filename : files)
	138	{
	139	if (!mapnik::util::exists(filename))
	140	{
	141	continue;
	142	}
	143
	144	if (mapnik::detail::is_csv(filename) \|\| mapnik::detail::is_geojson(filename))
	145	{
	146	files_to_process.push_back(filename);
	147	}
	148	}
	149
	150	if (files_to_process.size() == 0)
	151	{
	152	std::clog << "no files to index" << std::endl;
	153	return EXIT_FAILURE;
	154	}
	155
135	156	std::clog << "max tree depth:" << depth << std::endl;
136	157	std::clog << "split ratio:" << ratio << std::endl;
137	158
138		if (files.size() == 0)
139		{
140		std::clog << "no files to index" << std::endl;
141		return EXIT_FAILURE;
142		}
143
144	159	using box_type = mapnik::box2d<double>;
145	160	using item_type = std::pair<box_type, std::pair<std::size_t, std::size_t>>;
146	161
147		for (auto const& filename : files)
148		{
149		std::clog << "processing " << filename << std::endl;
150		if (!mapnik::util::exists (filename))
	162	for (auto const& filename : files_to_process)
	163	{
	164	if (!mapnik::util::exists(filename))
151	165	{
152	166	std::clog << "Error : file " << filename << " does not exist" << std::endl;
153	167	continue;

157	171	mapnik::box2d<double> extent;
158	172	if (mapnik::detail::is_csv(filename))
159	173	{
	174	std::clog << "processing '" << filename << "' as CSV\n";
160	175	auto result = mapnik::detail::process_csv_file(boxes, filename, manual_headers, separator, quote);
161	176	if (!result.first) continue;
162	177	extent = result.second;
163	178	}
164	179	else if (mapnik::detail::is_geojson(filename))
165	180	{
	181	std::clog << "processing '" << filename << "' as GeoJSON\n";
166	182	auto result = mapnik::detail::process_geojson_file(boxes, filename);
167	183	if (!result.first) continue;
168	184	extent = result.second;

188	204	{
189	205	tree.trim();
190	206	std::clog << "number nodes=" << tree.count() << std::endl;
191		//tree.print();
	207	std::clog << "number element=" << tree.count_items() << std::endl;
192	208	file.exceptions(std::ios::failbit \| std::ios::badbit);
193	209	tree.write(file);
194	210	file.flush();

+40

-15

utils/mapnik-index/process_csv_file.cpp less more

22	22	#include "process_csv_file.hpp"
23	23	#include "../../plugins/input/csv/csv_utils.hpp"
24	24	#include <mapnik/geometry_envelope.hpp>
25
	25	#include <mapnik/util/utf_conv_win.hpp>
	26
	27	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
26	28	#pragma GCC diagnostic push
27	29	#pragma GCC diagnostic ignored "-Wshadow"
28	30	#pragma GCC diagnostic ignored "-Wsign-conversion"

30	32	#include <boost/interprocess/streams/bufferstream.hpp>
31	33	#pragma GCC diagnostic pop
32	34	#include <mapnik/mapped_memory_cache.hpp>
	35	#endif
	36
	37	#include <fstream>
33	38
34	39	namespace mapnik { namespace detail {
35	40
36	41	template <typename T>
37	42	std::pair<bool,box2d<double>> process_csv_file(T & boxes, std::string const& filename, std::string const& manual_headers, char separator, char quote)
38	43	{
	44	mapnik::box2d<double> extent;
	45	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
39	46	using file_source_type = boost::interprocess::ibufferstream;
40	47	file_source_type csv_file;
41		mapnik::box2d<double> extent;
42	48	mapnik::mapped_region_ptr mapped_region;
43	49	boost::optional<mapnik::mapped_region_ptr> memory =
44	50	mapnik::mapped_memory_cache::instance().find(filename, true);

52	58	std::clog << "Error : cannot mmap " << filename << std::endl;
53	59	return std::make_pair(false, extent);
54	60	}
	61	#else
	62	#if defined(_WINDOWS)
	63	std::ifstream csv_file(mapnik::utf8_to_utf16(filename),std::ios_base::in \| std::ios_base::binary);
	64	#else
	65	std::ifstream csv_file(filename.c_str(),std::ios_base::in \| std::ios_base::binary);
	66	#endif
	67	if (!csv_file.is_open())
	68	{
	69	std::clog << "Error : cannot open " << filename << std::endl;
	70	return std::make_pair(false, extent);
	71	}
	72	#endif
55	73	auto file_length = ::detail::file_length(csv_file);
56	74	// set back to start
57	75	csv_file.seekg(0, std::ios::beg);

67	85	csv_utils::getline_csv(csv_file, csv_line, newline, quote);
68	86	if (separator == 0) separator = ::detail::detect_separator(csv_line);
69	87	csv_file.seekg(0, std::ios::beg);
70		int line_number = 1;
	88	int line_number = 0;
71	89	::detail::geometry_column_locator locator;
72	90	std::vector<std::string> headers;
73	91	std::clog << "Parsing CSV using SEPARATOR=" << separator << " QUOTE=" << quote << std::endl;

95	113	std::size_t index = 0;
96	114	for (auto & header : headers)
97	115	{
	116	mapnik::util::trim(header);
98	117	if (header.empty())
99	118	{
100	119	// create a placeholder for the empty header

122	141	}
123	142	}
124	143
125		if (locator.type == ::detail::geometry_column_locator::UNKNOWN)
126		{
127		std::clog << "CSV index: could not detect column headers with the name of wkt, geojson, x/y, or "
128		<< "latitude/longitude - this is required for reading geometry data" << std::endl;
	144	std::size_t num_headers = headers.size();
	145	if (!::detail::valid(locator, num_headers))
	146	{
	147	std::clog << "CSV index: could not detect column(s) with the name(s) of wkt, geojson, x/y, or "
	148	<< "latitude/longitude in:\n"
	149	<< csv_line
	150	<< "\n - this is required for reading geometry data"
	151	<< std::endl;
129	152	return std::make_pair(false, extent);
130	153	}
131	154
132		std::size_t num_headers = headers.size();
133	155	auto pos = csv_file.tellg();
134	156
135	157	// handle rare case of a single line of data and user-provided headers

144	166	is_first_row = true;
145	167	}
146	168	}
147
148	169	while (is_first_row \|\| csv_utils::getline_csv(csv_file, csv_line, newline, quote))
149	170	{
	171	++line_number;
150	172	auto record_offset = pos;
151	173	auto record_size = csv_line.length();
152	174	pos = csv_file.tellg();

168	190	unsigned num_fields = values.size();
169	191	if (num_fields > num_headers \|\| num_fields < num_headers)
170	192	{
	193	// skip this row
171	194	std::ostringstream s;
172	195	s << "CSV Index: # of columns("
173	196	<< num_fields << ") > # of headers("
174		<< num_headers << ") parsed for row " << line_number << "\n";
175		std::clog << s.str() << std::endl;
176		return std::make_pair(false, extent);
	197	<< num_headers << ") parsed for row " << line_number;
	198	throw mapnik::datasource_exception(s.str());
177	199	}
178	200
179	201	auto geom = ::detail::extract_geometry(values, locator);

190	212	s << "CSV Index: expected geometry column: could not parse row "
191	213	<< line_number << " "
192	214	<< values[locator.index] << "'";
193		std::clog << s.str() << std::endl;;
194		}
	215	throw mapnik::datasource_exception(s.str());
	216	}
	217	}
	218	catch (mapnik::datasource_exception const& ex )
	219	{
	220	std::clog << ex.what() << " at line: " << line_number << std::endl;
195	221	}
196	222	catch (std::exception const& ex)
197	223	{

200	226	<< " - found " << headers.size() << " with values like: " << csv_line << "\n"
201	227	<< " and got error like: " << ex.what();
202	228	std::clog << s.str() << std::endl;
203		return std::make_pair(false, extent);
204	229	}
205	230	}
206	231	return std::make_pair(true, extent);;

+24

-2

utils/mapnik-index/process_geojson_file.cpp less more

23	23	#include <mapnik/geometry.hpp>
24	24	#include <mapnik/geometry_envelope.hpp>
25	25	#include <mapnik/geometry_adapters.hpp>
	26	#include <mapnik/util/file_io.hpp>
	27	#include <mapnik/util/utf_conv_win.hpp>
	28
	29	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
26	30	#pragma GCC diagnostic push
27	31	#pragma GCC diagnostic ignored "-Wshadow"
28	32	#pragma GCC diagnostic ignored "-Wsign-compare"

32	36	#include <boost/spirit/include/qi.hpp>
33	37	#pragma GCC diagnostic pop
34	38	#include <mapnik/mapped_memory_cache.hpp>
	39	#endif
	40
35	41	#include <mapnik/json/positions_grammar.hpp>
36	42	#include <mapnik/json/extract_bounding_box_grammar_impl.hpp>
37	43

46	52	std::pair<bool,box2d<double>> process_geojson_file(T & boxes, std::string const& filename)
47	53	{
48	54	mapnik::box2d<double> extent;
	55	#if defined(MAPNIK_MEMORY_MAPPED_FILE)
49	56	mapnik::mapped_region_ptr mapped_region;
50	57	boost::optional<mapnik::mapped_region_ptr> memory =
51	58	mapnik::mapped_memory_cache::instance().find(filename, true);
52	59	if (!memory)
53	60	{
54		std::clog << "Error : cannot mmap " << filename << std::endl;
	61	std::clog << "Error : cannot memory map " << filename << std::endl;
55	62	return std::make_pair(false, extent);
56	63	}
57	64	else

60	67	}
61	68	char const* start = reinterpret_cast<char const*>(mapped_region->get_address());
62	69	char const* end = start + mapped_region->get_size();
	70	#else
	71	mapnik::util::file file(filename);
	72	if (!file.open())
	73	{
	74	std::clog << "Error : cannot open " << filename << std::endl;
	75	return std::make_pair(false, extent);
	76	}
	77	std::string file_buffer;
	78	file_buffer.resize(file.size());
	79	std::fread(&file_buffer[0], file.size(), 1, file.get());
	80	char const* start = file_buffer.c_str();
	81	char const* end = start + file_buffer.length();
	82	#endif
	83
63	84	boost::spirit::standard::space_type space;
64	85	try
65	86	{
66	87	if (!boost::spirit::qi::phrase_parse(start, end, (geojson_datasource_static_bbox_grammar)(boost::phoenix::ref(boxes)) , space))
67	88	{
68		std::clog << "mapnik-index (GeoJSON) : could not parse: '" << filename << "'";
	89	std::clog << "mapnik-index (GeoJSON) : could extract bounding boxes from : '" << filename << "'";
	90	std::clog << " expected FeatureCollection" << std::endl;
69	91	return std::make_pair(false, extent);
70	92	}
71	93	}

+21

-2

utils/shapefile/shapefile_reader.py less more

24	24	print "NULL shape"
25	25	elif _type == 11: #PointZ
26	26	test_pointz(record)
	27	elif _type == 5:
	28	test_polygon(record)
27	29
28	30	def test_pointz(record):
29	31	if len(record) != 36 :

32	34	_type,x,y,z,m = struct.unpack("<idddd",record)
33	35	if _type != 11:
34	36	print>>sys.stderr,"BAD SHAPE FILE: expected PointZ or NullShape got",_type
	37	sys.exit(1)
	38
	39	def test_polygon(record):
	40	_type, x0, y0, x1, y0, num_parts, num_points = struct.unpack("<iddddii", record[0:44])
	41	if _type != 5:
	42	print>>sys.stderr, "BAD SHAPE FILE: expected Polygon or NullShape got", _type
	43	sys.exit(1)
	44	length = len(record)
	45	rec_length = 44 + num_parts * 4 + num_points * 16
	46	if rec_length <> length:
	47	print>>sys.stderr, "BAD SHAPE FILE: expected", rec_length, "got", length
35	48	sys.exit(1)
36	49
37	50	if __name__ == "__main__" :

51	64	_,_,_,_,_,_,shx_file_length = header[0].unpack_from(shx.read(28))
52	65	_,_,lox,loy,hix,hiy,_,_,_,_ = header[1].unpack_from(shx.read(72))
53	66
	67	shx_bbox = [lox,loy,hix,hiy]
	68
54	69	# SHP header
55	70	_,_,_,_,_,_,shp_file_length = header[0].unpack_from(shp.read(28))
56	71	version,_type,lox,loy,hix,hiy,_,_,_,_ = header[1].unpack_from(shp.read(72))
	72
	73	shp_bbox = [lox,loy,hix,hiy]
	74	if shx_bbox <> shp_bbox :
	75	print "BAD SHAPE FILE: bounding box mismatch in .shp and .shx", shp_bbox, shx_bbox
	76	sys.exit(1)
57	77
58	78	print "SHX FILE_LENGTH=",shx_file_length,"bytes"
59	79	print "SHP FILE_LENGTH=",shp_file_length,"bytes"

65	85	calc_total_size = 50
66	86	count = 0
67	87	while shx.tell() < shx_file_length * 2 :
68		offset,shx_content_length = record.unpack_from(shx.read(8))
	88	offset,shx_content_length = record.unpack_from(shx.read(8))
69	89	shp.seek(offset*2, os.SEEK_SET)
70	90	record_number,content_length = record_header.unpack_from(shp.read(8))
71
72	91	if shx_content_length <> content_length:
73	92	print "BAD SHAPE FILE: content_lenght mismatch in SHP and SHX",shx_content_length,content_length
74	93	sys.exit(1)

+85

-112

utils/shapeindex/shapeindex.cpp less more

42	42	{
43	43	using namespace mapnik;
44	44	namespace po = boost::program_options;
45		using std::string;
46		using std::vector;
47		using std::clog;
48		using std::endl;
49	45
50	46	bool verbose=false;
51	47	unsigned int depth=DEFAULT_DEPTH;
52	48	double ratio=DEFAULT_RATIO;
53		vector<string> shape_files;
	49	std::vector<std::string> shape_files;
54	50
55	51	try
56	52	{

61	57	("verbose,v","verbose output")
62	58	("depth,d", po::value<unsigned int>(), "max tree depth\n(default 8)")
63	59	("ratio,r",po::value<double>(),"split ratio (default 0.55)")
64		("shape_files",po::value<vector<string> >(),"shape files to index: file1 file2 ...fileN")
	60	("shape_files",po::value<std::vector<std::string> >(),"shape files to index: file1 file2 ...fileN")
65	61	;
66	62
67	63	po::positional_options_description p;

72	68
73	69	if (vm.count("version"))
74	70	{
75		clog << "version 0.3.0" <<std::endl;
	71	std::clog << "version 0.3.0" <<std::endl;
76	72	return 1;
77	73	}
78	74
79	75	if (vm.count("help"))
80	76	{
81		clog << desc << endl;
	77	std::clog << desc << std::endl;
82	78	return 1;
83	79	}
84	80	if (vm.count("verbose"))

96	92
97	93	if (vm.count("shape_files"))
98	94	{
99		shape_files=vm["shape_files"].as< vector<string> >();
	95	shape_files=vm["shape_files"].as< std::vector<std::string> >();
100	96	}
101	97	}
102	98	catch (std::exception const& ex)
103	99	{
104		clog << "Error: " << ex.what() << endl;
	100	std::clog << "Error: " << ex.what() << std::endl;
105	101	return -1;
106	102	}
107	103
108		clog << "max tree depth:" << depth << endl;
109		clog << "split ratio:" << ratio << endl;
110
111		//vector<string>::const_iterator itr = shape_files.begin();
	104	std::clog << "max tree depth:" << depth << std::endl;
	105	std::clog << "split ratio:" << ratio << std::endl;
	106
112	107	if (shape_files.size() == 0)
113	108	{
114		clog << "no shape files to index" << endl;
	109	std::clog << "no shape files to index" << std::endl;
115	110	return 0;
116	111	}
117	112	for (auto const& filename : shape_files)
118	113	{
119		clog << "processing " << filename << endl;
	114	std::clog << "processing " << filename << std::endl;
120	115	std::string shapename (filename);
121	116	boost::algorithm::ireplace_last(shapename,".shp","");
122	117	std::string shapename_full (shapename + ".shp");
123
	118	std::string shxname(shapename + ".shx");
124	119	if (! mapnik::util::exists (shapename_full))
125	120	{
126		clog << "Error : file " << shapename_full << " does not exist" << endl;
127		continue;
128		}
129
	121	std::clog << "Error : file " << shapename_full << " does not exist" << std::endl;
	122	continue;
	123	}
	124	if (! mapnik::util::exists(shxname))
	125	{
	126	std::clog << "Error : shapefile index file (*.shx) " << shxname << " does not exist" << std::endl;
	127	continue;
	128	}
130	129	shape_file shp (shapename_full);
131	130
132		if (! shp.is_open()) {
133		clog << "Error : cannot open " << shapename_full << endl;
134		continue;
135		}
136
137		int code = shp.read_xdr_integer(); //file_code == 9994
138		clog << code << endl;
139		shp.skip(5*4);
140
141		int file_length=shp.read_xdr_integer();
142		int version=shp.read_ndr_integer();
143		int shape_type=shp.read_ndr_integer();
	131	if (! shp.is_open())
	132	{
	133	std::clog << "Error : cannot open " << shapename_full << std::endl;
	134	continue;
	135	}
	136
	137	shape_file shx (shxname);
	138	if (!shx.is_open())
	139	{
	140	std::clog << "Error : cannot open " << shxname << std::endl;
	141	continue;
	142	}
	143
	144	int code = shx.read_xdr_integer(); //file_code == 9994
	145	std::clog << code << std::endl;
	146	shx.skip(5*4);
	147
	148	int file_length=shx.read_xdr_integer();
	149	int version=shx.read_ndr_integer();
	150	int shape_type=shx.read_ndr_integer();
144	151	box2d<double> extent;
145		shp.read_envelope(extent);
146
147
148		clog << "length=" << file_length << endl;
149		clog << "version=" << version << endl;
150		clog << "type=" << shape_type << endl;
151		clog << "extent:" << extent << endl;
152
153		int pos=50;
154		shp.seek(pos*2);
155		mapnik::quad_tree<int> tree(extent,depth,ratio);
156		int count=0;
157		while (true) {
158
159		long offset=shp.pos();
160		int record_number=shp.read_xdr_integer();
161		int content_length=shp.read_xdr_integer();
	152	shx.read_envelope(extent);
	153
	154
	155	std::clog << "length=" << file_length << std::endl;
	156	std::clog << "version=" << version << std::endl;
	157	std::clog << "type=" << shape_type << std::endl;
	158	std::clog << "extent:" << extent << std::endl;
	159
	160	int pos = 50;
	161	shx.seek(pos * 2);
	162	mapnik::quad_tree<int> tree(extent, depth, ratio);
	163	int count = 0;
	164
	165	while (true)
	166	{
	167	int offset = shx.read_xdr_integer();
	168	int content_length = shx.read_xdr_integer();
	169	pos += 4;
	170	box2d<double> item_ext;
	171	shp.seek(offset * 2);
	172	int record_number = shp.read_xdr_integer();
	173	if (content_length != shp.read_xdr_integer())
	174	{
	175	std::clog << "Content length mismatch for record number " << record_number << std::endl;
	176	continue;
	177	}
162	178	shape_type = shp.read_ndr_integer();
163		box2d<double> item_ext;
164		if (shape_type==shape_io::shape_null)
165		{
166		if (pos >= file_length)
167		{
168		break;
169		}
170		else
171		{
172		// still need to increment pos, or the pos counter
173		// won't indicate EOF until too late.
174		pos+=4+content_length;
175		continue;
176		}
177		}
178		else if (shape_type==shape_io::shape_point)
	179
	180	if (shape_type==shape_io::shape_point
	181	\|\| shape_type==shape_io::shape_pointm
	182	\|\| shape_type == shape_io::shape_pointz)
179	183	{
180	184	double x=shp.read_double();
181	185	double y=shp.read_double();
182	186	item_ext=box2d<double>(x,y,x,y);
183	187	}
184		else if (shape_type==shape_io::shape_pointm)
185		{
186		double x=shp.read_double();
187		double y=shp.read_double();
188		// skip m
189		shp.read_double();
190		item_ext=box2d<double>(x,y,x,y);
191		}
192		else if (shape_type==shape_io::shape_pointz)
193		{
194		double x=shp.read_double();
195		double y=shp.read_double();
196		// skip z
197		shp.read_double();
198		// According to ESRI shapefile doc
199		// A PointZ consists of a triplet of double-precision coordinates in the order X, Y, Z plus a
200		// measure.
201		// PointZ
202		// {
203		// Double X // X coordinate
204		// Double Y // Y coordinate
205		// Double Z // Z coordinate
206		// Double M // Measure
207		// }
208		// But OGR creates shapefiles with M missing so we need to skip M only if present
209		// NOTE: content_length is in 16-bit words
210		if ( content_length == 18)
211		{
212		shp.read_double();
213		}
214		item_ext=box2d<double>(x,y,x,y);
215		}
216	188	else
217	189	{
218	190	shp.read_envelope(item_ext);
219		shp.skip(2content_length-48-4);
220		}
221		tree.insert(offset,item_ext);
	191	}
	192
	193	tree.insert(offset * 2,item_ext);
	194
222	195	if (verbose)
223	196	{
224		clog << "record number " << record_number << " box=" << item_ext << endl;
225		}
226
227		pos+=4+content_length;
	197	std::clog << "record number " << record_number << " box=" << item_ext << std::endl;
	198	}
228	199	++count;
229
230	200	if (pos >= file_length) break;
231	201	}
232	202
233		clog << " number shapes=" << count << endl;
	203	std::clog << " number shapes=" << count << std::endl;
234	204
235	205	std::fstream file((shapename+".index").c_str(),
236	206	std::ios::in \| std::ios::out \| std::ios::trunc \| std::ios::binary);
237		if (!file) {
238		clog << "cannot open index file for writing file \""
239		<< (shapename+".index") << "\"" << endl;
240		} else {
	207	if (!file)
	208	{
	209	std::clog << "cannot open index file for writing file \""
	210	<< (shapename+".index") << "\"" << std::endl;
	211	}
	212	else
	213	{
241	214	tree.trim();
242	215	std::clog << " number nodes=" << tree.count() << std::endl;
243	216	file.exceptions(std::ios::failbit \| std::ios::badbit);

247	220	}
248	221	}
249	222
250		clog << "done!" << endl;
	223	std::clog << "done!" << std::endl;
251	224	return 0;
252	225	}

-1

utils/svg2png/svg2png.cpp less more

107	107	#else
108	108	s << "xdg-open " << png_name;
109	109	#endif
110		int ret = system(s.str().c_str());
	110	int ret = std::system(s.str().c_str());
111	111	if (ret != 0)
112	112	status = ret;
113	113	}