Codebase list mkgmap / e042a28
Imported Upstream version 0.0.0+svn3478 Bas Couwenberg 9 years ago
19 changed file(s) with 382 addition(s) and 64 deletion(s). Raw diff Collapse all Expand all
123123 If the index is created from previously compiled .img files, then the
124124 same code page and sorting options (eg. --code-page, --latin1 etc) must
125125 be used as were used to compile the individual map tiles.
126
127 --x-split-name-index
128 A temporary option to enable indexing each part of a street name separately.
129 So for example if the street is "Aleksandra Gryglewskiego" then you will be able to
130 search for it as both "Aleksandra" and "Gryglewskiego". It will also increase the
131 size of the index. Useful in countries where searching for the first word in name
132 is not the right thing to do.
133
134 Note that this option is still experimental and there may be problems. If you find
135 any let us know!
126136
127137 --bounds=directory|zipfile
128138 A directory or a zipfile containing the preprocessed bounds files.
0 svn.version: 3436
1 build.timestamp: 2015-02-02T06:43:48+0000
0 svn.version: 3478
1 build.timestamp: 2015-02-26T07:49:21+0000
1212 brand=${name} { delete brand; }
1313
1414 # None of operator, brand given
15 ref=* & (operator!=* & brand!=*) & (highway=bus_stop | railway=tram_stop | railway=halt | railway=station) { name '${name} ${ref}' | '${ref}' }
1516 ref=* & (operator!=* & brand!=*) { name '${ref} ${name}' | '${ref}' }
1617
1718 # Both operator and brand given
2324 }
2425
2526 # One of operator or brand given
27 operator=* & brand!=* & (highway=bus_stop | railway=tram_stop | railway=halt | railway=station) {
28 name '${name} ${ref} ${operator}' |
29 '${name} ${operator}' |
30 '${ref} ${operator}' |
31 '${operator}'
32 }
33
2634 operator=* & brand!=* {
2735 name '${operator}: ${ref} ${name}' |
2836 '${operator}: ${name}' |
162162
163163 railway=abandoned [0x0a road_class=0 road_speed=1 resolution 22]
164164 railway=platform [0x16 road_class=0 road_speed=0 resolution 23]
165 railway=* & !(tunnel=yes) [0x14 resolution 22]
165 # Railway lines, note that many devices display type 0x14 only at resolution 24
166 (railway=rail | railway=tram | railway=disused | railway=subway | railway=narrow_gauge | railway=light_rail | railway=preserved) & !(tunnel=yes) [0x14 resolution 22]
166167
167168 (man_made=cable|(man_made=* & man_made ~ '.*pipe.*')) & area!=yes &
168169 tunnel!=yes & location != underground
4141 public boolean addByte(int b) {
4242 if (b == 0) {
4343 needreset = true;
44 out.write(0);
4544 return true;
4645 }
4746
8787 Collator collator = getConfig().getSort().getCollator();
8888
8989 String lastName = null;
90 String lastPartialName = null;
9091 Mdr5Record lastCity = null;
9192 int record = 0;
92 int cityRecord = 0;
93 int cityRecord = 1;
9394 int lastMapNumber = 0;
9495
9596 for (SortKey<Mdr7Record> key : keys) {
9697 Mdr7Record street = key.getObject();
9798
9899 String name = street.getName();
100 String partialName = street.getPartialName();
99101 Mdr5Record city = street.getCity();
100102
101103 boolean citySameByName = city.isSameByName(collator, lastCity);
103105 int mapNumber = city.getMapIndex();
104106
105107 // Only save a single copy of each street name.
106 if (!citySameByName || mapNumber != lastMapNumber || lastName == null || collator.compare(name, lastName) != 0) {
108 if (!citySameByName || mapNumber != lastMapNumber || lastName == null || lastPartialName == null
109 || !name.equals(lastName)
110 || !partialName.equals(lastPartialName))
111 {
107112 record++;
108113
109114 streets.add(street);
110115 lastName = name;
116 lastPartialName = partialName;
111117 }
112118
113119 // The mdr20 value changes for each new city name
114120 if (citySameByName) {
121 assert cityRecord!=0;
115122 city.setMdr20(cityRecord);
116123 } else {
117124 // New city name, this marks the start of a new section in mdr20
125 assert cityRecord != 0;
118126 cityRecord = record;
119127 city.setMdr20(cityRecord);
120128 lastCity = city;
136144 * Unknown.
137145 */
138146 public int getExtraValue() {
139 return isForDevice() ? 0xa : 0x8800;
147 return isForDevice() ? 0xe : 0x8800;
140148 }
141149 }
103103 */
104104 public int getExtraValue() {
105105 if (isForDevice())
106 return 0x600a;
106 return 0x600e;
107107 else
108108 return 0x11000;
109109 }
1515 import java.util.List;
1616
1717 import uk.me.parabola.imgfmt.app.ImgFileWriter;
18 import uk.me.parabola.imgfmt.app.Label;
1918
2019 /**
2120 * Common code for 20, 21, 22 which are all lists of streets ordered in
3837 int size = getSizes().getStreetSizeFlagged();
3938
4039 boolean hasLabel = hasFlag(0x2);
40
41 String lastPartial = null;
4142 int recordNumber = 0;
4243 for (Mdr7Record street : streets) {
4344 assert street.getMapIndex() == street.getCity().getMapIndex() : street.getMapIndex() + "/" + street.getCity().getMapIndex();
4445 addIndexPointer(street.getMapIndex(), ++recordNumber);
4546
4647 int index = street.getIndex();
47 String name = Label.stripGarminCodes(street.getName());
48
49 int flag = 1;
48
49 String name = street.getName();
50
51 int repeat = 1;
5052 if (name.equals(lastName) && sameGroup(street, prev))
51 flag = 0;
52 lastName = name;
53 prev = street;
53 repeat = 0;
5454
5555 if (hasLabel) {
5656 putMapIndex(writer, street.getMapIndex());
5757 int offset = street.getLabelOffset();
58 if (flag != 0)
58 if (repeat != 0)
5959 offset |= 0x800000;
60
61 int trailing = 0;
62 String partialName = street.getPartialName();
63 if (!partialName.equals(lastPartial)) {
64 trailing |= 1;
65 offset |= 0x800000;
66 }
67
6068 writer.put3(offset);
61 writer.put((byte) flag);
62 }
63 else
64 putN(writer, size, (index << 1) | flag);
69 writer.put(street.getOutNameOffset());
70
71 writer.put((byte) trailing);
72
73 lastPartial = partialName;
74 } else
75 putN(writer, size, (index << 1) | repeat);
76
77 lastName = name;
78 prev = street;
6579 }
6680 }
6781
7791 public int getItemSize() {
7892 int size;
7993 if (isForDevice()) {
80 size = getSizes().getMapSize() + 3 + 1;
94 // map-index, label, name-offset, 1byte flag
95 size = getSizes().getMapSize() + 3 + 1 + 1;
8196 } else {
8297 size = getSizes().getStreetSizeFlagged();
8398 }
1515 import java.util.Collections;
1616 import java.util.List;
1717
18 import uk.me.parabola.imgfmt.MapFailedException;
1819 import uk.me.parabola.imgfmt.app.ImgFileWriter;
19 import uk.me.parabola.imgfmt.app.Label;
20 import uk.me.parabola.imgfmt.app.srt.MultiSortKey;
21 import uk.me.parabola.imgfmt.app.srt.Sort;
2022 import uk.me.parabola.imgfmt.app.srt.SortKey;
2123
2224 /**
2628 * @author Steve Ratcliffe
2729 */
2830 public class Mdr7 extends MdrMapSection {
31 public static final int MDR7_HAS_STRING = 0x01;
32 public static final int MDR7_HAS_NAME_OFFSET = 0x20;
33 public static final int MDR7_PARTIAL_SHIFT = 6;
34 public static final int MDR7_U1 = 0x2;
35 public static final int MDR7_U2 = 0x4;
36
37 private static final int MAX_NAME_OFFSET = 127;
38
39 private final int codepage;
40 private final boolean isMulti;
41 private final boolean splitName;
42
2943 private List<Mdr7Record> allStreets = new ArrayList<>();
3044 private List<Mdr7Record> streets = new ArrayList<>();
3145
46 private final int u2size = 1;
47
3248 public Mdr7(MdrConfig config) {
3349 setConfig(config);
50 Sort sort = config.getSort();
51 splitName = config.isSplitName();
52 codepage = sort.getCodepage();
53 isMulti = sort.isMulti();
3454 }
3555
3656 public void addStreet(int mapId, String name, int lblOffset, int strOff, Mdr5Record mdrCity) {
57 // Find a name prefix, which is either a shield or a word ending 0x1e. We are treating
58 // a shield as a prefix of length one.
59 int prefix = 0;
60 if (name.charAt(0) < 7)
61 prefix = 1;
62 int sep = name.indexOf(0x1e);
63 if (sep > 0)
64 prefix = sep + 1;
65
66 // Find a name suffix which begins with 0x1f
67 sep = name.indexOf(0x1f);
68 int suffix = 0;
69 if (sep > 0)
70 suffix = sep;
71
72 // Large values can't actually work.
73 if (prefix >= MAX_NAME_OFFSET || suffix >= MAX_NAME_OFFSET)
74 return;
75
3776 Mdr7Record st = new Mdr7Record();
3877 st.setMapIndex(mapId);
3978 st.setLabelOffset(lblOffset);
4079 st.setStringOffset(strOff);
4180 st.setName(name);
4281 st.setCity(mdrCity);
82 st.setPrefixOffset((byte) prefix);
83 st.setSuffixOffset((byte) suffix);
4384 allStreets.add(st);
85
86 if (!splitName)
87 return;
88
89 boolean start = false;
90 boolean inWord = false;
91
92 int c;
93 int outOffset = 0;
94
95 int end = Math.min((suffix > 0) ? suffix : name.length() - prefix - 1, MAX_NAME_OFFSET);
96 for (int nameOffset = 0; nameOffset < end; nameOffset += Character.charCount(c)) {
97 c = name.codePointAt(prefix + nameOffset);
98
99 // Don't use any word after a bracket
100 if (c == '(')
101 break;
102
103 if (!Character.isLetterOrDigit(c)) {
104 start = true;
105 inWord = false;
106 } else if (start && Character.isLetterOrDigit(c)) {
107 inWord = true;
108 }
109
110 if (start && inWord && outOffset > 0) {
111 st = new Mdr7Record();
112 st.setMapIndex(mapId);
113 st.setLabelOffset(lblOffset);
114 st.setStringOffset(strOff);
115 st.setName(name);
116 st.setCity(mdrCity);
117 st.setNameOffset((byte) nameOffset);
118 st.setOutNameOffset((byte) outOffset);
119 st.setPrefixOffset((byte) prefix);
120 st.setSuffixOffset((byte) suffix);
121 //System.out.println(st.getName() + ": add partial " + st.getPartialName());
122 allStreets.add(st);
123 start = false;
124 }
125
126 outOffset += outSize(c);
127 if (outOffset > MAX_NAME_OFFSET)
128 break;
129 }
130 }
131
132 /**
133 * Return the number of bytes that the given character will consume in the output encoded
134 * format.
135 */
136 private int outSize(int c) {
137 if (codepage == 65001) {
138 // For unicode a simple lookup gives the number of bytes.
139 if (c < 0x80) {
140 return 1;
141 } else if (c <= 0x7FF) {
142 return 2;
143 } else if (c <= 0xFFFF) {
144 return 3;
145 } else if (c <= 0x10FFFF) {
146 return 4;
147 } else {
148 throw new MapFailedException(String.format("Invalid code point: 0x%x", c));
149 }
150 } else if (!isMulti) {
151 // The traditional single byte code-pages, always one byte.
152 return 1;
153 } else {
154 // Other multi-byte code-pages (eg ms932); can't currently create index for these anyway.
155 return 0;
156 }
44157 }
45158
46159 /**
47160 * Since we change the number of records by removing some after sorting,
48161 * we sort and de-duplicate here.
162 * This is a performance critical part of the index creation process
163 * as it requires a lot of heap to store the sort keys.
49164 */
50165 protected void preWriteImpl() {
51 List<SortKey<Mdr7Record>> sortedStreets = MdrUtils.sortList(getConfig().getSort(), allStreets);
166 Sort sort = getConfig().getSort();
167 List<SortKey<Mdr7Record>> sortedStreets = new ArrayList<>(allStreets.size());
168 for (Mdr7Record m : allStreets) {
169 String partialName = m.getPartialName();
170 String name = m.getName();
171 SortKey<Mdr7Record> nameKey = sort.createSortKey(m, m.getName(), m.getMapIndex());
172 SortKey<Mdr7Record> partialKey = name.equals(partialName) ? nameKey : sort.createSortKey(m, partialName);
173 MultiSortKey<Mdr7Record> sortKey = new MultiSortKey<>(partialKey, nameKey, null);
174 sortedStreets.add(sortKey);
175 }
176 Collections.sort(sortedStreets);
52177
53178 // De-duplicate the street names so that there is only one entry
54179 // per map for the same name.
55180 int recordNumber = 0;
56181 Mdr7Record last = new Mdr7Record();
57 for (SortKey<Mdr7Record> sk : sortedStreets) {
182 for (int i = 0; i < sortedStreets.size(); i++){
183 SortKey<Mdr7Record> sk = sortedStreets.get(i);
58184 Mdr7Record r = sk.getObject();
59 if (r.getMapIndex() != last.getMapIndex() || !r.getName().equals(last.getName())) {
185 if (r.getMapIndex() == last.getMapIndex()
186 && r.getName().equals(last.getName()) // currently think equals is correct, not collator.compare()
187 && r.getPartialName().equals(last.getPartialName()))
188 {
189 // This has the same name (and map number) as the previous one. Save the pointer to that one
190 // which is going into the file.
191 r.setIndex(recordNumber);
192 } else {
60193 recordNumber++;
61194 last = r;
62195 r.setIndex(recordNumber);
63196 streets.add(r);
64 } else {
65 // This has the same name (and map number) as the previous one. Save the pointer to that one
66 // which is going into the file.
67 r.setIndex(recordNumber);
68 }
197 }
198 // release memory
199 sortedStreets.set(i, null);
69200 }
70201 }
71202
72203 public void writeSectData(ImgFileWriter writer) {
73204 String lastName = null;
74 boolean hasStrings = hasFlag(0x1);
205 String lastPartial = null;
206 boolean hasStrings = hasFlag(MDR7_HAS_STRING);
207 boolean hasNameOffset = hasFlag(MDR7_HAS_NAME_OFFSET);
208
75209 for (Mdr7Record s : streets) {
76210 addIndexPointer(s.getMapIndex(), s.getIndex());
77211
78212 putMapIndex(writer, s.getMapIndex());
79213 int lab = s.getLabelOffset();
80 String name = Label.stripGarminCodes(s.getName());
81 int trailingFlags = 0;
214 String name = s.getName();
82215 if (!name.equals(lastName)) {
83216 lab |= 0x800000;
84217 lastName = name;
85 trailingFlags = 1;
86 }
218 }
219
220 String partialName = s.getPartialName();
221 int trailingFlags = 0;
222 if (!partialName.equals(lastPartial)) {
223 trailingFlags |= 1;
224 lab |= 0x800000; // If it is not a partial repeat, then it is not a complete repeat either
225 }
226 lastPartial = partialName;
227
87228 writer.put3(lab);
88229 if (hasStrings)
89230 putStringOffset(writer, s.getStringOffset());
90
91 writer.put((byte) trailingFlags);
231
232 if (hasNameOffset)
233 writer.put(s.getOutNameOffset());
234
235 putN(writer, u2size, trailingFlags);
92236 }
93237 }
94238
98242 */
99243 public int getItemSize() {
100244 PointerSizes sizes = getSizes();
101 int size = sizes.getMapSize() + 3 + 1;
245 int size = sizes.getMapSize() + 3 + u2size;
102246 if (!isForDevice())
103247 size += sizes.getStrOffSize();
248 if ((getExtraValue() & MDR7_HAS_NAME_OFFSET) != 0)
249 size += 1;
104250 return size;
105251 }
106252
112258 * Value of 3 possibly the existence of the lbl field.
113259 */
114260 public int getExtraValue() {
115 int magic = 0x42;
261 int magic = MDR7_U1 | MDR7_HAS_NAME_OFFSET | (u2size << MDR7_PARTIAL_SHIFT);
262
116263 if (isForDevice()) {
117 magic |= 0x4;
264 magic |= MDR7_U2;
118265 } else {
119 magic |= 0x1; //strings
266 magic |= MDR7_HAS_STRING;
120267 }
121268
122269 return magic;
2121 private String name;
2222 private int index;
2323 private Mdr5Record city;
24
25 // For searching on partial names
26 private byte nameOffset; // offset into the name where matching should start
27 private byte outNameOffset; // offset into the encoded output name
28 private byte prefixOffset; // offset after 0x1e prefix
29 private byte suffixOffset; // offset just before 0x1f suffix
2430
2531 public int getLabelOffset() {
2632 return labelOffset;
6268 return city;
6369 }
6470
71 public void setNameOffset(byte nameOffset) {
72 this.nameOffset = nameOffset;
73 }
74
75 public byte getOutNameOffset() {
76 return outNameOffset;
77 }
78
79 public void setOutNameOffset(byte outNameOffset) {
80 this.outNameOffset = outNameOffset;
81 }
82
83 public void setPrefixOffset(byte prefixOffset) {
84 this.prefixOffset = prefixOffset;
85 }
86
87 public void setSuffixOffset(byte suffixOffset) {
88 this.suffixOffset = suffixOffset;
89 }
90
91 /**
92 * Get the name starting at the given nameOffset.
93 *
94 * To avoid creating unnecessary objects, a check is made for an offset of zero
95 * and the original name string is returned.
96 *
97 * @return A substring of name, starting at the nameOffset value.
98 */
99 public String getPartialName() {
100 if (nameOffset == 0 && prefixOffset == 0 && suffixOffset == 0)
101 return name;
102 else if ((suffixOffset & 0xff) > 0)
103 return name.substring((nameOffset & 0xff) + (prefixOffset & 0xff), (suffixOffset & 0xff));
104 else
105 return name.substring((nameOffset & 0xff) + (prefixOffset & 0xff));
106 }
107
65108 public String toString() {
66109 return name + " in " + city.getName();
67110 }
3131 private int headerLen = DEFAULT_HEADER_LEN;
3232 private Sort sort;
3333 private File outputDir;
34 private boolean splitName;
3435
3536 /**
3637 * True if we are creating the file, rather than reading it.
8687 if (outputDir != null)
8788 this.outputDir = new File(outputDir);
8889 }
90
91 public void setSplitName(boolean splitName) {
92 this.splitName = splitName;
93 }
94
95 public boolean isSplitName() {
96 return splitName;
97 }
8998 }
8282 // There is a separate MDR and SRT file for each family id in the gmapsupp
8383 private final Map<Integer, MdrBuilder> mdrBuilderMap = new LinkedHashMap<Integer, MdrBuilder>();
8484 private final Map<Integer, Sort> sortMap = new LinkedHashMap<Integer, Sort>();
85 private boolean splitName;
8586
8687
8788 public void init(CommandArgs args) {
8990 mapsetName = args.get("mapset-name", "OSM map set");
9091 overallDescription = args.getDescription();
9192 outputDir = args.getOutputDir();
93 splitName = args.get("split-name-index", false);
9294 }
9395
9496 /**
105107 return mdrBuilder;
106108
107109 mdrBuilder = new MdrBuilder();
108 mdrBuilder.initForDevice(sort, outputDir);
110 mdrBuilder.initForDevice(sort, outputDir, splitName);
109111 mdrBuilderMap.put(familyId, mdrBuilder);
110112 return mdrBuilder;
111113 }
110110 config.setForDevice(false);
111111 config.setOutputDir(outputDir);
112112 config.setSort(sort);
113 config.setSplitName(args.get("split-name-index", false));
113114
114115 // Wrap the MDR channel with the MDRFile object
115116 mdrFile = new MDRFile(mdrChan, config);
127128 }
128129 }
129130
130 void initForDevice(Sort sort, String outputDir) {
131 void initForDevice(Sort sort, String outputDir, boolean splitName) {
131132 // Set the options that we are using for the mdr.
132133 MdrConfig config = new MdrConfig();
133134 config.setHeaderLen(568);
134135 config.setWritable(true);
135136 config.setForDevice(true);
136137 config.setSort(sort);
138 config.setSplitName(splitName);
137139
138140 // Wrap the MDR channel with the MDRFile object
139141 try {
351351 // Transform ((first | second) & topSecond)
352352 // into (first & topSecond) | (second & topSecond)
353353
354 Op first = op1.getFirst();
355 OrOp orOp = new OrOp();
356
357 Op topSecond = top.getSecond();
358
359 AndOp and1 = new AndOp();
360 and1.setFirst(first);
361 and1.setSecond(topSecond);
362
363 AndOp and2 = new AndOp();
364 Op second = rearrangeExpression(op1.getSecond());
354 return distrubute(op1, top.getSecond());
355 } else {
356 // This shouldn't happen
357 throw new SyntaxException("X3:" + op1.getType());
358 }
359 return top;
360 }
361
362 private static OrOp distrubute(Op op1, Op topSecond) {
363 Op first = op1.getFirst();
364 OrOp orOp = new OrOp();
365
366 BinaryOp and1 = new AndOp();
367 and1.setFirst(first);
368 and1.setSecond(topSecond);
369
370 BinaryOp and2 = new AndOp();
371 Op second = rearrangeExpression(op1.getSecond());
372 if (second.isType(OR)) {
373 and2 = distrubute(second, topSecond);
374 } else {
365375 and2.setFirst(second);
366376 and2.setSecond(topSecond);
367
368 orOp.setFirst(and1);
369 orOp.setSecond(and2);
370 return orOp;
371 } else {
372 // This shouldn't happen
373 throw new SyntaxException("X3:" + op1.getType());
374 }
375 return top;
377 }
378 orOp.setFirst(and1);
379 orOp.setSecond(and2);
380
381 return orOp;
376382 }
377383
378384 /**
4646 if (fileName != null)
4747 fmt.format("(%s:%d): ", fileName, lineNumber);
4848
49 fmt.format(super.getMessage());
49 fmt.format("%s", super.getMessage());
5050 return fmt.toString();
5151 }
5252 }
0
1 WAY
2 highway=primary
3 name=b
4 a=2
5 b=5
6
7 <<<lines>>>
8
9 (highway=secondary | a=2 | b=5) & (highway=service | a=0 | b=5) { set name='a${name}' }
10
11 highway=* [0x2]
12
13 <finalize>
14 highway=* {name '${name}' }
15
16 <<<results>>>
17 WAY 1: Line 0x2, labels=[ab, null, null, null], res=24-24 (1/1),(2/2),
0
1 WAY
2 highway=primary
3 name=b
4 c=60
5 d=50
6
7 <<<lines>>>
8
9 (highway=primary | name=b | d=50) & highway=primary {
10 set name='a${name}'
11 }
12
13 highway=primary [0x2]
14
15 <finalize>
16 highway=* {name '${name}'}
17 <<<results>>>
18 WAY 1: Line 0x2, labels=[ab, null, null, null], res=24-24 (1/1),(2/2),
0
1 WAY
2 highway=tertiary
3 name=b
4 oneway=1
5 cycleway=opposite
6
7 WAY 2
8 highway=tertiary
9 name=b
10 oneway=1
11 cycleway=opposite_lane
12
13
14 <<<lines>>>
15 highway ~ '(secondary|tertiary|unclassified|residential|minor|living_street|service)'
16 & oneway=*
17 & (cycleway=opposite | cycleway=opposite_lane | cycleway=opposite_track )
18 { set name='a${name}' }
19 [0x2 ]
20
21
22 <finalize>
23 highway=* {name '${name}' }
24
25 <<<results>>>
26 WAY 1: Line 0x2, labels=[ab, null, null, null], res=24-24 oneway (1/1),(2/2),
27 WAY 2: Line 0x2, labels=[ab, null, null, null], res=24-24 oneway (1/1),(2/2),
28
29
405405 type = getFirstType(rs, el);
406406 assertNotNull(type);
407407
408 el = el.copy(); // Copy for LinkedOp which remembers the last matched element
408409 el.addTag("cycleway", "opposite_lane");
409410 type = getFirstType(rs, el);
410411 assertNotNull(type);