Libosmium  2.8.0
Fast and flexible C++ library for working with OpenStreetMap data
collector.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_RELATIONS_COLLECTOR_HPP
2 #define OSMIUM_RELATIONS_COLLECTOR_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <algorithm>
37 #include <cassert>
38 #include <cstddef>
39 #include <cstdint>
40 #include <functional>
41 #include <iomanip>
42 //#include <iostream>
43 #include <vector>
44 
45 #include <osmium/fwd.hpp>
46 #include <osmium/osm/item_type.hpp>
47 #include <osmium/osm/object.hpp>
48 #include <osmium/osm/relation.hpp> // IWYU pragma: keep
49 #include <osmium/osm/types.hpp>
50 #include <osmium/handler.hpp>
51 #include <osmium/memory/buffer.hpp>
52 #include <osmium/util/iterator.hpp>
53 #include <osmium/visitor.hpp>
54 
55 #include <osmium/relations/detail/relation_meta.hpp>
56 #include <osmium/relations/detail/member_meta.hpp>
57 
58 namespace osmium {
59 
63  namespace relations {
64 
65  namespace detail {
66 
67  } // namespace detail
68 
97  template <typename TCollector, bool TNodes, bool TWays, bool TRelations>
98  class Collector {
99 
104 
105  TCollector& m_collector;
106 
107  public:
108 
109  HandlerPass1(TCollector& collector) noexcept :
110  m_collector(collector) {
111  }
112 
113  void relation(const osmium::Relation& relation) {
114  if (m_collector.keep_relation(relation)) {
115  m_collector.add_relation(relation);
116  }
117  }
118 
119  }; // class HandlerPass1
120 
121  public:
122 
127 
128  TCollector& m_collector;
129 
130  public:
131 
132  HandlerPass2(TCollector& collector) noexcept :
133  m_collector(collector) {
134  }
135 
136  void node(const osmium::Node& node) {
137  if (TNodes) {
138  if (! m_collector.find_and_add_object(node)) {
139  m_collector.node_not_in_any_relation(node);
140  }
141  }
142  }
143 
144  void way(const osmium::Way& way) {
145  if (TWays) {
146  if (! m_collector.find_and_add_object(way)) {
147  m_collector.way_not_in_any_relation(way);
148  }
149  }
150  }
151 
152  void relation(const osmium::Relation& relation) {
153  if (TRelations) {
154  if (! m_collector.find_and_add_object(relation)) {
155  m_collector.relation_not_in_any_relation(relation);
156  }
157  }
158  }
159 
160  void flush() {
161  m_collector.flush();
162  }
163 
164  }; // class HandlerPass2
165 
166  private:
167 
168  HandlerPass2 m_handler_pass2;
169 
170  // All relations we are interested in will be kept in this buffer
172 
173  // All members we are interested in will be kept in this buffer
175 
177  std::vector<RelationMeta> m_relations;
178 
183  using mm_vector_type = std::vector<MemberMeta>;
184  using mm_iterator = mm_vector_type::iterator;
185  mm_vector_type m_member_meta[3];
186 
187  int m_count_complete = 0;
188 
189  using callback_func_type = std::function<void(osmium::memory::Buffer&&)>;
191 
192  static constexpr size_t initial_buffer_size = 1024 * 1024;
193 
195  auto& mmv = member_meta(type);
196  return make_range(std::equal_range(mmv.begin(), mmv.end(), MemberMeta(id)));
197  }
198 
199  public:
200 
205  m_handler_pass2(*static_cast<TCollector*>(this)),
206  m_relations_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
207  m_members_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
208  m_relations(),
209  m_member_meta() {
210  }
211 
212  protected:
213 
214  std::vector<MemberMeta>& member_meta(const item_type type) {
215  return m_member_meta[static_cast<uint16_t>(type) - 1];
216  }
217 
219  return m_callback;
220  }
221 
222  const std::vector<RelationMeta>& relations() const {
223  return m_relations;
224  }
225 
235  bool keep_relation(const osmium::Relation& /*relation*/) const {
236  return true;
237  }
238 
249  bool keep_member(const osmium::relations::RelationMeta& /*relation_meta*/, const osmium::RelationMember& /*member*/) const {
250  return true;
251  }
252 
260  void node_not_in_any_relation(const osmium::Node& /*node*/) {
261  }
262 
270  void way_not_in_any_relation(const osmium::Way& /*way*/) {
271  }
272 
280  void relation_not_in_any_relation(const osmium::Relation& /*relation*/) {
281  }
282 
294  void flush() {
295  }
296 
302  m_relations.erase(
303  std::remove_if(m_relations.begin(), m_relations.end(), has_all_members()),
304  m_relations.end()
305  );
306  }
307 
308  const osmium::Relation& get_relation(size_t offset) const {
309  assert(m_relations_buffer.committed() > offset);
310  return m_relations_buffer.get<osmium::Relation>(offset);
311  }
312 
316  const osmium::Relation& get_relation(const RelationMeta& relation_meta) const {
317  return get_relation(relation_meta.relation_offset());
318  }
319 
323  const osmium::Relation& get_relation(const MemberMeta& member_meta) const {
324  return get_relation(m_relations[member_meta.relation_pos()]);
325  }
326 
327  osmium::OSMObject& get_member(size_t offset) const {
328  assert(m_members_buffer.committed() > offset);
329  return m_members_buffer.get<osmium::OSMObject>(offset);
330  }
331 
332  private:
333 
342  void add_relation(const osmium::Relation& relation) {
343  const size_t offset = m_relations_buffer.committed();
344  m_relations_buffer.add_item(relation);
345 
346  RelationMeta relation_meta(offset);
347 
348  int n = 0;
349  for (auto& member : m_relations_buffer.get<osmium::Relation>(offset).members()) {
350  if (static_cast<TCollector*>(this)->keep_member(relation_meta, member)) {
351  member_meta(member.type()).emplace_back(member.ref(), m_relations.size(), n);
352  relation_meta.increment_need_members();
353  } else {
354  member.ref(0); // set member id to zero to indicate we are not interested
355  }
356  ++n;
357  }
358 
359  assert(offset == m_relations_buffer.committed());
360  if (relation_meta.has_all_members()) {
361  m_relations_buffer.rollback();
362  } else {
363  m_relations_buffer.commit();
364  m_relations.push_back(std::move(relation_meta));
365  }
366  }
367 
373  std::sort(m_member_meta[0].begin(), m_member_meta[0].end());
374  std::sort(m_member_meta[1].begin(), m_member_meta[1].end());
375  std::sort(m_member_meta[2].begin(), m_member_meta[2].end());
376  }
377 
379  return std::count_if(range.begin(), range.end(), [](MemberMeta& mm) {
380  return !mm.removed();
381  });
382  }
383 
392  auto range = find_member_meta(object.type(), object.id());
393 
394  if (count_not_removed(range) == 0) {
395  // nothing found
396  return false;
397  }
398 
399  {
400  members_buffer().add_item(object);
401  const size_t member_offset = members_buffer().commit();
402 
403  for (auto& member_meta : range) {
404  member_meta.set_buffer_offset(member_offset);
405  }
406  }
407 
408  for (auto& member_meta : range) {
409  if (member_meta.removed()) {
410  break;
411  }
412  assert(member_meta.member_id() == object.id());
413  assert(member_meta.relation_pos() < m_relations.size());
414  RelationMeta& relation_meta = m_relations[member_meta.relation_pos()];
415  assert(member_meta.member_pos() < get_relation(relation_meta).members().size());
416  relation_meta.got_one_member();
417  if (relation_meta.has_all_members()) {
418  const size_t relation_offset = member_meta.relation_pos();
419  static_cast<TCollector*>(this)->complete_relation(relation_meta);
420  clear_member_metas(relation_meta);
421  m_relations[relation_offset] = RelationMeta();
422  possibly_purge_removed_members();
423  }
424  }
425 
426  return true;
427  }
428 
429  void clear_member_metas(const osmium::relations::RelationMeta& relation_meta) {
430  const osmium::Relation& relation = get_relation(relation_meta);
431  for (const auto& member : relation.members()) {
432  if (member.ref() != 0) {
433  auto range = find_member_meta(member.type(), member.ref());
434  assert(!range.empty());
435 
436  // if this is the last time this object was needed
437  // then mark it as removed
438  if (count_not_removed(range) == 1) {
439  get_member(range.begin()->buffer_offset()).set_removed(true);
440  }
441 
442  for (auto& member_meta : range) {
443  if (!member_meta.removed() && relation.id() == get_relation(member_meta).id()) {
444  member_meta.remove();
445  break;
446  }
447  }
448  }
449  }
450  }
451 
452  public:
453 
454  uint64_t used_memory() const {
455  const uint64_t nmembers = m_member_meta[0].capacity() + m_member_meta[1].capacity() + m_member_meta[2].capacity();
456  const uint64_t members = nmembers * sizeof(MemberMeta);
457  const uint64_t relations = m_relations.capacity() * sizeof(RelationMeta);
458  const uint64_t relations_buffer_capacity = m_relations_buffer.capacity();
459  const uint64_t members_buffer_capacity = m_members_buffer.capacity();
460 
461  std::cerr << " nR = m_relations.capacity() ........... = " << std::setw(12) << m_relations.capacity() << "\n";
462  std::cerr << " nMN = m_member_meta[NODE].capacity() ... = " << std::setw(12) << m_member_meta[0].capacity() << "\n";
463  std::cerr << " nMW = m_member_meta[WAY].capacity() .... = " << std::setw(12) << m_member_meta[1].capacity() << "\n";
464  std::cerr << " nMR = m_member_meta[RELATION].capacity() = " << std::setw(12) << m_member_meta[2].capacity() << "\n";
465  std::cerr << " nM = m_member_meta[*].capacity() ...... = " << std::setw(12) << nmembers << "\n";
466 
467  std::cerr << " sRM = sizeof(RelationMeta) ............. = " << std::setw(12) << sizeof(RelationMeta) << "\n";
468  std::cerr << " sMM = sizeof(MemberMeta) ............... = " << std::setw(12) << sizeof(MemberMeta) << "\n\n";
469 
470  std::cerr << " nR * sRM ............................... = " << std::setw(12) << relations << "\n";
471  std::cerr << " nM * sMM ............................... = " << std::setw(12) << members << "\n";
472  std::cerr << " relations_buffer_capacity .............. = " << std::setw(12) << relations_buffer_capacity << "\n";
473  std::cerr << " members_buffer_capacity ................ = " << std::setw(12) << members_buffer_capacity << "\n";
474 
475  const uint64_t total = relations + members + relations_buffer_capacity + members_buffer_capacity;
476 
477  std::cerr << " total .................................. = " << std::setw(12) << total << "\n";
478  std::cerr << " =======================================================\n";
479 
480  return relations_buffer_capacity + members_buffer_capacity + relations + members;
481  }
482 
486  HandlerPass2& handler(const callback_func_type& callback = nullptr) {
487  m_callback = callback;
488  return m_handler_pass2;
489  }
490 
492  return m_members_buffer;
493  }
494 
496  const auto range = find_member_meta(type, id);
497  assert(!range.empty());
498  return range.begin()->buffer_offset();
499  }
500 
501  template <typename TIter>
502  void read_relations(TIter begin, TIter end) {
503  HandlerPass1 handler(*static_cast<TCollector*>(this));
504  osmium::apply(begin, end, handler);
505  sort_member_meta();
506  }
507 
508  template <typename TSource>
509  void read_relations(TSource& source) {
510  read_relations(std::begin(source), std::end(source));
511  source.close();
512  }
513 
514  void moving_in_buffer(size_t old_offset, size_t new_offset) {
515  const osmium::OSMObject& object = m_members_buffer.get<osmium::OSMObject>(old_offset);
516  auto range = find_member_meta(object.type(), object.id());
517  for (auto& member_meta : range) {
518  assert(member_meta.buffer_offset() == old_offset);
519  member_meta.set_buffer_offset(new_offset);
520  }
521  }
522 
530  ++m_count_complete;
531  if (m_count_complete > 10000) { // XXX
532 // const size_t size_before = m_members_buffer.committed();
533  m_members_buffer.purge_removed(this);
534 /*
535  const size_t size_after = m_members_buffer.committed();
536  double percent = static_cast<double>(size_before - size_after);
537  percent /= size_before;
538  percent *= 100;
539  std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast<int>(percent) << "%)\n";
540 */
541  m_count_complete = 0;
542  }
543  }
544 
553  std::vector<const osmium::Relation*> get_incomplete_relations() const {
554  std::vector<const osmium::Relation*> relations;
555  for (const auto& relation_meta : m_relations) {
556  if (!relation_meta.has_all_members()) {
557  relations.push_back(&get_relation(relation_meta));
558  }
559  }
560  return relations;
561  }
562 
563  }; // class Collector
564 
565  } // namespace relations
566 
567 } // namespace osmium
568 
569 #endif // OSMIUM_RELATIONS_COLLECTOR_HPP
std::vector< const osmium::Relation * > get_incomplete_relations() const
Definition: collector.hpp:553
void relation(const osmium::Relation &relation)
Definition: collector.hpp:152
void clear_member_metas(const osmium::relations::RelationMeta &relation_meta)
Definition: collector.hpp:429
Definition: iterator.hpp:42
callback_func_type m_callback
Definition: collector.hpp:190
osmium::memory::Buffer & members_buffer()
Definition: collector.hpp:491
type
Definition: entity_bits.hpp:63
iterator_range< It > make_range(P &&p)
Definition: iterator.hpp:76
RelationMemberList & members()
Definition: relation.hpp:177
void way(const osmium::Way &way)
Definition: collector.hpp:144
item_type
Definition: item_type.hpp:43
void clean_assembled_relations()
Definition: collector.hpp:301
void moving_in_buffer(size_t old_offset, size_t new_offset)
Definition: collector.hpp:514
Definition: relation.hpp:165
size_t capacity() const noexcept
Definition: buffer.hpp:233
Definition: handler.hpp:45
bool keep_member(const osmium::relations::RelationMeta &, const osmium::RelationMember &) const
Definition: collector.hpp:249
osmium::memory::Buffer m_members_buffer
Definition: collector.hpp:174
const std::vector< RelationMeta > & relations() const
Definition: collector.hpp:222
void read_relations(TSource &source)
Definition: collector.hpp:509
size_t get_offset(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:495
void relation_not_in_any_relation(const osmium::Relation &)
Definition: collector.hpp:280
Definition: way.hpp:65
std::vector< MemberMeta > & member_meta(const item_type type)
Definition: collector.hpp:214
bool find_and_add_object(const osmium::OSMObject &object)
Definition: collector.hpp:391
Definition: collector.hpp:126
HandlerPass2(TCollector &collector) noexcept
Definition: collector.hpp:132
std::vector< RelationMeta > m_relations
Vector with all relations we are interested in.
Definition: collector.hpp:177
void sort_member_meta()
Definition: collector.hpp:372
void apply(TIterator it, TIterator end, THandlers &...handlers)
Definition: visitor.hpp:234
void node_not_in_any_relation(const osmium::Node &)
Definition: collector.hpp:260
std::function< void(osmium::memory::Buffer &&)> callback_func_type
Definition: collector.hpp:189
Definition: relation.hpp:54
TCollector & m_collector
Definition: collector.hpp:105
Namespace for everything in the Osmium library.
Definition: assembler.hpp:66
T & add_item(const T &item)
Definition: buffer.hpp:457
Definition: collector.hpp:98
iterator_range< mm_iterator > find_member_meta(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:194
const osmium::Relation & get_relation(size_t offset) const
Definition: collector.hpp:308
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:700
Definition: attr.hpp:298
HandlerPass2 m_handler_pass2
Definition: collector.hpp:168
Collector()
Definition: collector.hpp:204
osmium::OSMObject & get_member(size_t offset) const
Definition: collector.hpp:327
void relation(const osmium::Relation &relation)
Definition: collector.hpp:113
osmium::io::InputIterator< osmium::io::Reader > end(osmium::io::Reader &)
Definition: reader_iterator.hpp:45
Definition: collector.hpp:103
int64_t object_id_type
Type for OSM object (node, way, or relation) IDs.
Definition: types.hpp:45
void flush()
Definition: collector.hpp:160
size_t committed() const noexcept
Definition: buffer.hpp:241
std::vector< MemberMeta > mm_vector_type
Definition: collector.hpp:183
osmium::memory::Buffer m_relations_buffer
Definition: collector.hpp:171
uint64_t used_memory() const
Definition: collector.hpp:454
callback_func_type callback()
Definition: collector.hpp:218
Definition: buffer.hpp:97
T & get(const size_t offset) const
Definition: buffer.hpp:379
void add_relation(const osmium::Relation &relation)
Definition: collector.hpp:342
HandlerPass2 & handler(const callback_func_type &callback=nullptr)
Definition: collector.hpp:486
object_id_type id() const noexcept
Get ID of this object.
Definition: object.hpp:112
void flush()
Definition: collector.hpp:294
Definition: node.hpp:47
void way_not_in_any_relation(const osmium::Way &)
Definition: collector.hpp:270
void node(const osmium::Node &node)
Definition: collector.hpp:136
void possibly_purge_removed_members()
Definition: collector.hpp:529
bool keep_relation(const osmium::Relation &) const
Definition: collector.hpp:235
const osmium::Relation & get_relation(const MemberMeta &member_meta) const
Definition: collector.hpp:323
It begin() const
Definition: iterator.hpp:58
TCollector & m_collector
Definition: collector.hpp:128
void read_relations(TIter begin, TIter end)
Definition: collector.hpp:502
osmium::io::InputIterator< osmium::io::Reader > begin(osmium::io::Reader &reader)
Definition: reader_iterator.hpp:41
HandlerPass1(TCollector &collector) noexcept
Definition: collector.hpp:109
void rollback()
Definition: buffer.hpp:349
const osmium::Relation & get_relation(const RelationMeta &relation_meta) const
Definition: collector.hpp:316
static iterator_range< mm_iterator >::iterator::difference_type count_not_removed(const iterator_range< mm_iterator > &range)
Definition: collector.hpp:378
It end() const
Definition: iterator.hpp:62
Definition: object.hpp:60
size_t commit()
Definition: buffer.hpp:335