2 * This file is part of mapper
4 * Copyright (C) 2008 Kaj-Michael Lang
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License along
17 * with this program; if not, write to the Free Software Foundation, Inc.,
18 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
22 * Routines to read OSM planet XML file and store it in a sqlite3 database.
23 * Reads in all nodes (if used, skips nodes outside bounding box)
24 * Special POI nodes are stored in POI table.
25 * Place POI nodes are stored in place table.
27 * Ways are read in and their data (name, type, etc) are stored
28 * in way, way_name and way_ref tables.
30 * Nodes used by they ways are stored in way_n2n table.
38 #include <sys/types.h>
43 #include <glib/gstdio.h>
51 #include "osm-db-import.h"
56 /* #define VERBOSE_KEYS */
59 /* Use g_convert to transliterate names.. my iconv seems to be fucked so this didn't work... */
60 /* #define TRANSLIT_NAMES */
62 #define FILE_BUFFER (128*1024)
64 static guint node_cnt=0; /* Nodes */
65 static guint node_skip_cnt=0; /* Skipped nodes */
66 static guint noded_cnt=0; /* Nodes with (usable) data */
67 static guint way_cnt=0; /* Ways */
68 static guint way_names=0; /* Ways with name */
69 static guint way_refs=0; /* Ways with ref or int_ref */
71 static guint dbnode_cnt=0;
72 static guint dbnoded_cnt=0;
73 static guint dbway_cnt=0;
75 /* For threaded importing */
76 static GThread* import_thread=NULL;
77 static GSourceFunc osm_import_progress_cb=NULL;
78 static osm_import_data_req osm_import_req;
79 static guint import_sid=0;
81 static gboolean is_update=FALSE;
107 /* Node types table */
108 /* XXX: Add support for parent category */
113 { "amenity", "fuel", NODE_AMENITY_FUEL },
114 { "amenity", "parking", NODE_AMENITY_PARKING },
116 { "amenity", "pub", NODE_AMENITY_PUB },
117 { "amenity", "nightclub", NODE_AMENITY_NIGHTCLUB },
118 { "amenity", "biergarten", NODE_AMENITY_PUB },
119 { "amenity", "cafe", NODE_AMENITY_CAFE },
120 { "amenity", "fast_food", NODE_AMENITY_FOOD },
121 { "amenity", "restaurant", NODE_AMENITY_RESTAURANT },
123 { "amenity", "telephone", NODE_AMENITY_TELEPHONE },
124 { "amenity", "toilets", NODE_AMENITY_WC },
126 { "amenity", "hospital", NODE_AMENITY_HOSPITAL },
127 { "amenity", "doctors", NODE_AMENITY_HOSPITAL },
128 { "amenity", "pharmacy", NODE_AMENITY_PHARMACY },
130 { "amenity", "post_office", NODE_AMENITY_POST },
131 { "amenity", "post_box", NODE_AMENITY_POST_BOX },
133 { "amenity", "cinema", NODE_AMENITY_CINEMA },
134 { "amenity", "theatre", NODE_AMENITY_THEATRE },
136 { "amenity", "atm", NODE_AMENITY_ATM },
137 { "amenity", "bank", NODE_AMENITY_BANK },
139 { "amenity", "police", NODE_AMENITY_POLICE },
141 { "amenity", "speed_trap", NODE_AMENITY_SPEEDCAM },
142 { "amenity", "speed_camera", NODE_AMENITY_SPEEDCAM },
143 { "amenity", "speed camera", NODE_AMENITY_SPEEDCAM },
145 { "amenity", "place_of_worship",NODE_AMENITY_POW },
147 { "amenity", "school", NODE_AMENITY_SCHOOL },
148 { "amenity", "college", NODE_AMENITY_COLLEGE },
149 { "amenity", "university", NODE_AMENITY_COLLEGE },
151 { "amenity", "library", NODE_AMENITY_LIBRARY },
152 { "amenity", "townhall", NODE_AMENITY_TOWNHALL },
154 { "amenity", "supermarket", NODE_AMENITY_SHOP_SUPERMARKET },
155 { "amenity", "shopping_centre", NODE_AMENITY_SHOPPING_CENTER },
156 { "amenity", "shop", NODE_AMENITY_SHOP },
157 { "amenity", "shops", NODE_AMENITY_SHOP },
158 { "amenity", "shopping", NODE_AMENITY_SHOPPING_CENTER },
159 { "amenity", "shopping_mall",NODE_AMENITY_SHOPPING_CENTER },
160 { "amenity", "cycle_shop", NODE_AMENITY_SHOP },
161 { "amenity", "bike_shop", NODE_AMENITY_SHOP },
162 { "amenity", "coffee_shop", NODE_AMENITY_SHOP },
163 { "amenity", "indoor_shopping_centre", NODE_AMENITY_SHOP },
164 { "amenity", "farm_shop", NODE_AMENITY_SHOP },
165 { "amenity", "tea_shop", NODE_AMENITY_SHOP },
168 { "shop", "supermarket", NODE_AMENITY_SHOP_SUPERMARKET },
169 { "shop", "alcohol", NODE_AMENITY_SHOP_ALCOHOL },
170 { "shop", "flowers", NODE_AMENITY_SHOP_FLOWERS },
171 { "shop", "bakery", NODE_AMENITY_SHOP },
172 { "shop", "butcher", NODE_AMENITY_SHOP },
173 { "shop", "clothing", NODE_AMENITY_SHOP },
174 { "shop", "souvenir", NODE_AMENITY_SHOP },
175 { "shop", "bicycles", NODE_AMENITY_SHOP },
176 { "shop", "grocers", NODE_AMENITY_SHOP },
177 { "shop", "newsagents", NODE_AMENITY_SHOP },
178 { "shop", "convenience", NODE_AMENITY_SHOP },
179 { "shop", "bakers", NODE_AMENITY_SHOP },
180 { "shop", "garden_centre",NODE_AMENITY_SHOP },
181 { "shop", "photography", NODE_AMENITY_SHOP },
182 { "shop", "general_store",NODE_AMENITY_SHOP },
183 { "shop", "food", NODE_AMENITY_SHOP },
184 { "shop", "drinks", NODE_AMENITY_SHOP },
185 { "shop", "sex", NODE_AMENITY_SHOP_ADULT },
187 { "shop", "pharmacy", NODE_AMENITY_PHARMACY },
190 { "sport" , "swimming", NODE_SPORT_SWIMMING },
191 { "sport" , "golf", NODE_SPORT_GOLF },
192 { "sport" , "tennis", NODE_SPORT_TENNIS },
193 { "sport" , "football", NODE_SPORT_FOOTBALL },
194 { "sport" , "soccer", NODE_SPORT_SOCCER },
195 { "sport" , "baskteball", NODE_SPORT_BASKETBALL },
196 { "sport" , "rugby", NODE_SPORT_RUGBY },
197 { "sport" , "skating", NODE_SPORT_SKATING },
198 { "sport" , "hockey", NODE_SPORT_HOCKEY },
199 { "sport" , "skateboard", NODE_SPORT_SKATEBOARD },
200 { "sport" , "bowling", NODE_SPORT_BOWLING },
201 { "sport" , "10pin", NODE_SPORT_BOWLING },
202 { "sport" , "motor", NODE_SPORT_MOTOR },
203 { "sport" , "shooting_range",NODE_SPORT_SHOOTING },
204 { "sport" , "paintball", NODE_SPORT_PAINTBALL },
205 { "sport" , "horse_racing",NODE_SPORT_HORSES },
206 { "sport" , "horse", NODE_SPORT_HORSES },
207 { "sport" , "horses", NODE_SPORT_HORSES },
208 { "sport" , "dog_racing", NODE_SPORT_DOG },
209 { "sport" , "pelota", NODE_SPORT_PELOTA },
210 { "sport" , "racquet", NODE_SPORT_RACQUET },
211 { "sport" , "equestrian", NODE_SPORT_HORSES },
212 { "sport" , "baseball", NODE_SPORT_BASEBALL },
213 { "sport" , "cricket", NODE_SPORT_CRICKET },
214 { "sport" , "croquet", NODE_SPORT_CROQUET },
215 { "sport" , "cycling", NODE_SPORT_CYCLING },
216 { "sport" , "bowls", NODE_SPORT_BOWLS },
217 { "sport" , "athletics", NODE_SPORT_ATHLETICS },
218 { "sport" , "gymnastics", NODE_SPORT_GYMNASTICS },
219 { "sport" , "multi", NODE_SPORT_OTHER },
220 { "leisure", "sport_centre",NODE_SPORT_CENTER },
223 { "tourism", "information", NODE_TOURISM_INFO },
224 { "tourism", "camp_site", NODE_TOURISM_CAMP_SITE },
225 { "tourism", "caravan_site",NODE_TOURISM_CARAVAN_SITE },
226 { "tourism", "picnic_site", NODE_TOURISM_PICNIC_SITE },
227 { "tourism", "theme_park", NODE_TOURISM_THEME_PARK },
228 { "tourism", "hotel", NODE_TOURISM_HOTEL },
229 { "tourism", "motel", NODE_TOURISM_MOTEL },
230 { "tourism", "hostel", NODE_TOURISM_HOSTEL },
231 { "tourism", "attraction", NODE_TOURISM_ATTRACTION },
232 { "tourism", "zoo", NODE_TOURISM_ATTRACTION },
234 { "historic", "ruins", NODE_TOURISM_ATTRACTION },
235 { "historic", "monument", NODE_TOURISM_ATTRACTION },
236 { "historic", "memorial", NODE_TOURISM_ATTRACTION },
237 { "historic", "museum", NODE_HISTORIC_MUSEUM },
238 { "historic", "castle", NODE_HISTORIC_CASTLE },
240 { "railway", "station", NODE_RAILWAY_STATION },
241 { "railway", "halt", NODE_RAILWAY_HALT },
243 { "aeroway", "terminal", NODE_AIRPORT_TERMINAL },
244 { "aeroway", "aerodrome", NODE_AIRPORT_TERMINAL },
247 { "place", "city", NODE_PLACE_CITY },
248 { "place", "town", NODE_PLACE_TOWN },
249 { "place", "village", NODE_PLACE_VILLAGE },
250 { "place", "hamlet", NODE_PLACE_HAMLET },
251 { "place", "locality", NODE_PLACE_LOCALITY },
252 { "place", "suburb", NODE_PLACE_SUBURB },
253 { "place", "island", NODE_PLACE_ISLAND },
255 { "highway", "traffic_signals", NODE_TRAFFIC_SIGNALS },
256 { "highway", "motorway_junction", NODE_JUNCTION },
257 { "highway", "services", NODE_AMENITY_PARKING },
258 { "highway", "toll_booth", NODE_TOLLBOOTH },
259 { "highway", "gate", NODE_GATE },
261 { NULL, NULL, NODE_PLAIN }
264 /* Array to get id number and defaults for ways of different types */
269 gboolean oneway, link, area, car, foot;
271 { "highway", "motorway",120,WAY_MOTORWAY, TRUE, FALSE, FALSE, TRUE, FALSE },
272 { "highway", "motorway_link",120,WAY_MOTORWAY, TRUE, TRUE, FALSE, TRUE, FALSE },
273 { "highway", "trunk",100,WAY_TRUNK, FALSE, FALSE, FALSE, TRUE, FALSE },
274 { "highway", "trunk_link",100,WAY_TRUNK, FALSE, TRUE, FALSE, TRUE, FALSE },
275 { "highway", "primary",80,WAY_PRIMARY, FALSE, FALSE, FALSE, TRUE, TRUE },
276 { "highway", "primary_link",60,WAY_PRIMARY, FALSE, TRUE, FALSE, TRUE, TRUE },
277 { "highway", "secondary",80,WAY_SECONDARY, FALSE, FALSE, FALSE, TRUE, TRUE },
278 { "highway", "secondary_link",60,WAY_SECONDARY, FALSE, TRUE, FALSE, TRUE, TRUE },
279 { "highway", "tertiary",60,WAY_TERTIARY, FALSE, FALSE, FALSE, TRUE, TRUE },
280 { "highway", "unclassified",50,WAY_UNCLASSIFIED, FALSE, FALSE, FALSE, TRUE, TRUE },
281 { "highway", "byway",40,WAY_UNCLASSIFIED, FALSE, FALSE, FALSE, TRUE, TRUE },
282 { "highway", "residential",40,WAY_RESIDENTIAL, FALSE, FALSE, FALSE, TRUE, TRUE },
283 { "highway", "service",20,WAY_SERVICE, FALSE, FALSE, FALSE, TRUE, TRUE },
284 { "highway", "track",20,WAY_TRACK, FALSE, FALSE, FALSE, TRUE, TRUE },
285 { "highway", "unsurfaced",60,WAY_TRACK, FALSE, FALSE, FALSE, TRUE, TRUE },
286 { "highway", "minor",60,WAY_TRACK, FALSE, FALSE, FALSE, TRUE, TRUE },
287 { "highway", "pedestrian",20,WAY_FOOTWAY, FALSE, FALSE, FALSE, FALSE, TRUE },
288 { "highway", "footway",1,WAY_FOOTWAY, FALSE, FALSE, FALSE, FALSE, TRUE },
289 { "highway", "steps",0,WAY_FOOTWAY, FALSE, FALSE, FALSE, FALSE, TRUE},
290 { "highway", "bridleway",10,WAY_FOOTWAY, FALSE, FALSE, FALSE, FALSE, TRUE },
291 { "highway", "cycleway",10,WAY_CYCLEWAY, FALSE, FALSE, FALSE, FALSE, TRUE },
292 { "railway", "rail",0,WAY_RAIL, FALSE, FALSE, FALSE, FALSE, FALSE },
293 { "aeroway", "runway",0,WAY_RUNWAY, FALSE, FALSE, FALSE, FALSE, FALSE },
294 { "aeroway", "taxiway",0,WAY_TAXIWAY, FALSE, FALSE, FALSE, FALSE, FALSE },
295 { "natural", "water",0,WAY_WATER, FALSE, FALSE, TRUE, FALSE, FALSE },
296 { "waterway", "river",0,WAY_WATER, FALSE, FALSE, FALSE, FALSE, FALSE },
297 { "waterway", "canal",0,WAY_WATER, FALSE, FALSE, FALSE, FALSE, FALSE },
298 { "waterway", "stream",0,WAY_WATER, FALSE, FALSE, FALSE, FALSE, FALSE },
299 { "building", "*",0,WAY_UNWAYED, FALSE, FALSE, TRUE, FALSE, FALSE },
300 { NULL, NULL, 0, WAY_UNWAYED, FALSE, FALSE, FALSE, FALSE, FALSE }
304 tag_parent_t tag_parent=IS_NONE;
306 static GHashTable *osm_nodes;
307 static GHashTable *osm_node_tags;
308 static GHashTable *osm_way_tags;
309 static GSList *osm_ways;
310 static GSList *osm_poi;
312 static GHashTable *osm_place_country;
313 static GHashTable *osm_place_region;
314 static GHashTable *osm_place_city;
315 static GHashTable *osm_place_suburb;
316 static GHashTable *osm_place_village;
317 static GHashTable *osm_node_isin;
318 static GHashTable *osm_way_isin;
320 static node *cnode=NULL;
321 static way *cway=NULL;
325 sqlite3_stmt *insert_poi;
326 sqlite3_stmt *delete_osm_poi;
329 sqlite3_stmt *insert_place;
330 sqlite3_stmt *delete_places;
333 sqlite3_stmt *insert_node;
334 sqlite3_stmt *delete_nodes;
335 sqlite3_stmt *select_node;
336 sqlite3_stmt *update_node;
339 sqlite3_stmt *insert_way_data;
340 sqlite3_stmt *insert_way_ref;
341 sqlite3_stmt *insert_way_pc;
342 sqlite3_stmt *insert_way_name;
343 sqlite3_stmt *insert_way_names_nls;
344 sqlite3_stmt *insert_way_n2n;
345 sqlite3_stmt *delete_way;
346 sqlite3_stmt *delete_way_n2n;
347 sqlite3_stmt *delete_way_name;
348 sqlite3_stmt *delete_way_names_nls;
349 sqlite3_stmt *delete_way_ref;
350 sqlite3_stmt *delete_way_pc;
352 static struct sql_stmt sql;
354 static struct map_bbox bbox;
355 static gboolean use_bbox;
357 static void osm_free_way_data(way *w);
358 static void print_way(way *w);
360 static gboolean osm_db_prepare(sqlite3 *db);
361 static gboolean db_insert_node(node *n);
362 static guint32 osm_find_way_place(way *w, node_type_t nt);
364 /****************************************************/
366 /****************************************************/
371 sqlite3_finalize(sql.insert_poi);
372 sqlite3_finalize(sql.delete_osm_poi);
374 sqlite3_finalize(sql.insert_node);
375 sqlite3_finalize(sql.select_node);
376 sqlite3_finalize(sql.delete_nodes);
377 sqlite3_finalize(sql.update_node);
379 sqlite3_finalize(sql.insert_place);
380 sqlite3_finalize(sql.delete_places);
382 sqlite3_finalize(sql.delete_way);
383 sqlite3_finalize(sql.insert_way_data);
385 sqlite3_finalize(sql.delete_way_name);
386 sqlite3_finalize(sql.insert_way_name);
388 sqlite3_finalize(sql.delete_way_n2n);
389 sqlite3_finalize(sql.insert_way_n2n);
391 sqlite3_finalize(sql.delete_way_pc);
392 sqlite3_finalize(sql.insert_way_pc);
394 sqlite3_finalize(sql.delete_way_names_nls);
395 sqlite3_finalize(sql.insert_way_names_nls);
399 osm_db_prepare(sqlite3 *db)
402 sqlite3_prepare_v2(db, "insert or replace into nodes (nid,ilat,ilon,rlat,rlon,l,f) values (?,?,?,?,?,0,?)", -1, &sql.insert_node, NULL);
403 sqlite3_prepare_v2(db, "select ilat,ilon,l from nodes where nid=?", -1, &sql.select_node, NULL);
404 sqlite3_prepare_v2(db, "delete from nodes", -1, &sql.delete_nodes, NULL);
405 sqlite3_prepare_v2(db, "update nodes set l=l+1 where nid=?", -1, &sql.update_node, NULL);
408 sqlite3_prepare_v2(db, "insert or replace into places (nid,type,name,isin_c,isin_p) values (?, ?, ?, ?, ?)", -1, &sql.insert_place, NULL);
409 sqlite3_prepare_v2(db, "delete from places", -1, &sql.delete_places, NULL);
412 if (sqlite3_prepare_v2(db, "insert or replace into poi (osm_id, lat, lon, label, cat_id, public, source, priority, isin_c, isin_p, desc, url, postal_code) "
413 " values (?, ?, ?, ?, ?, 1, 1, ?, ?, ?, ?, ?, ?)", -1, &sql.insert_poi, NULL)!=SQLITE_OK)
414 g_printf("SQL: %s\n", sqlite3_errmsg(db));
416 sqlite3_prepare_v2(db, "delete from poi where osm_id>0 and source=1", -1, &sql.delete_osm_poi, NULL);
419 sqlite3_prepare_v2(db, "insert or replace into way (wid,nodes,type,flags,speed,isin_c,isin_p,lat,lon) values (?, ?, ?, ?, ?, ?, ?, ?, ?)", -1, &sql.insert_way_data, NULL);
420 sqlite3_prepare_v2(db, "delete from way", -1, &sql.delete_way, NULL);
423 sqlite3_prepare_v2(db, "insert into way_n2n (wid,f,t) values (?,?,?)", -1, &sql.insert_way_n2n, NULL);
424 sqlite3_prepare_v2(db, "delete from way_n2n where wid=?", -1, &sql.delete_way_n2n, NULL);
427 sqlite3_prepare_v2(db, "insert or replace into way_names (wid,name,norm) values (?, ?, ?)", -1, &sql.insert_way_name, NULL);
428 sqlite3_prepare_v2(db, "delete from way_names", -1, &sql.delete_way_name, NULL);
430 /* Way postal codes */
431 sqlite3_prepare_v2(db, "insert or replace into way_pc (wid,pc) values (?, ?)", -1, &sql.insert_way_pc, NULL);
432 sqlite3_prepare_v2(db, "delete from way_pc", -1, &sql.delete_way_pc, NULL);
434 /* Other language names for ways */
435 sqlite3_prepare_v2(db, "insert into way_names_nls (wid,lang,name, norm) values (?, ?, ?, ?)", -1, &sql.insert_way_names_nls, NULL);
436 sqlite3_prepare_v2(db, "delete from way_names_nls where wid=?", -1, &sql.delete_way_names_nls, NULL);
438 /* Way ref and int_ref */
439 sqlite3_prepare_v2(db, "insert or replace into way_ref (rid,ref,int_ref) values (?, ?, ?)", -1, &sql.insert_way_ref, NULL);
440 sqlite3_prepare_v2(db, "delete from way_ref", -1, &sql.delete_way_ref, NULL);
445 /********************************************************************/
452 g_printf("Way #%d(N:%d T:%d S:%d IS: %d/%d): %s [%s:%s:%s]\n",
454 g_slist_length(w->nodes),
456 w->data ? w->data->speed : 0,
457 w->data ? w->data->isin_c : -1,
458 w->data ? w->data->isin_p : -1,
459 w->data ? w->data->name ? w->data->name : "" : "",
460 w->flags & W_ONEWAY ? "-" : "=",
461 w->flags & W_ROUNDABOUT ? "O" : "-",
462 w->flags & W_LINK ? "|" : " ");
471 g_printf("Node #%d: T:%d IS: %d/%d [%s]\n",
474 n->data ? n->data->isin_c : -1,
475 n->data ? n->data->isin_p : -1,
476 n->data ? n->data->name : "");
480 /********************************************************************/
483 db_insert_node(node *n)
489 lat=lat2mp_int(n->lat);
490 lon=lon2mp_int(n->lon);
492 sqlite3_bind_int(sql.insert_node, 1, n->id);
494 /* Projected and integerized lat/lot */
495 sqlite3_bind_int(sql.insert_node, 2, lat);
496 sqlite3_bind_int(sql.insert_node, 3, lon);
498 sqlite3_bind_double(sql.insert_node, 4, n->lat);
499 sqlite3_bind_double(sql.insert_node, 5, n->lon);
500 sqlite3_bind_int(sql.insert_node, 6, n->type);
502 db_exec(db, sql.insert_node);
508 db_insert_place(node *n)
515 sqlite3_bind_int(sql.insert_place, 1, n->id);
516 sqlite3_bind_int(sql.insert_place, 2, n->type);
517 sqlite3_bind_text(sql.insert_place, 3, n->data->name, -1, SQLITE_TRANSIENT);
518 sqlite3_bind_int(sql.insert_place, 4, n->data->isin_p);
519 sqlite3_bind_int(sql.insert_place, 5, n->data->isin_c);
521 return db_exec(db,sql.insert_place);
525 db_insert_poi(node *n)
528 sqlite3_bind_int(sql.insert_poi, 1, n->id);
529 sqlite3_bind_double(sql.insert_poi, 2, n->lat);
530 sqlite3_bind_double(sql.insert_poi, 3, n->lon);
532 sqlite3_bind_text(sql.insert_poi, 4, n->data->name, -1, SQLITE_TRANSIENT);
534 sqlite3_bind_text(sql.insert_poi, 4, "", -1, SQLITE_TRANSIENT);
535 sqlite3_bind_int(sql.insert_poi, 5, n->type);
536 sqlite3_bind_int(sql.insert_poi, 6, n->type/100);
537 sqlite3_bind_int(sql.insert_poi, 7, n->data->isin_c);
538 sqlite3_bind_int(sql.insert_poi, 8, n->data->isin_p);
541 sqlite3_bind_text(sql.insert_poi, 9, n->data->desc, -1, SQLITE_TRANSIENT);
543 sqlite3_bind_text(sql.insert_poi, 10, n->data->url, -1, SQLITE_TRANSIENT);
544 if (n->data->postal_code)
545 sqlite3_bind_text(sql.insert_poi, 11, n->data->postal_code, -1, SQLITE_TRANSIENT);
547 return db_exec(db,sql.insert_poi);
551 * Update node usage count
554 db_update_node_links(node *n)
557 sqlite3_bind_int(sql.update_node, 1, n->id);
559 return db_exec(db,sql.update_node);
563 * Insert way,node1,node2 triplet
566 db_insert_way_n2n(way *w, node *nf, node *nt)
568 g_return_val_if_fail(w, FALSE);
569 g_return_val_if_fail(nf, FALSE);
570 g_return_val_if_fail(nt, FALSE);
572 sqlite3_bind_int(sql.insert_way_n2n, 1, w->id);
573 sqlite3_bind_int(sql.insert_way_n2n, 2, nf->id);
574 sqlite3_bind_int(sql.insert_way_n2n, 3, nt->id);
577 g_printf("%d [%d - %d]\n", w->id, nf->id, nt->id);
580 db_exec(db,sql.insert_way_n2n);
581 db_update_node_links(nf);
582 db_update_node_links(nt);
587 * Insert way ref and int_ref
590 db_insert_way_ref(way *w)
595 if (!w->data->ref && !w->data->int_ref)
600 sqlite3_bind_int(sql.insert_way_ref, 1, w->id);
602 sqlite3_bind_text(sql.insert_way_ref, 2, w->data->ref, -1, SQLITE_TRANSIENT);
603 if (w->data->int_ref)
604 sqlite3_bind_text(sql.insert_way_ref, 3, w->data->int_ref, -1, SQLITE_TRANSIENT);
606 return db_exec(db,sql.insert_way_ref);
613 db_insert_way_name(way *w)
622 sqlite3_bind_int(sql.insert_way_name, 1, w->id);
623 sqlite3_bind_text(sql.insert_way_name, 2, w->data->name, -1, SQLITE_TRANSIENT);
625 #ifdef TRANSLIT_NAMES
628 norm=g_convert(w->data->name, -1, "ASCII//TRANSLIT//IGNORE", "utf8", NULL, NULL, NULL);
629 if (norm && strcmp(w->data->name, norm)!=0) {
630 sqlite3_bind_text(sql.insert_way_name, 3, norm, -1, SQLITE_TRANSIENT);
637 return db_exec(db,sql.insert_way_name);
641 db_delete_way_names_nls(way *w)
643 sqlite3_bind_int(sql.delete_way_names_nls, 1, w->id);
644 return db_exec(db,sql.delete_way_names_nls);
648 db_insert_way_pc(way *w)
652 if (!w->data->postal_code)
655 sqlite3_bind_int(sql.insert_way_pc, 1, w->id);
656 sqlite3_bind_text(sql.insert_way_pc, 2, w->data->postal_code, -1, SQLITE_TRANSIENT);
658 return db_exec(db,sql.insert_way_pc);
662 db_delete_way_pc(way *w)
664 sqlite3_bind_int(sql.delete_way_pc, 1, w->id);
665 return db_exec(db,sql.delete_way_pc);
669 db_insert_way_names_nls_cb(gpointer key, gpointer value, gpointer user_data)
671 way *w=(way *)user_data;
673 sqlite3_bind_int(sql.insert_way_names_nls, 1, w->id);
674 sqlite3_bind_text(sql.insert_way_names_nls, 2, (gchar *)key, -1, SQLITE_TRANSIENT);
675 sqlite3_bind_text(sql.insert_way_names_nls, 3, (gchar *)value, -1, SQLITE_TRANSIENT);
676 #ifdef TRANSLIT_NAMES
679 norm=g_convert((gchar *value), -1, "ASCII//TRANSLIT//IGNORE", "utf8", NULL, NULL, NULL);
680 if (norm && strcmp((gchar *)value, norm)!=0) {
681 sqlite3_bind_text(sql.insert_way_names_nls, 4, norm, -1, SQLITE_TRANSIENT);
687 db_exec(db,sql.insert_way_names_nls);
691 db_insert_way_names_nls(way *w)
698 g_hash_table_foreach(w->data->names, db_insert_way_names_nls_cb, w);
702 * Insert all data for the given way
709 db_insert_way(way *w)
718 /* Skip things we don't use (yet) */
719 if (w->type==WAY_UNWAYED || w->type>WAY_ROAD_END)
723 for (iter=w->nodes; iter!=NULL; iter=iter->next) {
726 db_insert_way_n2n(w, iter->data, iter->next->data);
733 w->data->isin_p=osm_find_way_place(w, NODE_PLACE_CITY);
734 w->data->isin_c=osm_find_way_place(w, NODE_PLACE_COUNTRY);
739 sqlite3_bind_int(sql.insert_way_data, 1, w->id);
740 sqlite3_bind_int(sql.insert_way_data, 2, w->ncnt);
741 sqlite3_bind_int(sql.insert_way_data, 3, w->type);
742 sqlite3_bind_int(sql.insert_way_data, 4, w->flags);
744 sqlite3_bind_int(sql.insert_way_data, 5, w->data->speed);
745 sqlite3_bind_int(sql.insert_way_data, 6, w->data->isin_c);
746 sqlite3_bind_int(sql.insert_way_data, 7, w->data->isin_p);
749 /* Get middle node, use it as a rough way location */
750 ncnt=g_slist_length(w->nodes);
752 wmn=g_slist_nth_data(w->nodes, ncnt/2);
754 sqlite3_bind_double(sql.insert_way_data, 8, wmn->lat);
755 sqlite3_bind_double(sql.insert_way_data, 9, wmn->lon);
757 g_printerr("Failed to get way middlepoint node for location information!\n");
761 db_exec(db,sql.insert_way_data);
763 db_insert_way_ref(w);
764 db_insert_way_name(w);
765 db_insert_way_names_nls(w);
768 osm_free_way_data(w);
772 /********************************************************************/
775 get_attr_key_value(const gchar **p, gchar *key)
781 if (strncmp(*d, key, strlen(key))==0) {
792 check_tag(const gchar *tag)
794 if (strcmp(tag,"node")==0) return IN_NODE_TAG;
795 else if (strcmp(tag,"nd")==0) return IN_WNODE_TAG;
796 else if (strcmp(tag,"way")==0) return IN_WAY_TAG;
797 else if (strcmp(tag,"tag")==0) return IN_KEY_TAG;
798 else if (strcmp(tag,"osm")==0) return IN_OSM_TAG;
799 else if (strcmp(tag,"bound")==0) return IN_BOUND_TAG;
800 else if (strcmp(tag,"relation")==0) return IN_RELATION_TAG;
801 else if (strcmp(tag,"member")==0) return IN_MEMBER_TAG;
806 find_nls_names(gpointer key, gpointer value, gpointer user_data)
814 nls=(GHashTable *)user_data;
816 /* Check if it is a name key, return if not. */
817 if (g_str_has_prefix(k, "name:")==FALSE)
820 tmp=g_strrstr(k, ":");
826 g_hash_table_insert(nls, g_strdup(tmp), g_strdup(v));
828 g_printf("NLS(%s): [%s]\n", tmp, v);
832 /********************************************************************/
839 g_printf("N: %d [%f:%f][%s](%d)\n",
840 n->id, n->lat, n->lon,
841 n->data->name ? n->data->name : "-",
844 g_printf("N: %d [%f:%f]\n",
845 n->id, n->lat, n->lon);
851 dump_array(const gchar **p)
857 g_printf("[%s]", *d);
864 static inline gboolean
865 osm_node_check_box(gdouble nlat, gdouble nlon)
869 return (nlat > bbox.lat_min && nlat < bbox.lat_max && nlon > bbox.lon_min && nlon < bbox.lon_max) ? TRUE : FALSE;
873 osm_new_node_data(node *n)
879 n->data=g_slice_new0(node_data);
885 osm_free_node_data(node *n)
890 g_free(n->data->name);
892 g_free(n->data->url);
894 g_free(n->data->desc);
895 if (n->data->postal_code)
896 g_free(n->data->postal_code);
897 g_slice_free(node_data, n->data);
903 osm_new_node(gint id, gdouble lat, gdouble lon)
907 n=g_slice_new0(node);
916 osm_free_node(node *n)
919 g_slice_free(node, n);
923 osm_find_node(guint32 nid)
928 return g_hash_table_lookup(osm_nodes, GINT_TO_POINTER(nid));
932 osm_new_way_data(way *w)
939 w->data=g_slice_new0(way_data);
943 osm_free_way_data(way *w)
949 g_free(w->data->name);
951 g_free(w->data->ref);
952 if (w->data->int_ref)
953 g_free(w->data->int_ref);
954 g_slice_free(way_data, w->data);
968 /* Add to list of ways */
976 g_slist_free(w->nodes);
977 g_slice_free(way, w);
981 osm_way_add_to_list(way *w)
984 osm_ways=g_slist_prepend(osm_ways, w);
988 osm_way_new_node(way *w, gint nid)
993 n=osm_find_node(nid);
994 w->nodes=g_slist_prepend(w->nodes, n);
999 * Search the place hash table for the location of the node.
1003 osm_find_node_place(node *n)
1012 isin=g_hash_table_lookup(osm_node_isin, GINT_TO_POINTER(n->id));
1018 while (*place!=NULL) {
1020 ps=g_strstrip(*place);
1022 g_printf("Checking (%d) [%s] in [%s]\n",n->type, n->data->name, ps);
1025 case NODE_PLACE_CITY:
1026 case NODE_PLACE_TOWN:
1027 case NODE_PLACE_VILLAGE:
1028 case NODE_PLACE_HAMLET:
1029 t=g_hash_table_lookup(osm_place_region, ps);
1032 t=g_hash_table_lookup(osm_place_country, ps);
1036 case NODE_PLACE_SUBURB:
1037 case NODE_PLACE_LOCALITY:
1038 t=g_hash_table_lookup(osm_place_city, ps);
1042 case NODE_PLACE_ISLAND:
1046 t=g_hash_table_lookup(osm_place_city, ps);
1058 osm_find_way_place(way *w, node_type_t nt)
1063 isin=g_hash_table_lookup(osm_way_isin, GINT_TO_POINTER(w->id));
1068 while (*place!=NULL) {
1072 ps=g_strstrip(*place);
1075 g_printf("Checking (%d) in [%s]\n",w->id, ps);
1078 case NODE_PLACE_CITY:
1079 case NODE_PLACE_TOWN:
1080 case NODE_PLACE_VILLAGE:
1081 case NODE_PLACE_HAMLET:
1082 case NODE_PLACE_LOCALITY:
1083 t=g_hash_table_lookup(osm_place_city, ps);
1087 case NODE_PLACE_COUNTRY:
1088 t=g_hash_table_lookup(osm_place_country, ps);
1093 g_assert_not_reached();
1102 /***********************************************************************/
1105 osm_node_save_node(gint key, gpointer value, gpointer user_data)
1107 node *n=(node *)value;
1111 if (dbnode_cnt % 20000==0)
1112 g_printf("\rNodes: %f%% (%u/%u)\n",
1113 ((float)dbnode_cnt/(float)(node_cnt-node_skip_cnt))*100,
1114 dbnode_cnt, node_cnt);
1118 * Check node type and insert as POI or Place
1119 * Discard extra data after insert.
1122 osm_node_save_poi(node *n, gpointer user_data)
1125 g_printerr("ERROR: null poi\n");
1130 g_printerr("POI node with no data ?\n");
1134 n->data->isin_p=osm_find_node_place(n);
1137 if (n->type>NODE_POI_START && n->type<NODE_POI_END) {
1140 osm_free_node_data(n);
1141 } else if (n->type>NODE_PLACE_START && n->type<NODE_PLACE_END) {
1145 osm_free_node_data(n);
1153 osm_planet_poi_clear_nodes(void)
1155 g_print("Removing old OSM POIs...\n");
1156 db_transaction_begin(db);
1157 sqlite3_step(sql.delete_osm_poi);
1158 sqlite3_step(sql.delete_places);
1159 return db_transaction_commit(db);
1163 osm_planet_poi_save_nodes(void)
1165 g_print("Storing new POIs...\n");
1166 db_transaction_begin(db);
1167 g_slist_foreach(osm_poi, osm_node_save_poi, NULL);
1168 g_slist_free(osm_poi);
1169 return db_transaction_commit(db);
1172 /*********************************************************************/
1175 osm_planet_clear_nodes(void)
1177 g_print("Clearing old nodes...\n");
1178 db_transaction_begin(db);
1179 sqlite3_step(sql.delete_nodes);
1180 return db_transaction_commit(db);
1184 osm_planet_save_nodes(void)
1186 g_print("Storing new nodes...\n");
1187 db_transaction_begin(db);
1188 g_hash_table_foreach(osm_nodes, osm_node_save_node, NULL);
1189 return db_transaction_commit(db);
1192 /*********************************************************************/
1195 osm_way_save(way *value, gpointer user_data)
1198 db_transaction_begin(db);
1199 db_insert_way(value);
1200 db_transaction_commit(db);
1201 if (dbway_cnt % 15000==0 && dbway_cnt>0) {
1202 g_printf("\rWays: %f%% (%u/%u)\n",
1203 (((float)dbway_cnt/(float)way_cnt)*100),
1204 dbway_cnt, way_cnt);
1210 osm_planet_clear_ways(void)
1212 g_print("Clearing old way data...\n");
1213 db_transaction_begin(db);
1214 sqlite3_step(sql.delete_way);
1215 sqlite3_step(sql.delete_way_name);
1216 sqlite3_step(sql.delete_way_ref);
1217 sqlite3_step(sql.delete_way_n2n);
1218 db_transaction_commit(db);
1222 osm_planet_save_ways(void)
1224 g_print("Inserting new ways\n");
1225 g_slist_foreach(osm_ways, osm_way_save, NULL);
1228 /*********************************************************************/
1231 osm_planet_save_all_nodes(void)
1233 g_printf("Saving planet nodes to database:\n");
1235 osm_planet_poi_clear_nodes();
1236 osm_planet_poi_save_nodes();
1239 osm_planet_clear_nodes();
1240 osm_planet_clear_ways();
1242 osm_planet_save_nodes();
1246 osm_planet_save_all_ways(void)
1248 g_printf("Saving planet way to database:\n");
1250 osm_planet_save_ways();
1253 /***********************************************************************/
1256 _osm_tag_start(void *userData, const char *name, const char **atts)
1266 g_printf("Starting...\n");
1272 id=atoi(get_attr_key_value(atts, "id"));
1273 nlat=atof(get_attr_key_value(atts, "lat"));
1274 nlon=atof(get_attr_key_value(atts, "lon"));
1276 cnode=osm_new_node(id, nlat, nlon);
1277 osm_node_tags=g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free);
1282 id=atoi(get_attr_key_value(atts, "id"));
1283 cway=osm_new_way(id);
1284 osm_way_tags=g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free);
1287 ndref=atoi(get_attr_key_value(atts, "ref"));
1288 if (use_bbox==TRUE) {
1289 if (osm_find_node(ndref)==NULL) {
1294 osm_way_new_node(cway, ndref);
1297 k=get_attr_key_value(atts, "k");
1298 if (strcmp(k,"created_by")==0)
1300 if (strcmp(k,"source")==0)
1303 v=get_attr_key_value(atts, "v");
1305 g_printf("TAG: K=[%s] V=[%s]\n", k, v);
1308 switch (tag_parent) {
1310 g_printf("Tag key/value pair but unknown owner\n");
1317 /* Insert key/value pairs into hash table */
1319 g_printerr("In node tags but node is NULL!\n");
1322 g_hash_table_insert(osm_node_tags, g_strdup(k), g_strdup(v));
1328 g_printerr("In way tags but way is NULL!\n");
1331 g_hash_table_insert(osm_way_tags, g_strdup(k), g_strdup(v));
1332 osm_new_way_data(cway);
1341 /* Ignore for now */
1342 g_printf("Ignoring bound tag\n");
1344 case IN_RELATION_TAG:
1345 tag_parent=IS_RELATION;
1353 g_printf("Unknown tag: %s\n", name);
1358 #define GET_NODE_KEY(key, nfield) { \
1360 _kv=g_hash_table_lookup(osm_node_tags, key); \
1363 nfield=g_strstrip(g_utf8_normalize(_kv, -1, G_NORMALIZE_ALL_COMPOSE)); \
1365 gchar *_tmp=nfield; \
1366 gchar *_norm=g_utf8_normalize(_kv, -1, G_NORMALIZE_ALL_COMPOSE); \
1367 nfield=g_strdup_printf("%s %s", _tmp, g_strstrip(_norm)); \
1376 _osm_tag_end(void *userData, const char *name)
1385 if (node_cnt % 25000==0) {
1386 g_printf("Nodes: %u of %u, POIs: %u, Outside box: %u\n",
1387 node_cnt-node_skip_cnt, node_cnt, noded_cnt, node_skip_cnt);
1393 osm_new_node_data(cnode);
1395 for (i=0; nodeinfo[i].k; i++) {
1396 v=g_hash_table_lookup(osm_node_tags, nodeinfo[i].k);
1399 if (strcasecmp (v, nodeinfo[i].v)==0) {
1400 cnode->type=nodeinfo[i].type;
1405 /* Check if node is inside bounding box, if not skip it.
1406 * But keep it if it's something we might need for other nodes:
1407 * - Places (for is_in)
1410 if ((osm_node_check_box(cnode->lat, cnode->lon)==FALSE) && (cnode->type<NODE_PLACE_START)) {
1411 osm_free_node_data(cnode);
1412 osm_free_node(cnode);
1413 g_hash_table_destroy(osm_node_tags);
1418 g_hash_table_insert(osm_nodes, GINT_TO_POINTER(cnode->id), cnode);
1420 if (cnode->type!=NODE_PLAIN) {
1421 cnode->data->name=NULL;
1422 v=g_hash_table_lookup(osm_node_tags, "name");
1424 cnode->data->name=g_strstrip(g_utf8_normalize(v, -1, G_NORMALIZE_ALL_COMPOSE));
1426 GET_NODE_KEY("description", cnode->data->desc);
1427 GET_NODE_KEY("note", cnode->data->desc);
1428 GET_NODE_KEY("postal_code", cnode->data->postal_code);
1429 GET_NODE_KEY("address", cnode->data->desc);
1432 v=g_hash_table_lookup(osm_node_tags, "url");
1434 cnode->data->url=g_strstrip(g_strdup(v));
1436 v=g_hash_table_lookup(osm_node_tags, "wikipedia");
1437 if (v && strncmp(v,"http:", 5)==0)
1438 cnode->data->url=g_strstrip(g_strdup(v));
1442 cnode->data->isin_c=0;
1443 cnode->data->isin_p=0;
1444 v=g_hash_table_lookup(osm_node_tags, "is_in");
1447 isin=g_strsplit(v, ",", 10);
1448 g_hash_table_insert(osm_node_isin, GINT_TO_POINTER(cnode->id), isin);
1451 if (cnode->type==NODE_PLAIN) {
1452 osm_free_node_data(cnode);
1454 osm_poi=g_slist_prepend(osm_poi, cnode);
1455 if (cnode->data->name) {
1456 switch (cnode->type) {
1457 case NODE_PLACE_COUNTRY:
1458 g_hash_table_insert(osm_place_country, cnode->data->name, cnode);
1460 case NODE_PLACE_CITY:
1461 case NODE_PLACE_TOWN:
1462 g_hash_table_insert(osm_place_city, cnode->data->name, cnode);
1464 case NODE_PLACE_SUBURB:
1465 g_hash_table_insert(osm_place_suburb, cnode->data->name, cnode);
1467 case NODE_PLACE_VILLAGE:
1468 case NODE_PLACE_HAMLET:
1469 case NODE_PLACE_LOCALITY:
1470 g_hash_table_insert(osm_place_village, cnode->data->name, cnode);
1472 case NODE_PLACE_ISLAND:
1473 /* Ignore for now */
1479 g_hash_table_destroy(osm_node_tags);
1483 if (way_cnt % 1024==0) {
1484 g_printf("\rWays: %d\n", way_cnt);
1487 cway->nodes=g_slist_reverse(cway->nodes);
1489 for (i=0; wayinfo[i].k; i++) {
1490 v=g_hash_table_lookup(osm_way_tags, wayinfo[i].k);
1493 if (strcasecmp (v, wayinfo[i].v)==0) {
1494 if (wayinfo[i].link==TRUE)
1495 cway->flags|=W_LINK;
1496 if (wayinfo[i].area==TRUE)
1497 cway->flags|=W_AREA;
1498 if (wayinfo[i].oneway==TRUE)
1499 cway->flags|=W_ONEWAY;
1500 cway->type=wayinfo[i].type;
1501 if (cway->data->speed==0)
1502 cway->data->speed=wayinfo[i].defspeed;
1507 v=g_hash_table_lookup(osm_way_tags, "name");
1509 cway->data->name=g_utf8_normalize(v, -1, G_NORMALIZE_ALL_COMPOSE);
1510 /* Try to find other language names */
1511 cway->data->names=g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free);
1512 g_hash_table_foreach(osm_way_tags, find_nls_names, cway->data->names);
1513 if (g_hash_table_size(cway->data->names)==0) {
1514 g_hash_table_destroy(cway->data->names);
1515 cway->data->names=NULL;
1519 v=g_hash_table_lookup(osm_way_tags, "ref");
1521 cway->data->ref=g_strdup(v);
1522 v=g_hash_table_lookup(osm_way_tags, "int_ref");
1524 cway->data->int_ref=g_strdup(v);
1526 v=g_hash_table_lookup(osm_way_tags, "postal_code");
1528 cway->data->postal_code=g_strdup(v);
1530 v=g_hash_table_lookup(osm_way_tags, "oneway");
1532 cway->flags|=W_ONEWAY;
1533 if (strcmp(v, "-1")==0)
1534 cway->nodes=g_slist_reverse(cway->nodes);
1537 v=g_hash_table_lookup(osm_way_tags, "noexit");
1539 cway->flags|=W_NOEXIT;
1541 v=g_hash_table_lookup(osm_way_tags, "speedlimit");
1543 cway->data->speed=atoi(v);
1544 v=g_hash_table_lookup(osm_way_tags, "maxspeed");
1546 cway->data->speed=atoi(v);
1548 v=g_hash_table_lookup(osm_way_tags, "layer");
1550 cway->data->layer=atoi(v);
1552 v=g_hash_table_lookup(osm_way_tags, "junction");
1553 if (v && strcasecmp(v,"roundabout")==0) {
1554 cway->flags|=W_ROUNDABOUT;
1555 cway->flags|=W_ONEWAY;
1556 } else if (v && strcasecmp(v,"mini_roundabout")==0) {
1557 cway->flags|=W_ROUNDABOUT;
1558 cway->flags|=W_ONEWAY;
1561 /* XXX: Should check keys */
1562 v=g_hash_table_lookup(osm_way_tags, "access");
1563 if (v && (strcasecmp(v, "private")==0)) {
1564 cway->flags|=W_NOACCESS;
1569 v=g_hash_table_lookup(osm_way_tags, "is_in");
1572 isin=g_strsplit(v, ",", 10);
1573 g_hash_table_insert(osm_way_isin, GINT_TO_POINTER(cway->id), isin);
1576 if (cway->data && cway->data->name==NULL && cway->data->ref==NULL &&
1577 cway->data->int_ref==NULL && cway->data->layer==0 && cway->data->speed==0)
1578 osm_free_way_data(cway);
1581 osm_way_add_to_list(cway);
1586 g_hash_table_destroy(osm_way_tags);
1592 g_printf("\nPlanet loaded.\n");
1598 /************************************************************************/
1603 osm_nodes=g_hash_table_new(g_direct_hash, g_direct_equal);
1605 osm_place_country=g_hash_table_new(g_str_hash, g_str_equal);
1606 osm_place_city=g_hash_table_new(g_str_hash, g_str_equal);
1607 osm_place_suburb=g_hash_table_new(g_str_hash, g_str_equal);
1608 osm_place_village=g_hash_table_new(g_str_hash, g_str_equal);
1609 osm_place_region=g_hash_table_new(g_str_hash, g_str_equal);
1610 osm_node_isin=g_hash_table_new(g_direct_hash, g_direct_equal);
1611 osm_way_isin=g_hash_table_new(g_direct_hash, g_direct_equal);
1617 g_hash_table_destroy(osm_nodes);
1619 g_hash_table_destroy(osm_place_country);
1620 g_hash_table_destroy(osm_place_city);
1621 g_hash_table_destroy(osm_place_suburb);
1622 g_hash_table_destroy(osm_place_village);
1623 g_hash_table_destroy(osm_place_region);
1624 g_hash_table_destroy(osm_node_isin);
1627 /************************************************************************/
1630 print_fail(const gchar *msg, gint ret)
1632 g_printerr("ERROR: %s\n", msg);
1636 /************************************************************************/
1639 print_memory_usage(void)
1641 g_print("Memory usage per item:\n");
1642 g_printf("Node size: %d\n", (gint)sizeof(node));
1643 g_printf("NodeD size: %d\n", (gint)sizeof(node_data));
1644 g_printf("Way size: %d\n", (gint)sizeof(way));
1645 g_printf("WayD size: %d\n", (gint)sizeof(way_data));
1648 /************************************************************************
1650 ************************************************************************/
1653 osm_planet_parser_init(void)
1655 xp=XML_ParserCreate(NULL);
1656 XML_SetElementHandler(xp, _osm_tag_start, _osm_tag_end);
1661 osm_planet_parser_deinit(void)
1668 osm_planet_parse_buffer(const gchar *buffer, size_t r)
1670 if (XML_Parse(xp, buffer, r, r>0 ? 0:1) == XML_STATUS_ERROR) {
1671 g_printerr("Parse error at line %d:\n%s\n",
1672 (gint)XML_GetCurrentLineNumber(xp),
1673 XML_ErrorString(XML_GetErrorCode(xp)));
1680 osm_planet_parse_file(gchar *pfile)
1686 gchar buffer[FILE_BUFFER];
1689 f=fopen(pfile, "r");
1691 perror("fopen failed\n");
1695 b=BZ2_bzReadOpen(&bzerror, f, 0, 0, NULL, 0);
1696 if (bzerror != BZ_OK) {
1697 g_printf("BZ2_bzReadOpen failed\n");
1698 BZ2_bzReadClose(&bzerror, b);
1703 r=BZ2_bzRead(&bzerror, b, buffer, FILE_BUFFER);
1704 if ((bzerror!=BZ_STREAM_END) && (bzerror!=BZ_OK)) {
1708 if (!osm_planet_parse_buffer(buffer, r)) {
1712 } while (bzerror==BZ_OK);
1714 BZ2_bzReadClose(&bzerror, b);
1720 * Set up bounding box for import.
1724 osm_import_set_bbox(gboolean use_bb, gdouble latmin, gdouble lonmin, gdouble latmax, gdouble lonmax)
1727 bbox.lat_min=latmin;
1728 bbox.lon_min=lonmin;
1729 bbox.lat_max=latmax;
1730 bbox.lon_max=lonmax;
1731 g_printf("Skipping data outside of box: %f,%f - %f,%f\n",
1732 bbox.lat_min, bbox.lon_min, bbox.lat_max, bbox.lon_max);
1736 osm_print_import_stats(void)
1738 g_printf("Total nodes %d, POIs: %d and Ways %d.\n", node_cnt, noded_cnt, way_cnt);
1739 g_printf("Cities/Towns: %d\n", g_hash_table_size(osm_place_city));
1740 g_printf("Villages/Hamlets: %d\n", g_hash_table_size(osm_place_village));
1741 g_printf("Suburbs: %d\n", g_hash_table_size(osm_place_suburb));
1742 g_printf("Nodes: %d\n", g_hash_table_size(osm_nodes));
1746 * Simple helper to do all preparations and importing from planet -> database
1750 osm_import(const gchar *planet, const gchar *database)
1752 if (db_connect(&db, database)!=TRUE) {
1753 g_printerr("Database open failed: %s", database);
1757 /* Set some sqlite PRAGMAs to speed up import */
1758 sqlite3_exec(db, "PRAGMA cache_size=16000;", NULL, NULL, NULL);
1759 sqlite3_exec(db, "PRAGMA locking_mode=EXCLUSIVE;", NULL, NULL, NULL);
1760 sqlite3_exec(db, "PRAGMA journal_mode=PERSIST;", NULL, NULL, NULL);
1761 sqlite3_exec(db, "PRAGMA synchronous=OFF;", NULL, NULL, NULL);
1763 if (!osm_db_create(db)) {
1764 g_printerr("Failed to create osm tables or indexes\n");
1768 if (!osm_db_prepare(db)) {
1769 g_printerr("Failed to prepare sql statements\n");
1773 osm_planet_parser_init();
1775 if (osm_planet_parse_file(planet)==FALSE) {
1776 g_printerr("Failed to parse file: %s\n", planet);
1780 osm_print_import_stats();
1782 osm_planet_save_all_nodes();
1783 osm_planet_save_all_ways();
1784 osm_planet_parser_deinit();
1787 g_printf("All done.\n");
1792 osm_import_thread(gpointer user_data)
1795 osm_import_data_req *req=(osm_import_data_req *)user_data;
1798 g_assert(req->planet);
1801 osm_import_progress_cb=req->progress_cb!=NULL ? req->progress_cb : NULL;
1803 r=osm_import(req->planet, req->db);
1804 g_debug("OSM import result: %d", r);
1806 g_free(req->planet);
1809 if (req->done_cb!=NULL)
1810 g_idle_add(req->done_cb, GINT_TO_POINTER(r==TRUE ? 0 : 1));
1812 return r==TRUE ? 0 : 1;
1816 * Helper to start an import in the background using a thread.
1818 * Two callback can be given, one for progress feedback and one when the operation is done.
1819 * Done callback must call the join function.
1820 * Only one import thread can run at a time.
1824 osm_import_bg(const gchar *planet, const gchar *database, GSourceFunc progress_cb, GSourceFunc done_cb)
1828 g_return_val_if_fail(import_thread==NULL, FALSE);
1830 osm_import_req.planet=g_strdup(planet);
1831 osm_import_req.db=g_strdup(database);
1832 osm_import_req.progress_cb=progress_cb;
1833 osm_import_req.done_cb=done_cb;
1835 import_thread=g_thread_create(osm_import_thread, &osm_import_req, TRUE, &error);
1836 if (import_thread==NULL) {
1837 g_free(osm_import_req.planet);
1838 g_free(osm_import_req.db);
1839 g_printerr("Import thread creation failed.\n");
1842 if (osm_import_progress_cb!=NULL)
1843 import_sid=g_timeout_add(1000, osm_import_progress_cb, NULL);
1848 osm_import_join_bg(void)
1851 g_assert(import_thread!=NULL);
1854 g_source_remove(import_sid);
1855 ret=g_thread_join(import_thread);