err.no Git - mapper/blob - src/osm-db-import.c

   1 /*
   2  * This file is part of mapper
   3  *
   4  * Copyright (C) 2007 Kaj-Michael Lang
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along
  17  * with this program; if not, write to the Free Software Foundation, Inc.,
  18  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  19  */
  20
  21 /*
  22  * Routines to read OSM planet XML file and store it in a sqlite3 database.
  23  * Reads in all nodes (if used, skips nodes outside bounding box)
  24  * Special POI nodes are stored in POI table.
  25  * Place POI nodes are stored in place table.
  26  *
  27  * Ways are read in and their data (name, type, etc) are stored
  28  * in way, way_name and way_ref tables.
  29  *
  30  * Nodes used by they ways are stored in way_n2n table.
  31  *
  32  */
  33
  34 #include <stdio.h>
  35 #include <unistd.h>
  36 #include <string.h>
  37 #include <strings.h>
  38 #include <sys/types.h>
  39 #include <sys/stat.h>
  40 #include <fcntl.h>
  41 #include <math.h>
  42 #include <glib.h>
  43 #include <glib/gstdio.h>
  44 #include <sqlite3.h>
  45 #include <expat.h>
  46 #include <bzlib.h>
  47
  48 #include "osm.h"
  49 #include "latlon.h"
  50 #include "db.h"
  51 #include "osm-db-import.h"
  52
  53 #if 0
  54 #define VERBOSE
  55 #endif
  56 /* #define VERBOSE_KEYS */
  57
  58
  59 /* Use g_convert to transliterate names.. my iconv seems to be fucked so this didn't work... */
  60 /* #define TRANSLIT_NAMES */
  61
  62 #define FILE_BUFFER (128*1024)
  63
  64 static guint node_cnt=0;
  65 static guint node_skip_cnt=0;
  66 static guint noded_cnt=0;
  67 static guint way_cnt=0;
  68 static guint way_names=0;
  69 static guint way_refs=0;
  70
  71 static guint dbnode_cnt=0;
  72 static guint dbnoded_cnt=0;
  73 static guint dbway_cnt=0;
  74
  75 static gboolean is_update=FALSE;
  76 static XML_Parser xp;
  77
  78
  79 /* XML tag IDs */
  80 typedef enum {
  81         START,
  82         IN_OSM_TAG,
  83         IN_NODE_TAG,
  84         IN_WNODE_TAG,
  85         IN_WAY_TAG,
  86         IN_KEY_TAG,
  87         IN_BOUND_TAG,
  88         IN_RELATION_TAG,
  89         IN_MEMBER_TAG,
  90         END,
  91         ERROR
  92 } tag_state_t;
  93
  94 /* Parent tag type */
  95 typedef enum {
  96         IS_NONE,
  97         IS_NODE,
  98         IS_WAY,
  99         IS_RELATION
 100 } tag_parent_t;
 101
 102 /* Node types table */
 103 /* XXX: Add support for parent category */
 104 struct _nodeinfo {
 105         gchar *k, *v;
 106         node_type_t type;
 107 } nodeinfo[] = {
 108         { "amenity", "fuel",            NODE_AMENITY_FUEL },
 109         { "amenity", "parking",         NODE_AMENITY_PARKING },
 110
 111         { "amenity", "pub",                     NODE_AMENITY_PUB },
 112         { "amenity", "nightclub",       NODE_AMENITY_NIGHTCLUB },
 113         { "amenity", "biergarten",      NODE_AMENITY_PUB },
 114         { "amenity", "cafe",            NODE_AMENITY_CAFE },
 115         { "amenity", "fast_food",       NODE_AMENITY_FOOD },
 116         { "amenity", "restaurant",      NODE_AMENITY_FOOD },
 117
 118         { "amenity", "telephone",       NODE_AMENITY_TELEPHONE },
 119         { "amenity", "toilets",         NODE_AMENITY_WC },
 120
 121         { "amenity", "hospital",        NODE_AMENITY_HOSPITAL },
 122         { "amenity", "doctors",         NODE_AMENITY_HOSPITAL },
 123         { "amenity", "pharmacy",        NODE_AMENITY_PHARMACY },
 124
 125         { "amenity", "post_office",     NODE_AMENITY_POST },
 126         { "amenity", "post_box",        NODE_AMENITY_POST_BOX },
 127
 128         { "amenity", "cinema",          NODE_AMENITY_CINEMA },
 129         { "amenity", "theatre",         NODE_AMENITY_THEATRE },
 130
 131         { "amenity", "atm",                     NODE_AMENITY_ATM },
 132         { "amenity", "bank",            NODE_AMENITY_BANK },
 133
 134         { "amenity", "police",          NODE_AMENITY_POLICE },
 135         { "amenity", "speed_trap",      NODE_AMENITY_SPEEDCAM },
 136         { "amenity", "speed_camera",    NODE_AMENITY_SPEEDCAM },
 137         { "amenity", "speed camera",    NODE_AMENITY_SPEEDCAM },
 138
 139         { "amenity", "place_of_worship",NODE_AMENITY_POW },
 140
 141         { "amenity", "school",          NODE_AMENITY_SCHOOL },
 142         { "amenity", "college",         NODE_AMENITY_COLLEGE },
 143         { "amenity", "university",      NODE_AMENITY_COLLEGE },
 144
 145         { "amenity", "library", NODE_AMENITY_LIBRARY },
 146         { "amenity", "townhall",        NODE_AMENITY_TOWNHALL },
 147
 148         { "amenity", "supermarket",     NODE_AMENITY_SHOP },
 149         { "amenity", "shopping_centre", NODE_AMENITY_SHOP },
 150         { "amenity", "shop",            NODE_AMENITY_SHOP },
 151         { "amenity", "shops",           NODE_AMENITY_SHOP },
 152         { "amenity", "shopping",        NODE_AMENITY_SHOP },
 153         { "amenity", "shopping_mall",NODE_AMENITY_SHOP },
 154         { "amenity", "cycle_shop",      NODE_AMENITY_SHOP },
 155         { "amenity", "bike_shop",       NODE_AMENITY_SHOP },
 156         { "amenity", "coffee_shop",     NODE_AMENITY_SHOP },
 157         { "amenity", "indoor_shopping_centre",  NODE_AMENITY_SHOP },
 158         { "amenity", "farm_shop",       NODE_AMENITY_SHOP },
 159         { "amenity", "tea_shop",        NODE_AMENITY_SHOP },
 160
 161         /* Shops */
 162         { "shop",        "supermarket", NODE_AMENITY_SHOP },
 163         { "shop",        "bakery",              NODE_AMENITY_SHOP },
 164         { "shop",        "alcohol",             NODE_AMENITY_SHOP },
 165         { "shop",        "butcher",             NODE_AMENITY_SHOP },
 166         { "shop",        "flowers",             NODE_AMENITY_SHOP },
 167         { "shop",        "clothing",    NODE_AMENITY_SHOP },
 168         { "shop",        "souvenir",    NODE_AMENITY_SHOP },
 169         { "shop",        "bicycles",    NODE_AMENITY_SHOP },
 170         { "shop",        "grocers",             NODE_AMENITY_SHOP },
 171         { "shop",        "newsagents",  NODE_AMENITY_SHOP },
 172         { "shop",        "convenience", NODE_AMENITY_SHOP },
 173         { "shop",        "bakers",              NODE_AMENITY_SHOP },
 174         { "shop",        "garden_centre",NODE_AMENITY_SHOP },
 175         { "shop",        "photography", NODE_AMENITY_SHOP },
 176         { "shop",        "general_store",NODE_AMENITY_SHOP },
 177         { "shop",        "food",                NODE_AMENITY_SHOP },
 178         { "shop",        "drinks",              NODE_AMENITY_SHOP },
 179         { "shop",        "sex",                 NODE_AMENITY_SHOP_ADULT },
 180         { "shop",        "pharmacy",    NODE_AMENITY_PHARMACY },
 181
 182         /* Sport */
 183         { "sport"  , "swimming",        NODE_SPORT_SWIMMING },
 184         { "sport"  , "golf",            NODE_SPORT_GOLF },
 185         { "sport"  , "tennis",          NODE_SPORT_TENNIS },
 186         { "sport"  , "football",        NODE_SPORT_FOOTBALL },
 187         { "sport"  , "soccer",          NODE_SPORT_SOCCER },
 188         { "sport"  , "baskteball",      NODE_SPORT_BASKETBALL },
 189         { "sport"  , "rugby",           NODE_SPORT_RUGBY },
 190         { "sport"  , "skating",         NODE_SPORT_SKATING },
 191         { "sport"  , "hockey",          NODE_SPORT_HOCKEY },
 192         { "sport"  , "skateboard",      NODE_SPORT_SKATEBOARD },
 193         { "sport"  , "bowling",         NODE_SPORT_BOWLING },
 194         { "sport"  , "10pin",           NODE_SPORT_BOWLING },
 195         { "sport"  , "motor",           NODE_SPORT_MOTOR },
 196         { "sport"  , "shooting_range",NODE_SPORT_SHOOTING },
 197         { "sport"  , "paintball",       NODE_SPORT_PAINTBALL },
 198         { "sport"  , "horse_racing",NODE_SPORT_HORSES },
 199         { "sport"  , "horse",           NODE_SPORT_HORSES },
 200         { "sport"  , "horses",          NODE_SPORT_HORSES },
 201         { "sport"  , "dog_racing",      NODE_SPORT_DOG },
 202         { "sport"  , "pelota",          NODE_SPORT_PELOTA },
 203         { "sport"  , "racquet",         NODE_SPORT_RACQUET },
 204         { "sport"  , "equestrian",      NODE_SPORT_HORSES },
 205         { "sport"  , "baseball",        NODE_SPORT_BASEBALL },
 206         { "sport"  , "cricket",         NODE_SPORT_CRICKET },
 207         { "sport"  , "croquet",         NODE_SPORT_CROQUET },
 208         { "sport"  , "cycling",         NODE_SPORT_CYCLING },
 209         { "sport"  , "bowls",           NODE_SPORT_BOWLS },
 210         { "sport"  , "athletics",       NODE_SPORT_ATHLETICS },
 211         { "sport"  , "gymnastics",      NODE_SPORT_GYMNASTICS },
 212         { "sport"  , "multi",           NODE_SPORT_OTHER },
 213         { "leisure", "sport_centre",NODE_SPORT_CENTER },
 214
 215         /* Tourism */
 216         { "tourism", "information",     NODE_TOURISM_INFO },
 217         { "tourism", "camp_site",       NODE_TOURISM_CAMP_SITE },
 218         { "tourism", "caravan_site",NODE_TOURISM_CARAVAN_SITE },
 219         { "tourism", "picnic_site",     NODE_TOURISM_PICNIC_SITE },
 220         { "tourism", "theme_park",      NODE_TOURISM_THEME_PARK },
 221         { "tourism", "hotel",           NODE_TOURISM_HOTEL },
 222         { "tourism", "motel",           NODE_TOURISM_MOTEL },
 223         { "tourism", "hostel",          NODE_TOURISM_HOSTEL },
 224         { "tourism", "attraction",      NODE_TOURISM_ATTRACTION },
 225         { "tourism", "zoo",                     NODE_TOURISM_ATTRACTION },
 226
 227         { "historic", "ruins",          NODE_TOURISM_ATTRACTION },
 228         { "historic", "monument",       NODE_TOURISM_ATTRACTION },
 229         { "historic", "memorial",       NODE_TOURISM_ATTRACTION },
 230         { "historic", "museum",         NODE_HISTORIC_MUSEUM },
 231         { "historic", "castle",         NODE_HISTORIC_CASTLE },
 232
 233         { "railway", "station",         NODE_RAILWAY_STATION },
 234         { "railway", "halt",            NODE_RAILWAY_HALT },
 235
 236         { "aeroway", "terminal",        NODE_AIRPORT_TERMINAL },
 237
 238         /* Places */
 239         { "place", "city",                      NODE_PLACE_CITY },
 240         { "place", "town",                      NODE_PLACE_TOWN },
 241         { "place", "village",           NODE_PLACE_VILLAGE },
 242         { "place", "hamlet",            NODE_PLACE_HAMLET },
 243         { "place", "locality",          NODE_PLACE_LOCALITY },
 244         { "place", "suburb",            NODE_PLACE_SUBURB },
 245         { "place", "island",            NODE_PLACE_ISLAND },
 246
 247         { "highway", "traffic_signals", NODE_TRAFFIC_SIGNALS },
 248         { "highway", "motorway_junction",       NODE_JUNCTION },
 249         { "highway", "services",        NODE_AMENITY_PARKING },
 250         { "highway", "toll_booth",      NODE_TOLLBOOTH },
 251         { "highway", "gate",            NODE_GATE },
 252
 253         { NULL, NULL, NODE_PLAIN }
 254 };
 255
 256 /* Array to get id number and defaults for ways of different types */
 257 struct _wayinfo {
 258         gchar *k, *v;
 259         guint defspeed;
 260         way_type_t type;
 261         gboolean oneway, link, area, car, foot;
 262 } wayinfo[] = {
 263         { "highway", "motorway",120,WAY_MOTORWAY,               TRUE, FALSE, FALSE, TRUE, FALSE },
 264         { "highway", "motorway_link",120,WAY_MOTORWAY,  TRUE, TRUE, FALSE, TRUE, FALSE },
 265         { "highway", "trunk",100,WAY_TRUNK,                             FALSE, FALSE, FALSE, TRUE, FALSE },
 266         { "highway", "trunk_link",100,WAY_TRUNK,                FALSE, TRUE, FALSE, TRUE, FALSE },
 267         { "highway", "primary",80,WAY_PRIMARY,                  FALSE, FALSE, FALSE, TRUE, TRUE },
 268         { "highway", "primary_link",60,WAY_PRIMARY,             FALSE, TRUE, FALSE, TRUE, TRUE },
 269         { "highway", "secondary",80,WAY_SECONDARY,              FALSE, FALSE, FALSE, TRUE, TRUE },
 270         { "highway", "secondary_link",60,WAY_SECONDARY, FALSE, TRUE, FALSE, TRUE, TRUE },
 271         { "highway", "tertiary",60,WAY_TERTIARY,                FALSE, FALSE, FALSE, TRUE, TRUE },
 272         { "highway", "unclassified",50,WAY_UNCLASSIFIED,        FALSE, FALSE, FALSE, TRUE, TRUE },
 273         { "highway", "byway",40,WAY_UNCLASSIFIED,       FALSE, FALSE, FALSE, TRUE, TRUE },
 274         { "highway", "residential",40,WAY_RESIDENTIAL,  FALSE, FALSE, FALSE, TRUE, TRUE },
 275         { "highway", "service",20,WAY_SERVICE,                  FALSE, FALSE, FALSE, TRUE, TRUE },
 276         { "highway", "track",20,WAY_TRACK,                              FALSE, FALSE, FALSE, TRUE, TRUE },
 277         { "highway", "unsurfaced",60,WAY_TRACK,                 FALSE, FALSE, FALSE, TRUE, TRUE },
 278         { "highway", "minor",60,WAY_TRACK,                      FALSE, FALSE, FALSE, TRUE, TRUE },
 279         { "highway", "pedestrian",20,WAY_FOOTWAY,               FALSE, FALSE, FALSE, FALSE, TRUE },
 280         { "highway", "footway",1,WAY_FOOTWAY,                   FALSE, FALSE, FALSE, FALSE, TRUE },
 281         { "highway", "steps",0,WAY_FOOTWAY,                             FALSE, FALSE, FALSE, FALSE, TRUE},
 282         { "highway", "bridleway",10,WAY_FOOTWAY,                FALSE, FALSE, FALSE, FALSE, TRUE },
 283         { "highway", "cycleway",10,WAY_CYCLEWAY,                FALSE, FALSE, FALSE, FALSE, TRUE },
 284         { "railway", "rail",0,WAY_RAIL,                                 FALSE, FALSE, FALSE, FALSE, FALSE },
 285         { "aeroway", "runway",0,WAY_RUNWAY,                             FALSE, FALSE, FALSE, FALSE, FALSE },
 286         { "aeroway", "taxiway",0,WAY_TAXIWAY,                   FALSE, FALSE, FALSE, FALSE, FALSE },
 287         { "natural", "water",0,WAY_WATER,                               FALSE, FALSE, TRUE, FALSE, FALSE },
 288         { "waterway", "river",0,WAY_WATER,                              FALSE, FALSE, FALSE, FALSE, FALSE },
 289         { "waterway", "canal",0,WAY_WATER,                              FALSE, FALSE, FALSE, FALSE, FALSE },
 290         { "waterway", "stream",0,WAY_WATER,                             FALSE, FALSE, FALSE, FALSE, FALSE },
 291         { "building", "*",0,WAY_UNWAYED,                                FALSE, FALSE, TRUE, FALSE, FALSE },
 292         { NULL, NULL, 0, WAY_UNWAYED, FALSE, FALSE, FALSE, FALSE, FALSE }
 293 };
 294
 295 static sqlite3 *db;
 296 tag_parent_t tag_parent=IS_NONE;
 297
 298 static GHashTable *osm_nodes;
 299 static GHashTable *osm_node_tags;
 300 static GHashTable *osm_way_tags;
 301 static GSList *osm_ways;
 302 static GSList *osm_poi;
 303
 304 static GHashTable *osm_place_country;
 305 static GHashTable *osm_place_region;
 306 static GHashTable *osm_place_city;
 307 static GHashTable *osm_place_suburb;
 308 static GHashTable *osm_place_village;
 309 static GHashTable *osm_node_isin;
 310 static GHashTable *osm_way_isin;
 311
 312 static node *cnode=NULL;
 313 static way *cway=NULL;
 314
 315 struct sql_stmt {
 316         sqlite3_stmt *insert_poi;
 317         sqlite3_stmt *delete_osm_poi;
 318
 319         sqlite3_stmt *insert_node;
 320         sqlite3_stmt *delete_nodes;
 321         sqlite3_stmt *select_node;
 322         sqlite3_stmt *update_node;
 323
 324         sqlite3_stmt *insert_way_data;
 325         sqlite3_stmt *insert_way_ref;
 326         sqlite3_stmt *insert_way_pc;
 327         sqlite3_stmt *insert_way_name;
 328         sqlite3_stmt *insert_way_names_nls;
 329         sqlite3_stmt *insert_way_n2n;
 330         sqlite3_stmt *delete_way;
 331         sqlite3_stmt *delete_way_n2n;
 332         sqlite3_stmt *delete_way_name;
 333         sqlite3_stmt *delete_way_names_nls;
 334         sqlite3_stmt *delete_way_ref;
 335         sqlite3_stmt *delete_way_pc;
 336
 337         sqlite3_stmt *insert_place;
 338         sqlite3_stmt *delete_place;
 339 };
 340 static struct sql_stmt sql;
 341
 342
 343 static struct map_bbox bbox;
 344 static gboolean use_bbox;
 345
 346 static void osm_free_way_data(way *w);
 347 static void print_way(way *w);
 348
 349 static void db_prepare(sqlite3 *db);
 350 static gboolean db_insert_node(node *n);
 351 static guint32 osm_find_way_place(way *w, node_type_t nt);
 352
 353 /****************************************************/
 354 /* Functions */
 355 /****************************************************/
 356
 357 static void
 358 db_finalize(void)
 359 {
 360 sqlite3_finalize(sql.insert_poi);
 361 sqlite3_finalize(sql.delete_osm_poi);
 362
 363 sqlite3_finalize(sql.insert_node);
 364 sqlite3_finalize(sql.select_node);
 365 sqlite3_finalize(sql.delete_nodes);
 366 sqlite3_finalize(sql.update_node);
 367
 368 sqlite3_finalize(sql.insert_place);
 369 sqlite3_finalize(sql.delete_place);
 370
 371 sqlite3_finalize(sql.delete_way);
 372 sqlite3_finalize(sql.insert_way_data);
 373
 374 sqlite3_finalize(sql.delete_way_name);
 375 sqlite3_finalize(sql.insert_way_name);
 376
 377 sqlite3_finalize(sql.delete_way_n2n);
 378 sqlite3_finalize(sql.insert_way_n2n);
 379
 380 sqlite3_finalize(sql.delete_way_pc);
 381 sqlite3_finalize(sql.insert_way_pc);
 382
 383 sqlite3_finalize(sql.delete_way_names_nls);
 384 sqlite3_finalize(sql.insert_way_names_nls);
 385 }
 386
 387 void
 388 db_prepare(sqlite3 *db)
 389 {
 390 /* Way nodes */
 391 sqlite3_prepare_v2(db, "insert or replace into nodes (nid,ilat,ilon,rlat,rlon,l,f) values (?,?,?,?,?,0,?)", -1, &sql.insert_node, NULL);
 392 sqlite3_prepare_v2(db, "select ilat,ilon,l from nodes where nid=?", -1, &sql.select_node, NULL);
 393 sqlite3_prepare_v2(db, "delete from nodes", -1, &sql.delete_nodes, NULL);
 394 sqlite3_prepare_v2(db, "update nodes set l=l+1 where nid=?", -1, &sql.update_node, NULL);
 395
 396 /* Places */
 397 sqlite3_prepare_v2(db, "insert or replace into places (nid,type,name,isin_c,isin_p) values (?, ?, ?, ?, ?)", -1, &sql.insert_place, NULL);
 398 sqlite3_prepare_v2(db, "delete from places", -1, &sql.delete_place, NULL);
 399
 400 /* POI nodes */
 401 if (sqlite3_prepare_v2(db, "insert or replace into poi (osm_id, lat, lon, label, cat_id, public, source, priority, isin_c, isin_p, desc, url, postal_code) "
 402                                            " values (?, ?, ?, ?, ?, 1, 1, ?, ?, ?, ?, ?, ?)", -1, &sql.insert_poi, NULL)!=SQLITE_OK)
 403         g_printf("SQL: %s\n", sqlite3_errmsg(db));
 404
 405 sqlite3_prepare_v2(db, "delete from poi where osm_id>0 and source=1", -1, &sql.delete_osm_poi, NULL);
 406
 407 /* Ways */
 408 sqlite3_prepare_v2(db, "insert or replace into way (wid,nodes,type,flags,speed,isin_c,isin_p,lat,lon) values (?, ?, ?, ?, ?, ?, ?, ?, ?)", -1, &sql.insert_way_data, NULL);
 409 sqlite3_prepare_v2(db, "delete from way", -1, &sql.delete_way, NULL);
 410
 411 /* Way nodes */
 412 sqlite3_prepare_v2(db, "insert into way_n2n (wid,f,t) values (?,?,?)", -1, &sql.insert_way_n2n, NULL);
 413 sqlite3_prepare_v2(db, "delete from way_n2n where wid=?", -1, &sql.delete_way_n2n, NULL);
 414
 415 /* Way names */
 416 sqlite3_prepare_v2(db, "insert or replace into way_names (wid,name,norm) values (?, ?, ?)",  -1, &sql.insert_way_name, NULL);
 417 sqlite3_prepare_v2(db, "delete from way_names", -1, &sql.delete_way_name, NULL);
 418
 419 /* Way postal codes */
 420 sqlite3_prepare_v2(db, "insert or replace into way_pc (wid,pc) values (?, ?)",  -1, &sql.insert_way_pc, NULL);
 421 sqlite3_prepare_v2(db, "delete from way_pc", -1, &sql.delete_way_pc, NULL);
 422
 423 /* Other language names for ways */
 424 sqlite3_prepare_v2(db, "insert into way_names_nls (wid,lang,name, norm) values (?, ?, ?, ?)",  -1, &sql.insert_way_names_nls, NULL);
 425 sqlite3_prepare_v2(db, "delete from way_names_nls where wid=?", -1, &sql.delete_way_names_nls, NULL);
 426
 427 /* Way ref and int_ref */
 428 sqlite3_prepare_v2(db, "insert or replace into way_ref (rid,ref,int_ref) values (?, ?, ?)", -1, &sql.insert_way_ref, NULL);
 429 sqlite3_prepare_v2(db, "delete from way_ref", -1, &sql.delete_way_ref, NULL);
 430 }
 431
 432 /********************************************************************/
 433
 434 static void
 435 print_way(way *w)
 436 {
 437 #ifdef VERBOSE
 438 g_assert(w);
 439 g_printf("Way #%d(N:%d T:%d S:%d IS: %d/%d): %s [%s:%s:%s]\n",
 440                 w->id,
 441                 g_slist_length(w->nodes),
 442                 w->type,
 443                 w->data ? w->data->speed : 0,
 444                 w->data ? w->data->isin_c : -1,
 445                 w->data ? w->data->isin_p : -1,
 446                 w->data ? w->data->name ? w->data->name : "" : "",
 447                 w->flags & W_ONEWAY ? "-" : "=",
 448                 w->flags & W_ROUNDABOUT ? "O" : "-",
 449                 w->flags & W_LINK ? "|" : " ");
 450 #endif
 451 }
 452
 453 static void
 454 print_node(node *n)
 455 {
 456 #ifdef VERBOSE
 457 g_assert(n);
 458 g_printf("Node #%d: T:%d IS: %d/%d [%s]\n",
 459         n->id,
 460         n->type,
 461         n->data ? n->data->isin_c : -1,
 462         n->data ? n->data->isin_p : -1,
 463         n->data ? n->data->name : "");
 464 #endif
 465 }
 466
 467 /********************************************************************/
 468
 469 static gboolean
 470 db_insert_node(node *n)
 471 {
 472 gint32 lat, lon;
 473
 474 g_assert(n);
 475
 476 lat=lat2mp_int(n->lat);
 477 lon=lon2mp_int(n->lon);
 478
 479 sqlite3_bind_int(sql.insert_node, 1, n->id);
 480
 481 /* Projected and integerized lat/lot */
 482 sqlite3_bind_int(sql.insert_node, 2, lat);
 483 sqlite3_bind_int(sql.insert_node, 3, lon);
 484 /* Original */
 485 sqlite3_bind_double(sql.insert_node, 4, n->lat);
 486 sqlite3_bind_double(sql.insert_node, 5, n->lon);
 487 sqlite3_bind_int(sql.insert_node, 6, n->type);
 488
 489 db_exec(db, sql.insert_node);
 490
 491 return TRUE;
 492 }
 493
 494 static gboolean
 495 db_insert_place(node *n)
 496 {
 497 g_assert(n);
 498 if (!n->data)
 499         return FALSE;
 500 if (!n->data->name)
 501         return FALSE;
 502 sqlite3_bind_int(sql.insert_place, 1, n->id);
 503 sqlite3_bind_int(sql.insert_place, 2, n->type);
 504 sqlite3_bind_text(sql.insert_place, 3, n->data->name, -1, SQLITE_TRANSIENT);
 505 sqlite3_bind_int(sql.insert_place, 4, n->data->isin_p);
 506 sqlite3_bind_int(sql.insert_place, 5, n->data->isin_c);
 507
 508 return db_exec(db,sql.insert_place);
 509 }
 510
 511 static gboolean
 512 db_insert_poi(node *n)
 513 {
 514 g_assert(n);
 515 sqlite3_bind_int(sql.insert_poi, 1, n->id);
 516 sqlite3_bind_double(sql.insert_poi, 2, n->lat);
 517 sqlite3_bind_double(sql.insert_poi, 3, n->lon);
 518 if (n->data->name)
 519         sqlite3_bind_text(sql.insert_poi, 4, n->data->name, -1, SQLITE_TRANSIENT);
 520 else
 521         sqlite3_bind_text(sql.insert_poi, 4, "", -1, SQLITE_TRANSIENT);
 522 sqlite3_bind_int(sql.insert_poi, 5, n->type);
 523 sqlite3_bind_int(sql.insert_poi, 6, n->type/100);
 524 sqlite3_bind_int(sql.insert_poi, 7, n->data->isin_c);
 525 sqlite3_bind_int(sql.insert_poi, 8, n->data->isin_p);
 526
 527 if (n->data->desc)
 528         sqlite3_bind_text(sql.insert_poi, 9, n->data->desc, -1, SQLITE_TRANSIENT);
 529 if (n->data->url)
 530         sqlite3_bind_text(sql.insert_poi, 10, n->data->url, -1, SQLITE_TRANSIENT);
 531 if (n->data->postal_code)
 532         sqlite3_bind_text(sql.insert_poi, 11, n->data->postal_code, -1, SQLITE_TRANSIENT);
 533
 534 return db_exec(db,sql.insert_poi);
 535 }
 536
 537 /**
 538  * Update node usage count
 539  */
 540 static gboolean
 541 db_update_node_links(node *n)
 542 {
 543 g_assert(n);
 544 sqlite3_bind_int(sql.update_node, 1, n->id);
 545
 546 return db_exec(db,sql.update_node);
 547 }
 548
 549 /**
 550  * Insert way,node1,node2 triplet
 551  */
 552 static gboolean
 553 db_insert_way_n2n(way *w, node *nf, node *nt)
 554 {
 555 if (!w) {
 556         g_printf("NULL WAY\n");
 557         return FALSE;
 558 }
 559
 560 if (!nf) {
 561         g_printf("NULL NODE 1\n");
 562         return FALSE;
 563 }
 564
 565 if (!nt) {
 566         g_printf("NULL NODE 2\n");
 567         return FALSE;
 568 }
 569
 570 sqlite3_bind_int(sql.insert_way_n2n, 1, w->id);
 571 sqlite3_bind_int(sql.insert_way_n2n, 2, nf->id);
 572 sqlite3_bind_int(sql.insert_way_n2n, 3, nt->id);
 573
 574 #ifdef VERBOSE_N2N
 575 g_printf("%d [%d - %d]\n", w->id, nf->id, nt->id);
 576 #endif
 577
 578 db_exec(db,sql.insert_way_n2n);
 579 db_update_node_links(nf);
 580 db_update_node_links(nt);
 581 return TRUE;
 582 }
 583
 584 /**
 585  * Insert way ref and int_ref
 586  */
 587 static gboolean
 588 db_insert_way_ref(way *w)
 589 {
 590 if (!w->data)
 591         return TRUE;
 592
 593 if (!w->data->ref && !w->data->int_ref)
 594         return TRUE;
 595
 596 way_refs++;
 597
 598 sqlite3_bind_int(sql.insert_way_ref, 1, w->id);
 599 if (w->data->ref)
 600         sqlite3_bind_text(sql.insert_way_ref, 2, w->data->ref, -1, SQLITE_TRANSIENT);
 601 if (w->data->int_ref)
 602         sqlite3_bind_text(sql.insert_way_ref, 3, w->data->int_ref, -1, SQLITE_TRANSIENT);
 603
 604 return db_exec(db,sql.insert_way_ref);
 605 }
 606
 607 /**
 608  * Insert way name
 609  */
 610 static gboolean
 611 db_insert_way_name(way *w)
 612 {
 613 gchar *norm;
 614
 615 if (!w->data)
 616         return TRUE;
 617 if (!w->data->name)
 618         return TRUE;
 619
 620 way_names++;
 621
 622 sqlite3_bind_int(sql.insert_way_name, 1, w->id);
 623 sqlite3_bind_text(sql.insert_way_name, 2, w->data->name, -1, SQLITE_TRANSIENT);
 624
 625 #ifdef TRANSLIT_NAMES
 626 norm=g_convert(w->data->name, -1, "ASCII//TRANSLIT//IGNORE", "utf8", NULL, NULL, NULL);
 627 if (norm && strcmp(w->data->name, norm)!=0) {
 628         sqlite3_bind_text(sql.insert_way_name, 3, norm, -1, SQLITE_TRANSIENT);
 629 }
 630 if (norm)
 631         g_free(norm);
 632 #endif
 633
 634 return db_exec(db,sql.insert_way_name);
 635 }
 636
 637 static gboolean
 638 db_delete_way_names_nls(way *w)
 639 {
 640 sqlite3_bind_int(sql.delete_way_names_nls, 1, w->id);
 641 return db_exec(db,sql.delete_way_names_nls);
 642 }
 643
 644 static gboolean
 645 db_insert_way_pc(way *w)
 646 {
 647 if (!w->data)
 648         return TRUE;
 649 if (!w->data->postal_code)
 650         return TRUE;
 651
 652 sqlite3_bind_int(sql.insert_way_pc, 1, w->id);
 653 sqlite3_bind_text(sql.insert_way_pc, 2, w->data->postal_code, -1, SQLITE_TRANSIENT);
 654
 655 return db_exec(db,sql.insert_way_pc);
 656 }
 657
 658 static gboolean
 659 db_delete_way_pc(way *w)
 660 {
 661 sqlite3_bind_int(sql.delete_way_pc, 1, w->id);
 662 return db_exec(db,sql.delete_way_pc);
 663 }
 664
 665 static void
 666 db_insert_way_names_nls_cb(gpointer key, gpointer value, gpointer user_data)
 667 {
 668 gchar *norm;
 669
 670 way *w=(way *)user_data;
 671
 672 sqlite3_bind_int(sql.insert_way_names_nls, 1, w->id);
 673 sqlite3_bind_text(sql.insert_way_names_nls, 2, (gchar *)key, -1, SQLITE_TRANSIENT);
 674 sqlite3_bind_text(sql.insert_way_names_nls, 3, (gchar *)value, -1, SQLITE_TRANSIENT);
 675 #ifdef TRANSLIT_NAMES
 676 norm=g_convert((gchar *value), -1, "ASCII//TRANSLIT//IGNORE", "utf8", NULL, NULL, NULL);
 677 if (norm && strcmp((gchar *)value, norm)!=0) {
 678         sqlite3_bind_text(sql.insert_way_names_nls, 4, norm, -1, SQLITE_TRANSIENT);
 679 }
 680 if (norm)
 681         g_free(norm);
 682 #endif
 683 db_exec(db,sql.insert_way_names_nls);
 684 }
 685
 686 static void
 687 db_insert_way_names_nls(way *w)
 688 {
 689 if (!w->data)
 690         return;
 691 if (!w->data->names)
 692         return;
 693
 694 g_hash_table_foreach(w->data->names, db_insert_way_names_nls_cb, w);
 695 }
 696
 697 /**
 698  * Insert all data for the given way
 699  * - name
 700  * - ref
 701  * - nodes
 702  *
 703  */
 704 static gboolean
 705 db_insert_way(way *w)
 706 {
 707 GSList *iter;
 708 guint ncnt;
 709 node *wmn;
 710
 711 if (!w)
 712         return FALSE;
 713
 714 /* Skip things we don't use (yet) */
 715 if (w->type==WAY_UNWAYED || w->type>WAY_ROAD_END)
 716         return TRUE;
 717
 718 /* Insert nodes */
 719 for (iter=w->nodes; iter!=NULL; iter=iter->next) {
 720         if (!iter->next)
 721                 break;
 722         db_insert_way_n2n(w, iter->data, iter->next->data);
 723 }
 724
 725 if (w->id==0)
 726         return FALSE;
 727
 728 if (w->data) {
 729         w->data->isin_p=osm_find_way_place(w, NODE_PLACE_CITY);
 730         w->data->isin_c=osm_find_way_place(w, NODE_PLACE_COUNTRY);
 731 }
 732
 733 print_way(w);
 734
 735 sqlite3_bind_int(sql.insert_way_data, 1, w->id);
 736 sqlite3_bind_int(sql.insert_way_data, 2, w->ncnt);
 737 sqlite3_bind_int(sql.insert_way_data, 3, w->type);
 738 sqlite3_bind_int(sql.insert_way_data, 4, w->flags);
 739 if (w->data) {
 740         sqlite3_bind_int(sql.insert_way_data, 5, w->data->speed);
 741         sqlite3_bind_int(sql.insert_way_data, 6, w->data->isin_c);
 742         sqlite3_bind_int(sql.insert_way_data, 7, w->data->isin_p);
 743 }
 744
 745 /* Get middle node, use it as a rough way location */
 746 ncnt=g_slist_length(w->nodes);
 747 if (ncnt>1) {
 748         wmn=g_slist_nth_data(w->nodes, ncnt/2);
 749         if (wmn) {
 750                 sqlite3_bind_double(sql.insert_way_data, 8, wmn->lat);
 751                 sqlite3_bind_double(sql.insert_way_data, 9, wmn->lon);
 752         } else {
 753                 g_printerr("Failed to get way middlepoint node for location information!\n");
 754         }
 755 }
 756
 757 db_exec(db,sql.insert_way_data);
 758
 759 db_insert_way_ref(w);
 760 db_insert_way_name(w);
 761 db_insert_way_names_nls(w);
 762 db_insert_way_pc(w);
 763
 764 osm_free_way_data(w);
 765 return TRUE;
 766 }
 767
 768 /********************************************************************/
 769
 770 static gchar *
 771 get_attr_key_value(const gchar **p, gchar *key)
 772 {
 773 gchar **d;
 774
 775 d=p;
 776 while (*d!=NULL) {
 777         if (strncmp(*d, key, strlen(key))==0) {
 778                 d++;
 779                 return *d;
 780         }
 781         d++;
 782         d++;
 783 }
 784 return NULL;
 785 }
 786
 787 static tag_state_t
 788 check_tag(const gchar *tag)
 789 {
 790 if (strcmp(tag,"node")==0) return IN_NODE_TAG;
 791 else if (strcmp(tag,"nd")==0) return IN_WNODE_TAG;
 792 else if (strcmp(tag,"way")==0) return IN_WAY_TAG;
 793 else if (strcmp(tag,"tag")==0) return IN_KEY_TAG;
 794 else if (strcmp(tag,"osm")==0) return IN_OSM_TAG;
 795 else if (strcmp(tag,"bound")==0) return IN_BOUND_TAG;
 796 else if (strcmp(tag,"relation")==0) return IN_RELATION_TAG;
 797 else if (strcmp(tag,"member")==0) return IN_MEMBER_TAG;
 798 else return ERROR;
 799 }
 800
 801 static void
 802 find_nls_names(gpointer key, gpointer value, gpointer user_data)
 803 {
 804 gchar *k, *v;
 805 gchar *tmp;
 806 GHashTable *nls;
 807
 808 k=(gchar *)key;
 809 v=(gchar *)value;
 810 nls=(GHashTable *)user_data;
 811
 812 /* Check if it is a name key, return if not. */
 813 if (g_str_has_prefix(k, "name:")==FALSE)
 814         return;
 815
 816 tmp=g_strrstr(k, ":");
 817 if (!tmp)
 818         return;
 819 tmp++; /* skip : */
 820 if (*tmp==0)
 821         return;
 822 g_hash_table_insert(nls, g_strdup(tmp), g_strdup(v));
 823 #ifdef VERBOSE
 824 g_printf("NLS(%s): [%s]\n", tmp, v);
 825 #endif
 826 }
 827
 828 /********************************************************************/
 829
 830 static void
 831 node_print (node *n)
 832 {
 833 g_assert(n);
 834 if (n->data) {
 835         g_printf("N: %d [%f:%f][%s](%d)\n",
 836                 n->id, n->lat, n->lon,
 837                 n->data->name ? n->data->name : "-",
 838                 n->type);
 839 } else {
 840         g_printf("N: %d [%f:%f]\n",
 841                 n->id, n->lat, n->lon);
 842 }
 843 }
 844
 845 #ifdef DEBUG
 846 static void
 847 dump_array(const gchar **p)
 848 {
 849 char **d;
 850
 851 d=p;
 852 while (*d!=NULL) {
 853         g_printf("[%s]", *d);
 854         d++;
 855 }
 856 g_print("\n");
 857 }
 858 #endif
 859
 860 static inline gboolean
 861 osm_node_check_box(gdouble nlat, gdouble nlon)
 862 {
 863 if (use_bbox==FALSE)
 864         return TRUE;
 865 return (nlat > bbox.lat_min && nlat < bbox.lat_max && nlon > bbox.lon_min && nlon < bbox.lon_max) ? TRUE : FALSE;
 866 }
 867
 868 static void
 869 osm_new_node_data(node *n)
 870 {
 871 if (n==NULL)
 872         return;
 873 if (n->data!=NULL)
 874         return;
 875 n->data=g_slice_new(node_data);
 876 n->data->name=NULL;
 877 n->data->url=NULL;
 878 n->data->desc=NULL;
 879 n->data->postal_code=NULL;
 880 n->type=NODE_PLAIN;
 881 noded_cnt++;
 882 }
 883
 884 static void
 885 osm_free_node_data(node *n)
 886 {
 887 g_assert(n);
 888 g_assert(n->data);
 889 if (n->data->name)
 890         g_free(n->data->name);
 891 if (n->data->url)
 892         g_free(n->data->url);
 893 if (n->data->desc)
 894         g_free(n->data->desc);
 895 if (n->data->postal_code)
 896         g_free(n->data->postal_code);
 897 g_slice_free(node_data, n->data);
 898 n->data=NULL;
 899 noded_cnt--;
 900 }
 901
 902 static node *
 903 osm_new_node(gint id, gdouble lat, gdouble lon)
 904 {
 905 node *n=NULL;
 906
 907 n=g_slice_new(node);
 908 g_assert(n);
 909 n->id=id;
 910 n->lat=lat;
 911 n->lon=lon;
 912 n->data=(node_data *)NULL;
 913 return n;
 914 }
 915
 916 static void
 917 osm_free_node(node *n)
 918 {
 919 g_assert(n);
 920 g_slice_free(node, n);
 921 }
 922
 923 static node *
 924 osm_find_node(guint32 nid)
 925 {
 926 node *n;
 927
 928 g_assert(osm_nodes);
 929 return g_hash_table_lookup(osm_nodes, GINT_TO_POINTER(nid));
 930 }
 931
 932 static void
 933 osm_new_way_data(way *w)
 934 {
 935 if (w==NULL)
 936         return;
 937 if (w->data!=NULL)
 938         return;
 939
 940 w->data=g_slice_new(way_data);
 941 w->data->name=NULL;
 942 w->data->names=NULL;
 943 w->data->ref=NULL;
 944 w->data->int_ref=NULL;
 945 w->data->postal_code=NULL;
 946 w->data->layer=0;
 947 w->data->speed=0;
 948 }
 949
 950 static void
 951 osm_free_way_data(way *w)
 952 {
 953 g_assert(w);
 954 if (!w->data)
 955         return;
 956 if (w->data->name)
 957         g_free(w->data->name);
 958 if (w->data->ref)
 959         g_free(w->data->ref);
 960 if (w->data->int_ref)
 961         g_free(w->data->int_ref);
 962 g_slice_free(way_data, w->data);
 963 w->data=NULL;
 964 }
 965
 966 static way *
 967 osm_new_way(gint id)
 968 {
 969 way *w;
 970
 971 w=g_slice_new(way);
 972 g_assert(w);
 973 w->id=id;
 974 w->nodes=NULL;
 975 w->type=WAY_UNWAYED;
 976 w->data=NULL;
 977 w->ncnt=0;
 978 w->flags=0;
 979
 980 /* Add to list of ways */
 981 return w;
 982 }
 983
 984 static void
 985 osm_free_way(way *w)
 986 {
 987 if (w->nodes)
 988         g_slist_free(w->nodes);
 989 g_slice_free(way, w);
 990 }
 991
 992 static void
 993 osm_way_add_to_list(way *w)
 994 {
 995 g_assert(w);
 996 osm_ways=g_slist_prepend(osm_ways, w);
 997 }
 998
 999 static void
1000 osm_way_new_node(way *w, gint nid)
1001 {
1002 node *n;
1003
1004 g_assert(w);
1005 n=osm_find_node(nid);
1006 w->nodes=g_slist_prepend(w->nodes, n);
1007 w->ncnt++;
1008 }
1009
1010 /**
1011  * Search the place hash table for the location of the node.
1012  *
1013  */
1014 static guint32
1015 osm_find_node_place(node *n)
1016 {
1017 node *t;
1018 gchar **isin;
1019 gchar **place;
1020
1021 if (!n->data)
1022         return 0;
1023
1024 isin=g_hash_table_lookup(osm_node_isin, GINT_TO_POINTER(n->id));
1025
1026 if (!isin)
1027         return 0;
1028
1029 place=isin;
1030 while (*place!=NULL) {
1031         gchar *ps;
1032         ps=g_strstrip(*place);
1033 #ifdef VERBOSE
1034         g_printf("Checking (%d) [%s] in [%s]\n",n->type, n->data->name, ps);
1035 #endif
1036         switch (n->type) {
1037         case NODE_PLACE_CITY:
1038         case NODE_PLACE_TOWN:
1039         case NODE_PLACE_VILLAGE:
1040         case NODE_PLACE_HAMLET:
1041                 t=g_hash_table_lookup(osm_place_region, ps);
1042                 if (t)
1043                         return t->id;
1044                 t=g_hash_table_lookup(osm_place_country, ps);
1045                 if (t)
1046                         return t->id;
1047         break;
1048         case NODE_PLACE_SUBURB:
1049         case NODE_PLACE_LOCALITY:
1050                 t=g_hash_table_lookup(osm_place_city, ps);
1051                 if (t)
1052                         return t->id;
1053         break;
1054         case NODE_PLACE_ISLAND:
1055                 return 0;
1056         break;
1057         default:
1058                 t=g_hash_table_lookup(osm_place_city, ps);
1059                 if (t)
1060                         return t->id;
1061         break;
1062         }
1063         place++;
1064 }
1065
1066 return 0;
1067 }
1068
1069 static guint32
1070 osm_find_way_place(way *w, node_type_t nt)
1071 {
1072 gchar **isin;
1073 gchar **place;
1074
1075 isin=g_hash_table_lookup(osm_way_isin, GINT_TO_POINTER(w->id));
1076 if (!isin)
1077         return 0;
1078
1079 place=isin;
1080 while (*place!=NULL) {
1081         node *t;
1082         gchar *ps;
1083
1084         ps=g_strstrip(*place);
1085
1086 #ifdef VERBOSE
1087         g_printf("Checking (%d) in [%s]\n",w->id, ps);
1088 #endif
1089 switch (nt) {
1090         case NODE_PLACE_CITY:
1091         case NODE_PLACE_TOWN:
1092         case NODE_PLACE_VILLAGE:
1093         case NODE_PLACE_HAMLET:
1094         case NODE_PLACE_LOCALITY:
1095                 t=g_hash_table_lookup(osm_place_city, ps);
1096                 if (t)
1097                         return t->id;
1098         break;
1099         case NODE_PLACE_COUNTRY:
1100                 t=g_hash_table_lookup(osm_place_country, ps);
1101                 if (t)
1102                         return t->id;
1103         break;
1104         default:
1105                 g_assert_not_reached();
1106         break;
1107         }
1108         place++;
1109 }
1110
1111 return 0;
1112 }
1113
1114 /***********************************************************************/
1115
1116 static void
1117 osm_node_save_node(gint key, gpointer value, gpointer user_data)
1118 {
1119 node *n=(node *)value;
1120
1121 dbnode_cnt++;
1122 db_insert_node(n);
1123 if (dbnode_cnt % 26214==0)
1124         g_printf("\rNodes: %f%%\n",((float)dbnode_cnt/(float)node_cnt)*100);
1125 }
1126
1127 /**
1128  * Check node type and insert as POI or Place
1129  * Discard extra data after insert.
1130  */
1131 static gboolean
1132 osm_node_save_poi(node *n, gpointer user_data)
1133 {
1134 if (!n) {
1135         g_printerr("ERROR: null poi\n");
1136         return FALSE;
1137 }
1138
1139 if (!n->data) {
1140         g_printerr("POI node with no data ?\n");
1141         return FALSE;
1142 }
1143
1144 n->data->isin_p=osm_find_node_place(n);
1145 n->data->isin_c=0;
1146
1147 if (n->type>NODE_POI_START && n->type<NODE_POI_END) {
1148         print_node(n);
1149         db_insert_poi(n);
1150         osm_free_node_data(n);
1151 } else if (n->type>NODE_PLACE_START && n->type<NODE_PLACE_END) {
1152         print_node(n);
1153         db_insert_place(n);
1154 } else {
1155         osm_free_node_data(n);
1156         return FALSE;
1157 }
1158
1159 return TRUE;
1160 }
1161
1162 static gboolean
1163 osm_planet_poi_clear_nodes(void)
1164 {
1165 g_print("Removing old OSM POIs...\n");
1166 db_transaction_begin(db);
1167 sqlite3_step(sql.delete_osm_poi);
1168 sqlite3_step(sql.delete_place);
1169 return db_transaction_commit(db);
1170 }
1171
1172 static gboolean
1173 osm_planet_poi_save_nodes(void)
1174 {
1175 g_print("Storing new POIs...\n");
1176 db_transaction_begin(db);
1177 g_slist_foreach(osm_poi, osm_node_save_poi, NULL);
1178 g_slist_free(osm_poi);
1179 return db_transaction_commit(db);
1180 }
1181
1182 /*********************************************************************/
1183
1184 static void
1185 osm_planet_clear_nodes(void)
1186 {
1187 g_print("Clearing old nodes\n");
1188 sqlite3_step(sql.delete_nodes);
1189 }
1190
1191 static gboolean
1192 osm_planet_save_nodes(void)
1193 {
1194 g_print("Storing nodes...\n");
1195 db_transaction_begin(db);
1196 g_hash_table_foreach(osm_nodes, osm_node_save_node, NULL);
1197 return db_transaction_commit(db);
1198 }
1199
1200 /*********************************************************************/
1201
1202 static void
1203 osm_way_save(way *value, gpointer user_data)
1204 {
1205 dbway_cnt++;
1206 db_insert_way(value);
1207 if (dbway_cnt % 16384==0 && dbway_cnt>0) {
1208                 g_printf("\rWays: %f%%\n",(((float)dbway_cnt/(float)way_cnt)*100));
1209                 print_way(value);
1210 }
1211 }
1212
1213 static void
1214 osm_planet_clear_ways(void)
1215 {
1216 g_print("Clearing old data\n");
1217 sqlite3_step(sql.delete_way);
1218 sqlite3_step(sql.delete_way_name);
1219 sqlite3_step(sql.delete_way_ref);
1220 sqlite3_step(sql.delete_way_n2n);
1221 }
1222
1223 static gboolean
1224 osm_planet_save_ways(void)
1225 {
1226 g_print("Inserting new ways\n");
1227 db_transaction_begin(db);
1228 g_slist_foreach(osm_ways, osm_way_save, NULL);
1229 return db_transaction_commit(db);
1230 }
1231
1232 /*********************************************************************/
1233
1234 static void
1235 osm_planet_save_all_nodes(void)
1236 {
1237 g_printf("Saving planet nodes to database:\n");
1238
1239 osm_planet_poi_clear_nodes();
1240 osm_planet_poi_save_nodes();
1241
1242 if (!is_update) {
1243         osm_planet_clear_nodes();
1244         osm_planet_clear_ways();
1245 }
1246 osm_planet_save_nodes();
1247 }
1248
1249 static void
1250 osm_planet_save_all_ways(void)
1251 {
1252 g_printf("Saving planet way to database:\n");
1253
1254 osm_planet_save_ways();
1255 }
1256
1257 /***********************************************************************/
1258
1259 static void
1260 _osm_tag_start(void *userData, const char *name, const char **atts)
1261 {
1262 tag_state_t t;
1263 gchar *k, *v;
1264 guint32 id, ndref;
1265 gdouble nlat, nlon;
1266
1267 t=check_tag(name);
1268 switch (t) {
1269         case IN_OSM_TAG:
1270                 g_printf("Starting...\n");
1271         break;
1272         case IN_NODE_TAG:
1273                 tag_parent=IS_NODE;
1274                 node_cnt++;
1275
1276                 id=atoi(get_attr_key_value(atts, "id"));
1277                 nlat=atof(get_attr_key_value(atts, "lat"));
1278                 nlon=atof(get_attr_key_value(atts, "lon"));
1279
1280                 cnode=osm_new_node(id, nlat, nlon);
1281                 osm_node_tags=g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free);
1282         break;
1283         case IN_WAY_TAG:
1284                 tag_parent=IS_WAY;
1285                 way_cnt++;
1286                 id=atoi(get_attr_key_value(atts, "id"));
1287                 cway=osm_new_way(id);
1288                 osm_way_tags=g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free);
1289         break;
1290         case IN_WNODE_TAG:
1291                 ndref=atoi(get_attr_key_value(atts, "ref"));
1292                 if (use_bbox==TRUE) {
1293                         if (osm_find_node(ndref)==NULL) {
1294                                 cway->id=0;
1295                                 return;
1296                         }
1297                 }
1298                 osm_way_new_node(cway, ndref);
1299         break;
1300         case IN_KEY_TAG:
1301                 k=get_attr_key_value(atts, "k");
1302                 if (strcmp(k,"created_by")==0)
1303                         return;
1304                 if (strcmp(k,"source")==0)
1305                         return;
1306
1307                 v=get_attr_key_value(atts, "v");
1308 #ifdef VERBOSE_KEYS
1309                 g_printf("TAG: K=[%s] V=[%s]\n", k, v);
1310 #endif
1311
1312                 switch (tag_parent) {
1313                 case IS_NONE:
1314                         g_printf("Tag key/value pair but unknown owner\n");
1315                 break;
1316                 case IS_NODE:
1317                 {
1318                         if (!osm_node_tags)
1319                                 return;
1320
1321                         /* Insert key/value pairs into hash table */
1322                         if (cnode==NULL) {
1323                                 g_printerr("In node tags but node is NULL!\n");
1324                                 return;
1325                         }
1326                         g_hash_table_insert(osm_node_tags, g_strdup(k), g_strdup(v));
1327                 }
1328                 break;
1329                 case IS_WAY:
1330                 {
1331                         if (cway==NULL) {
1332                                 g_printerr("In way tags but way is NULL!\n");
1333                                 return;
1334                         }
1335                         g_hash_table_insert(osm_way_tags, g_strdup(k), g_strdup(v));
1336                         osm_new_way_data(cway);
1337                 }
1338                 break;
1339                 case IS_RELATION:
1340
1341                 break;
1342                 }
1343         break;
1344         case IN_BOUND_TAG:
1345                 /* Ignore for now */
1346                 g_printf("Ignoring bound tag\n");
1347         break;
1348         case IN_RELATION_TAG:
1349                 tag_parent=IS_RELATION;
1350
1351         break;
1352         case IN_MEMBER_TAG:
1353
1354         break;
1355         default:
1356                 tag_parent=IS_NONE;
1357                 g_printf("Unknown tag: %s\n", name);
1358         break;
1359 }
1360 }
1361
1362 static void
1363 _osm_tag_end(void *userData, const char *name)
1364 {
1365 tag_state_t t;
1366 gchar *v;
1367 guint i;
1368 t=check_tag(name);
1369 switch (t) {
1370         case IN_NODE_TAG:
1371
1372                 if (node_cnt % 262140==0) {
1373                         g_printf("Nodes: %d of %d, POIs: %d, Outside box: %d\n", node_cnt-node_skip_cnt, node_cnt, noded_cnt, node_skip_cnt);
1374                 }
1375
1376                 if (!osm_node_tags)
1377                         return;
1378
1379                 osm_new_node_data(cnode);
1380
1381                 for (i=0; nodeinfo[i].k; i++) {
1382                         v=g_hash_table_lookup(osm_node_tags, nodeinfo[i].k);
1383                         if (!v)
1384                                 continue;
1385                         if (strcasecmp (v, nodeinfo[i].v)==0) {
1386                                 cnode->type=nodeinfo[i].type;
1387                                 break;
1388                         }
1389                 }
1390
1391                 /* Check if node is inside bounding box, if not skip it.
1392                  * But keep it if it's something we might need for other nodes:
1393                  * - Places (for is_in)
1394                  * - ...
1395                  */
1396                 if ((osm_node_check_box(cnode->lat, cnode->lon)==FALSE) &&
1397                                 (cnode->type<NODE_PLACE_START)) {
1398                         osm_free_node_data(cnode);
1399                         osm_free_node(cnode);
1400                         g_hash_table_destroy(osm_node_tags);
1401                         node_skip_cnt++;
1402                         return;
1403                 }
1404
1405                 g_hash_table_insert(osm_nodes, GINT_TO_POINTER(cnode->id), cnode);
1406
1407                 if (cnode->type!=NODE_PLAIN) {
1408                         cnode->data->name=NULL;
1409                         v=g_hash_table_lookup(osm_node_tags, "name");
1410                         if (v)
1411                                 cnode->data->name=g_strstrip(g_utf8_normalize(v, -1, G_NORMALIZE_ALL_COMPOSE));
1412                         v=g_hash_table_lookup(osm_node_tags, "note");
1413                         if (v)
1414                                 cnode->data->desc=g_strstrip(g_strdup(v));
1415                         v=g_hash_table_lookup(osm_node_tags, "postal_code");
1416                         if (v)
1417                                 cnode->data->postal_code=g_strstrip(g_strdup(v));
1418
1419                         /* Links */
1420                         v=g_hash_table_lookup(osm_node_tags, "url");
1421                         if (v) {
1422                                 cnode->data->url=g_strstrip(g_strdup(v));
1423                         } else {
1424                                 v=g_hash_table_lookup(osm_node_tags, "wikipedia");
1425                                 if (v && strncmp(v,"http:", 5)==0)
1426                                         cnode->data->url=g_strstrip(g_strdup(v));
1427                         }
1428                 }
1429
1430                 cnode->data->isin_c=0;
1431                 cnode->data->isin_p=0;
1432                 v=g_hash_table_lookup(osm_node_tags, "is_in");
1433                 if (v) {
1434                         gchar **isin;
1435                         isin=g_strsplit(v, ",", 10);
1436                         g_hash_table_insert(osm_node_isin, GINT_TO_POINTER(cnode->id), isin);
1437                 }
1438
1439                 if (cnode->type==NODE_PLAIN) {
1440                         osm_free_node_data(cnode);
1441                 } else {
1442                         osm_poi=g_slist_prepend(osm_poi, cnode);
1443                         if (cnode->data->name) {
1444                                 switch (cnode->type) {
1445                                 case NODE_PLACE_COUNTRY:
1446                                         g_hash_table_insert(osm_place_country, cnode->data->name, cnode);
1447                                 break;
1448                                 case NODE_PLACE_CITY:
1449                                 case NODE_PLACE_TOWN:
1450                                         g_hash_table_insert(osm_place_city, cnode->data->name, cnode);
1451                                 break;
1452                                 case NODE_PLACE_SUBURB:
1453                                         g_hash_table_insert(osm_place_suburb, cnode->data->name, cnode);
1454                                 break;
1455                                 case NODE_PLACE_VILLAGE:
1456                                 case NODE_PLACE_HAMLET:
1457                                 case NODE_PLACE_LOCALITY:
1458                                         g_hash_table_insert(osm_place_village, cnode->data->name, cnode);
1459                                 break;
1460                                 case NODE_PLACE_ISLAND:
1461                                         /* Ignore for now */
1462                                 break;
1463                                 default:;
1464                                 }
1465                         }
1466                 }
1467                 g_hash_table_destroy(osm_node_tags);
1468                 cnode=NULL;
1469         break;
1470         case IN_WAY_TAG:
1471                 if (way_cnt % 1024==0) {
1472                         g_printf("\rWays: %d\n", way_cnt);
1473                 }
1474
1475                 cway->nodes=g_slist_reverse(cway->nodes);
1476
1477                 for (i=0; wayinfo[i].k; i++) {
1478                         v=g_hash_table_lookup(osm_way_tags, wayinfo[i].k);
1479                         if (!v)
1480                                 continue;
1481                         if (strcasecmp (v, wayinfo[i].v)==0) {
1482                                 if (wayinfo[i].link==TRUE)
1483                                         cway->flags|=W_LINK;
1484                                 if (wayinfo[i].area==TRUE)
1485                                         cway->flags|=W_AREA;
1486                                 if (wayinfo[i].oneway==TRUE)
1487                                         cway->flags|=W_ONEWAY;
1488                                 cway->type=wayinfo[i].type;
1489                                 if (cway->data->speed==0)
1490                                         cway->data->speed=wayinfo[i].defspeed;
1491                                 break;
1492                         }
1493                 }
1494
1495                 v=g_hash_table_lookup(osm_way_tags, "name");
1496                 if (v) {
1497                         cway->data->name=g_utf8_normalize(v, -1, G_NORMALIZE_ALL_COMPOSE);
1498                         /* Try to find other language names */
1499                         cway->data->names=g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free);
1500                         g_hash_table_foreach(osm_way_tags, find_nls_names, cway->data->names);
1501                         if (g_hash_table_size(cway->data->names)==0) {
1502                                 g_hash_table_destroy(cway->data->names);
1503                                 cway->data->names=NULL;
1504                         }
1505                 }
1506
1507                 v=g_hash_table_lookup(osm_way_tags, "ref");
1508                 if (v)
1509                         cway->data->ref=g_strdup(v);
1510                 v=g_hash_table_lookup(osm_way_tags, "int_ref");
1511                 if (v)
1512                         cway->data->int_ref=g_strdup(v);
1513
1514                 v=g_hash_table_lookup(osm_way_tags, "postal_code");
1515                 if (v)
1516                         cway->data->postal_code=g_strdup(v);
1517
1518                 /* XXX: somehow handle the silly -1 'reversed' oneway */
1519                 v=g_hash_table_lookup(osm_way_tags, "oneway");
1520                 if (v)
1521                         cway->flags|=W_ONEWAY;
1522
1523                 v=g_hash_table_lookup(osm_way_tags, "noexit");
1524                 if (v)
1525                         cway->flags|=W_NOEXIT;
1526
1527                 v=g_hash_table_lookup(osm_way_tags, "speedlimit");
1528                 if (v)
1529                         cway->data->speed=atoi(v);
1530                 v=g_hash_table_lookup(osm_way_tags, "maxspeed");
1531                 if (v)
1532                         cway->data->speed=atoi(v);
1533
1534 #if 0
1535                 v=g_hash_table_lookup(osm_way_tags, "layer");
1536                 if (v)
1537                         cway->data->layer=atoi(v);
1538 #endif
1539
1540                 v=g_hash_table_lookup(osm_way_tags, "junction");
1541                 if (v && strcasecmp(v,"roundabout")==0) {
1542                         cway->flags|=W_ROUNDABOUT;
1543                         cway->flags|=W_ONEWAY;
1544                 } else if (v && strcasecmp(v,"mini_roundabout")==0) {
1545                         cway->flags|=W_ROUNDABOUT;
1546                         cway->flags|=W_ONEWAY;
1547                 }
1548
1549                 /* XXX: Should check keys */
1550                 v=g_hash_table_lookup(osm_way_tags, "access");
1551                 if (v && (strcasecmp(v, "private")==0)) {
1552                         cway->flags|=W_NOACCESS;
1553                 }
1554
1555                 print_way(cway);
1556
1557                 v=g_hash_table_lookup(osm_way_tags, "is_in");
1558                 if (v) {
1559                         gchar **isin;
1560                         isin=g_strsplit(v, ",", 10);
1561                         g_hash_table_insert(osm_way_isin, GINT_TO_POINTER(cway->id), isin);
1562                 }
1563
1564                 if (cway->data && cway->data->name==NULL && cway->data->ref==NULL &&
1565                         cway->data->int_ref==NULL && cway->data->layer==0 && cway->data->speed==0)
1566                         osm_free_way_data(cway);
1567
1568                 if (cway->id!=0)
1569                         osm_way_add_to_list(cway);
1570                 else
1571                         osm_free_way(cway);
1572
1573                 cway=NULL;
1574                 g_hash_table_destroy(osm_way_tags);
1575         break;
1576         case IN_BOUND_TAG:
1577                 /* */
1578         break;
1579         case IN_OSM_TAG:
1580                 g_printf("\nPlanet loaded.\n");
1581         break;
1582         default:;
1583 }
1584 }
1585
1586 /************************************************************************/
1587
1588 static void
1589 storage_init(void)
1590 {
1591 osm_nodes=g_hash_table_new(g_direct_hash, g_direct_equal);
1592
1593 osm_place_country=g_hash_table_new(g_str_hash, g_str_equal);
1594 osm_place_city=g_hash_table_new(g_str_hash, g_str_equal);
1595 osm_place_suburb=g_hash_table_new(g_str_hash, g_str_equal);
1596 osm_place_village=g_hash_table_new(g_str_hash, g_str_equal);
1597 osm_place_region=g_hash_table_new(g_str_hash, g_str_equal);
1598 osm_node_isin=g_hash_table_new(g_direct_hash, g_direct_equal);
1599 osm_way_isin=g_hash_table_new(g_direct_hash, g_direct_equal);
1600 }
1601
1602 static void
1603 storage_free(void)
1604 {
1605 g_hash_table_destroy(osm_nodes);
1606
1607 g_hash_table_destroy(osm_place_country);
1608 g_hash_table_destroy(osm_place_city);
1609 g_hash_table_destroy(osm_place_suburb);
1610 g_hash_table_destroy(osm_place_village);
1611 g_hash_table_destroy(osm_place_region);
1612 g_hash_table_destroy(osm_node_isin);
1613 }
1614
1615 /************************************************************************/
1616
1617 static gint
1618 print_fail(const gchar *msg, gint ret)
1619 {
1620 g_printerr("ERROR: %s\n", msg);
1621 return ret;
1622 }
1623
1624 /************************************************************************/
1625
1626 static void
1627 print_memory_usage(void)
1628 {
1629 g_print("Memory usage per item:\n");
1630 g_printf("Node  size: %d\n", (gint)sizeof(node));
1631 g_printf("NodeD size: %d\n", (gint)sizeof(node_data));
1632 g_printf("Way   size: %d\n", (gint)sizeof(way));
1633 g_printf("WayD  size: %d\n", (gint)sizeof(way_data));
1634 }
1635
1636 /************************************************************************
1637  * Public inteface
1638  ************************************************************************/
1639
1640 void
1641 osm_planet_parser_init(void)
1642 {
1643 xp=XML_ParserCreate(NULL);
1644 XML_SetElementHandler(xp, _osm_tag_start, _osm_tag_end);
1645 storage_init();
1646 }
1647
1648 void
1649 osm_planet_parser_deinit(void)
1650 {
1651 XML_ParserFree(xp);
1652 storage_free();
1653 }
1654
1655 gboolean
1656 osm_planet_parse_buffer(gchar *buffer, size_t r)
1657 {
1658 if (XML_Parse(xp, buffer, r, r>0 ? 0:1) == XML_STATUS_ERROR) {
1659         g_printerr("Parse error at line %d:\n%s\n",
1660         XML_GetCurrentLineNumber(xp),
1661         XML_ErrorString(XML_GetErrorCode(xp)));
1662         return FALSE;
1663 }
1664 return TRUE;
1665 }
1666
1667 gboolean
1668 osm_planet_parse_file(gchar *pfile)
1669 {
1670 FILE *f;
1671 BZFILE *b;
1672 int bzerror;
1673 int r;
1674 gchar buffer[FILE_BUFFER];
1675 gboolean res=TRUE;
1676
1677 f=fopen(pfile, "r");
1678 if (!f) {
1679         perror("fopen failed\n");
1680         return FALSE;
1681 }
1682
1683 b=BZ2_bzReadOpen(&bzerror, f, 0, 0, NULL, 0);
1684 if (bzerror != BZ_OK) {
1685         g_printf("BZ2_bzReadOpen failed\n");
1686         BZ2_bzReadClose(&bzerror, b);
1687         return FALSE;
1688 }
1689
1690 do {
1691         r=BZ2_bzRead(&bzerror, b, buffer, FILE_BUFFER);
1692         if ((bzerror!=BZ_STREAM_END) && (bzerror!=BZ_OK)) {
1693                 res=FALSE;
1694                 break;
1695         }
1696         if (!osm_planet_parse_buffer(buffer, r)) {
1697                 res=FALSE;
1698                 break;
1699         }
1700 } while (bzerror==BZ_OK);
1701
1702 BZ2_bzReadClose(&bzerror, b);
1703 fclose(f);
1704 return res;
1705 }
1706
1707 /**
1708  * Set up bounding box for import.
1709  *
1710  */
1711 void
1712 osm_import_set_bbox(gboolean use_bb, gdouble latmin, gdouble lonmin, gdouble latmax, gdouble lonmax)
1713 {
1714 use_bbox=use_bb;
1715 bbox.lat_min=latmin;
1716 bbox.lon_min=lonmin;
1717 bbox.lat_max=latmax;
1718 bbox.lon_max=lonmax;
1719 g_printf("Skipping data outside of box: %f,%f - %f,%f\n",
1720         bbox.lat_min, bbox.lon_min,     bbox.lat_max, bbox.lon_max);
1721 }
1722
1723 /**
1724  * Simple helper to do all preparations and importing from planet -> database
1725  *
1726  */
1727 gboolean
1728 osm_import(const gchar *planet, const gchar *database)
1729 {
1730 if (db_connect(&db, database)!=TRUE) {
1731         g_printerr("Database open failed: %s", database);
1732         return FALSE;
1733 }
1734
1735 db_create_tables(db);
1736 db_create_indexes(db);
1737 db_prepare(db);
1738
1739 osm_planet_parser_init();
1740
1741 if (osm_planet_parse_file(planet)==FALSE) {
1742         g_printerr("Failed to parse file: %s\n", planet);
1743         return FALSE;
1744 }
1745
1746 g_printf("Total nodes %d, POIs: %d and Ways %d.\n",     node_cnt, noded_cnt, way_cnt);
1747 g_printf("Cities/Towns: %d\n", g_hash_table_size(osm_place_city));
1748 g_printf("Villages/Hamlets: %d\n", g_hash_table_size(osm_place_village));
1749 g_printf("Suburbs: %d\n", g_hash_table_size(osm_place_suburb));
1750 g_printf("Nodes: %d\n", g_hash_table_size(osm_nodes));
1751
1752 osm_planet_save_all_nodes();
1753 osm_planet_save_all_ways();
1754 osm_planet_parser_deinit();
1755 db_close(&db);
1756 return TRUE;
1757 }
1758