err.no Git - mapper/blob - src/osm-db-import.c

   1 /*
   2  * This file is part of mapper
   3  *
   4  * Copyright (C) 2007 Kaj-Michael Lang
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along
  17  * with this program; if not, write to the Free Software Foundation, Inc.,
  18  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  19  */
  20
  21 /*
  22  * Routines to read OSM planet XML file and store it in a sqlite3 database.
  23  * Reads in all nodes (if used, skips nodes outside bounding box)
  24  * Special POI nodes are stored in POI table.
  25  * Place POI nodes are stored in place table.
  26  *
  27  * Ways are read in and their data (name, type, etc) are stored
  28  * in way, way_name and way_ref tables.
  29  *
  30  * Nodes used by they ways are stored in way_n2n table.
  31  *
  32  */
  33
  34 #include <stdio.h>
  35 #include <unistd.h>
  36 #include <string.h>
  37 #include <strings.h>
  38 #include <sys/types.h>
  39 #include <sys/stat.h>
  40 #include <fcntl.h>
  41 #include <math.h>
  42 #include <glib.h>
  43 #include <glib/gstdio.h>
  44 #include <sqlite3.h>
  45 #include <expat.h>
  46 #include <bzlib.h>
  47
  48 #include "osm.h"
  49 #include "latlon.h"
  50 #include "db.h"
  51 #include "osm-db-import.h"
  52
  53 #if 0
  54 #define VERBOSE
  55 #endif
  56 /* #define VERBOSE_KEYS */
  57
  58
  59 /* Use g_convert to transliterate names.. my iconv seems to be fucked so this didn't work... */
  60 /* #define TRANSLIT_NAMES */
  61
  62 #define FILE_BUFFER (128*1024)
  63
  64 static guint node_cnt=0;                /* Nodes */
  65 static guint node_skip_cnt=0;   /* Skipped nodes */
  66 static guint noded_cnt=0;               /* Nodes with (usable) data */
  67 static guint way_cnt=0;                 /* Ways */
  68 static guint way_names=0;               /* Ways with name */
  69 static guint way_refs=0;                /* Ways with ref or int_ref */
  70
  71 static guint dbnode_cnt=0;
  72 static guint dbnoded_cnt=0;
  73 static guint dbway_cnt=0;
  74
  75 /* For threaded importing */
  76 static GThread* import_thread=NULL;
  77 static GSourceFunc osm_import_progress_cb=NULL;
  78 static osm_import_data_req osm_import_req;
  79 static guint import_sid=0;
  80
  81 static gboolean is_update=FALSE;
  82 static XML_Parser xp;
  83
  84 /* XML tag IDs */
  85 typedef enum {
  86         START,
  87         IN_OSM_TAG,
  88         IN_NODE_TAG,
  89         IN_WNODE_TAG,
  90         IN_WAY_TAG,
  91         IN_KEY_TAG,
  92         IN_BOUND_TAG,
  93         IN_RELATION_TAG,
  94         IN_MEMBER_TAG,
  95         END,
  96         ERROR
  97 } tag_state_t;
  98
  99 /* Parent tag type */
 100 typedef enum {
 101         IS_NONE,
 102         IS_NODE,
 103         IS_WAY,
 104         IS_RELATION
 105 } tag_parent_t;
 106
 107 /* Node types table */
 108 /* XXX: Add support for parent category */
 109 struct _nodeinfo {
 110         gchar *k, *v;
 111         node_type_t type;
 112 } nodeinfo[] = {
 113         { "amenity", "fuel",            NODE_AMENITY_FUEL },
 114         { "amenity", "parking",         NODE_AMENITY_PARKING },
 115
 116         { "amenity", "pub",                     NODE_AMENITY_PUB },
 117         { "amenity", "nightclub",       NODE_AMENITY_NIGHTCLUB },
 118         { "amenity", "biergarten",      NODE_AMENITY_PUB },
 119         { "amenity", "cafe",            NODE_AMENITY_CAFE },
 120         { "amenity", "fast_food",       NODE_AMENITY_FOOD },
 121         { "amenity", "restaurant",      NODE_AMENITY_RESTAURANT },
 122
 123         { "amenity", "telephone",       NODE_AMENITY_TELEPHONE },
 124         { "amenity", "toilets",         NODE_AMENITY_WC },
 125
 126         { "amenity", "hospital",        NODE_AMENITY_HOSPITAL },
 127         { "amenity", "doctors",         NODE_AMENITY_HOSPITAL },
 128         { "amenity", "pharmacy",        NODE_AMENITY_PHARMACY },
 129
 130         { "amenity", "post_office",     NODE_AMENITY_POST },
 131         { "amenity", "post_box",        NODE_AMENITY_POST_BOX },
 132
 133         { "amenity", "cinema",          NODE_AMENITY_CINEMA },
 134         { "amenity", "theatre",         NODE_AMENITY_THEATRE },
 135
 136         { "amenity", "atm",                     NODE_AMENITY_ATM },
 137         { "amenity", "bank",            NODE_AMENITY_BANK },
 138
 139         { "amenity", "police",          NODE_AMENITY_POLICE },
 140         { "amenity", "speed_trap",      NODE_AMENITY_SPEEDCAM },
 141         { "amenity", "speed_camera",    NODE_AMENITY_SPEEDCAM },
 142         { "amenity", "speed camera",    NODE_AMENITY_SPEEDCAM },
 143
 144         { "amenity", "place_of_worship",NODE_AMENITY_POW },
 145
 146         { "amenity", "school",          NODE_AMENITY_SCHOOL },
 147         { "amenity", "college",         NODE_AMENITY_COLLEGE },
 148         { "amenity", "university",      NODE_AMENITY_COLLEGE },
 149
 150         { "amenity", "library", NODE_AMENITY_LIBRARY },
 151         { "amenity", "townhall",        NODE_AMENITY_TOWNHALL },
 152
 153         { "amenity", "supermarket",     NODE_AMENITY_SHOP },
 154         { "amenity", "shopping_centre", NODE_AMENITY_SHOP },
 155         { "amenity", "shop",            NODE_AMENITY_SHOP },
 156         { "amenity", "shops",           NODE_AMENITY_SHOP },
 157         { "amenity", "shopping",        NODE_AMENITY_SHOP },
 158         { "amenity", "shopping_mall",NODE_AMENITY_SHOP },
 159         { "amenity", "cycle_shop",      NODE_AMENITY_SHOP },
 160         { "amenity", "bike_shop",       NODE_AMENITY_SHOP },
 161         { "amenity", "coffee_shop",     NODE_AMENITY_SHOP },
 162         { "amenity", "indoor_shopping_centre",  NODE_AMENITY_SHOP },
 163         { "amenity", "farm_shop",       NODE_AMENITY_SHOP },
 164         { "amenity", "tea_shop",        NODE_AMENITY_SHOP },
 165
 166         /* Shops */
 167         { "shop",        "supermarket", NODE_AMENITY_SHOP },
 168         { "shop",        "bakery",              NODE_AMENITY_SHOP },
 169         { "shop",        "alcohol",             NODE_AMENITY_SHOP },
 170         { "shop",        "butcher",             NODE_AMENITY_SHOP },
 171         { "shop",        "flowers",             NODE_AMENITY_SHOP },
 172         { "shop",        "clothing",    NODE_AMENITY_SHOP },
 173         { "shop",        "souvenir",    NODE_AMENITY_SHOP },
 174         { "shop",        "bicycles",    NODE_AMENITY_SHOP },
 175         { "shop",        "grocers",             NODE_AMENITY_SHOP },
 176         { "shop",        "newsagents",  NODE_AMENITY_SHOP },
 177         { "shop",        "convenience", NODE_AMENITY_SHOP },
 178         { "shop",        "bakers",              NODE_AMENITY_SHOP },
 179         { "shop",        "garden_centre",NODE_AMENITY_SHOP },
 180         { "shop",        "photography", NODE_AMENITY_SHOP },
 181         { "shop",        "general_store",NODE_AMENITY_SHOP },
 182         { "shop",        "food",                NODE_AMENITY_SHOP },
 183         { "shop",        "drinks",              NODE_AMENITY_SHOP },
 184         { "shop",        "sex",                 NODE_AMENITY_SHOP_ADULT },
 185         { "shop",        "pharmacy",    NODE_AMENITY_PHARMACY },
 186
 187         /* Sport */
 188         { "sport"  , "swimming",        NODE_SPORT_SWIMMING },
 189         { "sport"  , "golf",            NODE_SPORT_GOLF },
 190         { "sport"  , "tennis",          NODE_SPORT_TENNIS },
 191         { "sport"  , "football",        NODE_SPORT_FOOTBALL },
 192         { "sport"  , "soccer",          NODE_SPORT_SOCCER },
 193         { "sport"  , "baskteball",      NODE_SPORT_BASKETBALL },
 194         { "sport"  , "rugby",           NODE_SPORT_RUGBY },
 195         { "sport"  , "skating",         NODE_SPORT_SKATING },
 196         { "sport"  , "hockey",          NODE_SPORT_HOCKEY },
 197         { "sport"  , "skateboard",      NODE_SPORT_SKATEBOARD },
 198         { "sport"  , "bowling",         NODE_SPORT_BOWLING },
 199         { "sport"  , "10pin",           NODE_SPORT_BOWLING },
 200         { "sport"  , "motor",           NODE_SPORT_MOTOR },
 201         { "sport"  , "shooting_range",NODE_SPORT_SHOOTING },
 202         { "sport"  , "paintball",       NODE_SPORT_PAINTBALL },
 203         { "sport"  , "horse_racing",NODE_SPORT_HORSES },
 204         { "sport"  , "horse",           NODE_SPORT_HORSES },
 205         { "sport"  , "horses",          NODE_SPORT_HORSES },
 206         { "sport"  , "dog_racing",      NODE_SPORT_DOG },
 207         { "sport"  , "pelota",          NODE_SPORT_PELOTA },
 208         { "sport"  , "racquet",         NODE_SPORT_RACQUET },
 209         { "sport"  , "equestrian",      NODE_SPORT_HORSES },
 210         { "sport"  , "baseball",        NODE_SPORT_BASEBALL },
 211         { "sport"  , "cricket",         NODE_SPORT_CRICKET },
 212         { "sport"  , "croquet",         NODE_SPORT_CROQUET },
 213         { "sport"  , "cycling",         NODE_SPORT_CYCLING },
 214         { "sport"  , "bowls",           NODE_SPORT_BOWLS },
 215         { "sport"  , "athletics",       NODE_SPORT_ATHLETICS },
 216         { "sport"  , "gymnastics",      NODE_SPORT_GYMNASTICS },
 217         { "sport"  , "multi",           NODE_SPORT_OTHER },
 218         { "leisure", "sport_centre",NODE_SPORT_CENTER },
 219
 220         /* Tourism */
 221         { "tourism", "information",     NODE_TOURISM_INFO },
 222         { "tourism", "camp_site",       NODE_TOURISM_CAMP_SITE },
 223         { "tourism", "caravan_site",NODE_TOURISM_CARAVAN_SITE },
 224         { "tourism", "picnic_site",     NODE_TOURISM_PICNIC_SITE },
 225         { "tourism", "theme_park",      NODE_TOURISM_THEME_PARK },
 226         { "tourism", "hotel",           NODE_TOURISM_HOTEL },
 227         { "tourism", "motel",           NODE_TOURISM_MOTEL },
 228         { "tourism", "hostel",          NODE_TOURISM_HOSTEL },
 229         { "tourism", "attraction",      NODE_TOURISM_ATTRACTION },
 230         { "tourism", "zoo",                     NODE_TOURISM_ATTRACTION },
 231
 232         { "historic", "ruins",          NODE_TOURISM_ATTRACTION },
 233         { "historic", "monument",       NODE_TOURISM_ATTRACTION },
 234         { "historic", "memorial",       NODE_TOURISM_ATTRACTION },
 235         { "historic", "museum",         NODE_HISTORIC_MUSEUM },
 236         { "historic", "castle",         NODE_HISTORIC_CASTLE },
 237
 238         { "railway", "station",         NODE_RAILWAY_STATION },
 239         { "railway", "halt",            NODE_RAILWAY_HALT },
 240
 241         { "aeroway", "terminal",        NODE_AIRPORT_TERMINAL },
 242
 243         /* Places */
 244         { "place", "city",                      NODE_PLACE_CITY },
 245         { "place", "town",                      NODE_PLACE_TOWN },
 246         { "place", "village",           NODE_PLACE_VILLAGE },
 247         { "place", "hamlet",            NODE_PLACE_HAMLET },
 248         { "place", "locality",          NODE_PLACE_LOCALITY },
 249         { "place", "suburb",            NODE_PLACE_SUBURB },
 250         { "place", "island",            NODE_PLACE_ISLAND },
 251
 252         { "highway", "traffic_signals", NODE_TRAFFIC_SIGNALS },
 253         { "highway", "motorway_junction",       NODE_JUNCTION },
 254         { "highway", "services",        NODE_AMENITY_PARKING },
 255         { "highway", "toll_booth",      NODE_TOLLBOOTH },
 256         { "highway", "gate",            NODE_GATE },
 257
 258         { NULL, NULL, NODE_PLAIN }
 259 };
 260
 261 /* Array to get id number and defaults for ways of different types */
 262 struct _wayinfo {
 263         gchar *k, *v;
 264         guint defspeed;
 265         way_type_t type;
 266         gboolean oneway, link, area, car, foot;
 267 } wayinfo[] = {
 268         { "highway", "motorway",120,WAY_MOTORWAY,               TRUE, FALSE, FALSE, TRUE, FALSE },
 269         { "highway", "motorway_link",120,WAY_MOTORWAY,  TRUE, TRUE, FALSE, TRUE, FALSE },
 270         { "highway", "trunk",100,WAY_TRUNK,                             FALSE, FALSE, FALSE, TRUE, FALSE },
 271         { "highway", "trunk_link",100,WAY_TRUNK,                FALSE, TRUE, FALSE, TRUE, FALSE },
 272         { "highway", "primary",80,WAY_PRIMARY,                  FALSE, FALSE, FALSE, TRUE, TRUE },
 273         { "highway", "primary_link",60,WAY_PRIMARY,             FALSE, TRUE, FALSE, TRUE, TRUE },
 274         { "highway", "secondary",80,WAY_SECONDARY,              FALSE, FALSE, FALSE, TRUE, TRUE },
 275         { "highway", "secondary_link",60,WAY_SECONDARY, FALSE, TRUE, FALSE, TRUE, TRUE },
 276         { "highway", "tertiary",60,WAY_TERTIARY,                FALSE, FALSE, FALSE, TRUE, TRUE },
 277         { "highway", "unclassified",50,WAY_UNCLASSIFIED,        FALSE, FALSE, FALSE, TRUE, TRUE },
 278         { "highway", "byway",40,WAY_UNCLASSIFIED,       FALSE, FALSE, FALSE, TRUE, TRUE },
 279         { "highway", "residential",40,WAY_RESIDENTIAL,  FALSE, FALSE, FALSE, TRUE, TRUE },
 280         { "highway", "service",20,WAY_SERVICE,                  FALSE, FALSE, FALSE, TRUE, TRUE },
 281         { "highway", "track",20,WAY_TRACK,                              FALSE, FALSE, FALSE, TRUE, TRUE },
 282         { "highway", "unsurfaced",60,WAY_TRACK,                 FALSE, FALSE, FALSE, TRUE, TRUE },
 283         { "highway", "minor",60,WAY_TRACK,                      FALSE, FALSE, FALSE, TRUE, TRUE },
 284         { "highway", "pedestrian",20,WAY_FOOTWAY,               FALSE, FALSE, FALSE, FALSE, TRUE },
 285         { "highway", "footway",1,WAY_FOOTWAY,                   FALSE, FALSE, FALSE, FALSE, TRUE },
 286         { "highway", "steps",0,WAY_FOOTWAY,                             FALSE, FALSE, FALSE, FALSE, TRUE},
 287         { "highway", "bridleway",10,WAY_FOOTWAY,                FALSE, FALSE, FALSE, FALSE, TRUE },
 288         { "highway", "cycleway",10,WAY_CYCLEWAY,                FALSE, FALSE, FALSE, FALSE, TRUE },
 289         { "railway", "rail",0,WAY_RAIL,                                 FALSE, FALSE, FALSE, FALSE, FALSE },
 290         { "aeroway", "runway",0,WAY_RUNWAY,                             FALSE, FALSE, FALSE, FALSE, FALSE },
 291         { "aeroway", "taxiway",0,WAY_TAXIWAY,                   FALSE, FALSE, FALSE, FALSE, FALSE },
 292         { "natural", "water",0,WAY_WATER,                               FALSE, FALSE, TRUE, FALSE, FALSE },
 293         { "waterway", "river",0,WAY_WATER,                              FALSE, FALSE, FALSE, FALSE, FALSE },
 294         { "waterway", "canal",0,WAY_WATER,                              FALSE, FALSE, FALSE, FALSE, FALSE },
 295         { "waterway", "stream",0,WAY_WATER,                             FALSE, FALSE, FALSE, FALSE, FALSE },
 296         { "building", "*",0,WAY_UNWAYED,                                FALSE, FALSE, TRUE, FALSE, FALSE },
 297         { NULL, NULL, 0, WAY_UNWAYED, FALSE, FALSE, FALSE, FALSE, FALSE }
 298 };
 299
 300 static sqlite3 *db;
 301 tag_parent_t tag_parent=IS_NONE;
 302
 303 static GHashTable *osm_nodes;
 304 static GHashTable *osm_node_tags;
 305 static GHashTable *osm_way_tags;
 306 static GSList *osm_ways;
 307 static GSList *osm_poi;
 308
 309 static GHashTable *osm_place_country;
 310 static GHashTable *osm_place_region;
 311 static GHashTable *osm_place_city;
 312 static GHashTable *osm_place_suburb;
 313 static GHashTable *osm_place_village;
 314 static GHashTable *osm_node_isin;
 315 static GHashTable *osm_way_isin;
 316
 317 static node *cnode=NULL;
 318 static way *cway=NULL;
 319
 320 struct sql_stmt {
 321         sqlite3_stmt *insert_poi;
 322         sqlite3_stmt *delete_osm_poi;
 323
 324         sqlite3_stmt *insert_node;
 325         sqlite3_stmt *delete_nodes;
 326         sqlite3_stmt *select_node;
 327         sqlite3_stmt *update_node;
 328
 329         sqlite3_stmt *insert_way_data;
 330         sqlite3_stmt *insert_way_ref;
 331         sqlite3_stmt *insert_way_pc;
 332         sqlite3_stmt *insert_way_name;
 333         sqlite3_stmt *insert_way_names_nls;
 334         sqlite3_stmt *insert_way_n2n;
 335         sqlite3_stmt *delete_way;
 336         sqlite3_stmt *delete_way_n2n;
 337         sqlite3_stmt *delete_way_name;
 338         sqlite3_stmt *delete_way_names_nls;
 339         sqlite3_stmt *delete_way_ref;
 340         sqlite3_stmt *delete_way_pc;
 341
 342         sqlite3_stmt *insert_place;
 343         sqlite3_stmt *delete_place;
 344 };
 345 static struct sql_stmt sql;
 346
 347 static struct map_bbox bbox;
 348 static gboolean use_bbox;
 349
 350 static void osm_free_way_data(way *w);
 351 static void print_way(way *w);
 352
 353 static gboolean osm_db_prepare(sqlite3 *db);
 354 static gboolean db_insert_node(node *n);
 355 static guint32 osm_find_way_place(way *w, node_type_t nt);
 356
 357 /****************************************************/
 358 /* Functions */
 359 /****************************************************/
 360
 361 static void
 362 db_finalize(void)
 363 {
 364 sqlite3_finalize(sql.insert_poi);
 365 sqlite3_finalize(sql.delete_osm_poi);
 366
 367 sqlite3_finalize(sql.insert_node);
 368 sqlite3_finalize(sql.select_node);
 369 sqlite3_finalize(sql.delete_nodes);
 370 sqlite3_finalize(sql.update_node);
 371
 372 sqlite3_finalize(sql.insert_place);
 373 sqlite3_finalize(sql.delete_place);
 374
 375 sqlite3_finalize(sql.delete_way);
 376 sqlite3_finalize(sql.insert_way_data);
 377
 378 sqlite3_finalize(sql.delete_way_name);
 379 sqlite3_finalize(sql.insert_way_name);
 380
 381 sqlite3_finalize(sql.delete_way_n2n);
 382 sqlite3_finalize(sql.insert_way_n2n);
 383
 384 sqlite3_finalize(sql.delete_way_pc);
 385 sqlite3_finalize(sql.insert_way_pc);
 386
 387 sqlite3_finalize(sql.delete_way_names_nls);
 388 sqlite3_finalize(sql.insert_way_names_nls);
 389 }
 390
 391 static gboolean
 392 osm_db_prepare(sqlite3 *db)
 393 {
 394 /* Way nodes */
 395 sqlite3_prepare_v2(db, "insert or replace into nodes (nid,ilat,ilon,rlat,rlon,l,f) values (?,?,?,?,?,0,?)", -1, &sql.insert_node, NULL);
 396 sqlite3_prepare_v2(db, "select ilat,ilon,l from nodes where nid=?", -1, &sql.select_node, NULL);
 397 sqlite3_prepare_v2(db, "delete from nodes", -1, &sql.delete_nodes, NULL);
 398 sqlite3_prepare_v2(db, "update nodes set l=l+1 where nid=?", -1, &sql.update_node, NULL);
 399
 400 /* Places */
 401 sqlite3_prepare_v2(db, "insert or replace into places (nid,type,name,isin_c,isin_p) values (?, ?, ?, ?, ?)", -1, &sql.insert_place, NULL);
 402 sqlite3_prepare_v2(db, "delete from places", -1, &sql.delete_place, NULL);
 403
 404 /* POI nodes */
 405 if (sqlite3_prepare_v2(db, "insert or replace into poi (osm_id, lat, lon, label, cat_id, public, source, priority, isin_c, isin_p, desc, url, postal_code) "
 406                                            " values (?, ?, ?, ?, ?, 1, 1, ?, ?, ?, ?, ?, ?)", -1, &sql.insert_poi, NULL)!=SQLITE_OK)
 407         g_printf("SQL: %s\n", sqlite3_errmsg(db));
 408
 409 sqlite3_prepare_v2(db, "delete from poi where osm_id>0 and source=1", -1, &sql.delete_osm_poi, NULL);
 410
 411 /* Ways */
 412 sqlite3_prepare_v2(db, "insert or replace into way (wid,nodes,type,flags,speed,isin_c,isin_p,lat,lon) values (?, ?, ?, ?, ?, ?, ?, ?, ?)", -1, &sql.insert_way_data, NULL);
 413 sqlite3_prepare_v2(db, "delete from way", -1, &sql.delete_way, NULL);
 414
 415 /* Way nodes */
 416 sqlite3_prepare_v2(db, "insert into way_n2n (wid,f,t) values (?,?,?)", -1, &sql.insert_way_n2n, NULL);
 417 sqlite3_prepare_v2(db, "delete from way_n2n where wid=?", -1, &sql.delete_way_n2n, NULL);
 418
 419 /* Way names */
 420 sqlite3_prepare_v2(db, "insert or replace into way_names (wid,name,norm) values (?, ?, ?)",  -1, &sql.insert_way_name, NULL);
 421 sqlite3_prepare_v2(db, "delete from way_names", -1, &sql.delete_way_name, NULL);
 422
 423 /* Way postal codes */
 424 sqlite3_prepare_v2(db, "insert or replace into way_pc (wid,pc) values (?, ?)",  -1, &sql.insert_way_pc, NULL);
 425 sqlite3_prepare_v2(db, "delete from way_pc", -1, &sql.delete_way_pc, NULL);
 426
 427 /* Other language names for ways */
 428 sqlite3_prepare_v2(db, "insert into way_names_nls (wid,lang,name, norm) values (?, ?, ?, ?)",  -1, &sql.insert_way_names_nls, NULL);
 429 sqlite3_prepare_v2(db, "delete from way_names_nls where wid=?", -1, &sql.delete_way_names_nls, NULL);
 430
 431 /* Way ref and int_ref */
 432 sqlite3_prepare_v2(db, "insert or replace into way_ref (rid,ref,int_ref) values (?, ?, ?)", -1, &sql.insert_way_ref, NULL);
 433 sqlite3_prepare_v2(db, "delete from way_ref", -1, &sql.delete_way_ref, NULL);
 434
 435 return TRUE;
 436 }
 437
 438 /********************************************************************/
 439
 440 static void
 441 print_way(way *w)
 442 {
 443 #ifdef VERBOSE
 444 g_assert(w);
 445 g_printf("Way #%d(N:%d T:%d S:%d IS: %d/%d): %s [%s:%s:%s]\n",
 446                 w->id,
 447                 g_slist_length(w->nodes),
 448                 w->type,
 449                 w->data ? w->data->speed : 0,
 450                 w->data ? w->data->isin_c : -1,
 451                 w->data ? w->data->isin_p : -1,
 452                 w->data ? w->data->name ? w->data->name : "" : "",
 453                 w->flags & W_ONEWAY ? "-" : "=",
 454                 w->flags & W_ROUNDABOUT ? "O" : "-",
 455                 w->flags & W_LINK ? "|" : " ");
 456 #endif
 457 }
 458
 459 static void
 460 print_node(node *n)
 461 {
 462 #ifdef VERBOSE
 463 g_assert(n);
 464 g_printf("Node #%d: T:%d IS: %d/%d [%s]\n",
 465         n->id,
 466         n->type,
 467         n->data ? n->data->isin_c : -1,
 468         n->data ? n->data->isin_p : -1,
 469         n->data ? n->data->name : "");
 470 #endif
 471 }
 472
 473 /********************************************************************/
 474
 475 static gboolean
 476 db_insert_node(node *n)
 477 {
 478 gint32 lat, lon;
 479
 480 g_assert(n);
 481
 482 lat=lat2mp_int(n->lat);
 483 lon=lon2mp_int(n->lon);
 484
 485 sqlite3_bind_int(sql.insert_node, 1, n->id);
 486
 487 /* Projected and integerized lat/lot */
 488 sqlite3_bind_int(sql.insert_node, 2, lat);
 489 sqlite3_bind_int(sql.insert_node, 3, lon);
 490 /* Original */
 491 sqlite3_bind_double(sql.insert_node, 4, n->lat);
 492 sqlite3_bind_double(sql.insert_node, 5, n->lon);
 493 sqlite3_bind_int(sql.insert_node, 6, n->type);
 494
 495 db_exec(db, sql.insert_node);
 496
 497 return TRUE;
 498 }
 499
 500 static gboolean
 501 db_insert_place(node *n)
 502 {
 503 g_assert(n);
 504 if (!n->data)
 505         return FALSE;
 506 if (!n->data->name)
 507         return FALSE;
 508 sqlite3_bind_int(sql.insert_place, 1, n->id);
 509 sqlite3_bind_int(sql.insert_place, 2, n->type);
 510 sqlite3_bind_text(sql.insert_place, 3, n->data->name, -1, SQLITE_TRANSIENT);
 511 sqlite3_bind_int(sql.insert_place, 4, n->data->isin_p);
 512 sqlite3_bind_int(sql.insert_place, 5, n->data->isin_c);
 513
 514 return db_exec(db,sql.insert_place);
 515 }
 516
 517 static gboolean
 518 db_insert_poi(node *n)
 519 {
 520 g_assert(n);
 521 sqlite3_bind_int(sql.insert_poi, 1, n->id);
 522 sqlite3_bind_double(sql.insert_poi, 2, n->lat);
 523 sqlite3_bind_double(sql.insert_poi, 3, n->lon);
 524 if (n->data->name)
 525         sqlite3_bind_text(sql.insert_poi, 4, n->data->name, -1, SQLITE_TRANSIENT);
 526 else
 527         sqlite3_bind_text(sql.insert_poi, 4, "", -1, SQLITE_TRANSIENT);
 528 sqlite3_bind_int(sql.insert_poi, 5, n->type);
 529 sqlite3_bind_int(sql.insert_poi, 6, n->type/100);
 530 sqlite3_bind_int(sql.insert_poi, 7, n->data->isin_c);
 531 sqlite3_bind_int(sql.insert_poi, 8, n->data->isin_p);
 532
 533 if (n->data->desc)
 534         sqlite3_bind_text(sql.insert_poi, 9, n->data->desc, -1, SQLITE_TRANSIENT);
 535 if (n->data->url)
 536         sqlite3_bind_text(sql.insert_poi, 10, n->data->url, -1, SQLITE_TRANSIENT);
 537 if (n->data->postal_code)
 538         sqlite3_bind_text(sql.insert_poi, 11, n->data->postal_code, -1, SQLITE_TRANSIENT);
 539
 540 return db_exec(db,sql.insert_poi);
 541 }
 542
 543 /**
 544  * Update node usage count
 545  */
 546 static gboolean
 547 db_update_node_links(node *n)
 548 {
 549 g_assert(n);
 550 sqlite3_bind_int(sql.update_node, 1, n->id);
 551
 552 return db_exec(db,sql.update_node);
 553 }
 554
 555 /**
 556  * Insert way,node1,node2 triplet
 557  */
 558 static gboolean
 559 db_insert_way_n2n(way *w, node *nf, node *nt)
 560 {
 561 if (!w) {
 562         g_printf("NULL WAY\n");
 563         return FALSE;
 564 }
 565
 566 if (!nf) {
 567         g_printf("NULL NODE 1\n");
 568         return FALSE;
 569 }
 570
 571 if (!nt) {
 572         g_printf("NULL NODE 2\n");
 573         return FALSE;
 574 }
 575
 576 sqlite3_bind_int(sql.insert_way_n2n, 1, w->id);
 577 sqlite3_bind_int(sql.insert_way_n2n, 2, nf->id);
 578 sqlite3_bind_int(sql.insert_way_n2n, 3, nt->id);
 579
 580 #ifdef VERBOSE_N2N
 581 g_printf("%d [%d - %d]\n", w->id, nf->id, nt->id);
 582 #endif
 583
 584 db_exec(db,sql.insert_way_n2n);
 585 db_update_node_links(nf);
 586 db_update_node_links(nt);
 587 return TRUE;
 588 }
 589
 590 /**
 591  * Insert way ref and int_ref
 592  */
 593 static gboolean
 594 db_insert_way_ref(way *w)
 595 {
 596 if (!w->data)
 597         return TRUE;
 598
 599 if (!w->data->ref && !w->data->int_ref)
 600         return TRUE;
 601
 602 way_refs++;
 603
 604 sqlite3_bind_int(sql.insert_way_ref, 1, w->id);
 605 if (w->data->ref)
 606         sqlite3_bind_text(sql.insert_way_ref, 2, w->data->ref, -1, SQLITE_TRANSIENT);
 607 if (w->data->int_ref)
 608         sqlite3_bind_text(sql.insert_way_ref, 3, w->data->int_ref, -1, SQLITE_TRANSIENT);
 609
 610 return db_exec(db,sql.insert_way_ref);
 611 }
 612
 613 /**
 614  * Insert way name
 615  */
 616 static gboolean
 617 db_insert_way_name(way *w)
 618 {
 619 if (!w->data)
 620         return TRUE;
 621 if (!w->data->name)
 622         return TRUE;
 623
 624 way_names++;
 625
 626 sqlite3_bind_int(sql.insert_way_name, 1, w->id);
 627 sqlite3_bind_text(sql.insert_way_name, 2, w->data->name, -1, SQLITE_TRANSIENT);
 628
 629 #ifdef TRANSLIT_NAMES
 630 {
 631         gchar *norm;
 632         norm=g_convert(w->data->name, -1, "ASCII//TRANSLIT//IGNORE", "utf8", NULL, NULL, NULL);
 633         if (norm && strcmp(w->data->name, norm)!=0) {
 634                 sqlite3_bind_text(sql.insert_way_name, 3, norm, -1, SQLITE_TRANSIENT);
 635         }
 636         if (norm)
 637                 g_free(norm);
 638 }
 639 #endif
 640
 641 return db_exec(db,sql.insert_way_name);
 642 }
 643
 644 static gboolean
 645 db_delete_way_names_nls(way *w)
 646 {
 647 sqlite3_bind_int(sql.delete_way_names_nls, 1, w->id);
 648 return db_exec(db,sql.delete_way_names_nls);
 649 }
 650
 651 static gboolean
 652 db_insert_way_pc(way *w)
 653 {
 654 if (!w->data)
 655         return TRUE;
 656 if (!w->data->postal_code)
 657         return TRUE;
 658
 659 sqlite3_bind_int(sql.insert_way_pc, 1, w->id);
 660 sqlite3_bind_text(sql.insert_way_pc, 2, w->data->postal_code, -1, SQLITE_TRANSIENT);
 661
 662 return db_exec(db,sql.insert_way_pc);
 663 }
 664
 665 static gboolean
 666 db_delete_way_pc(way *w)
 667 {
 668 sqlite3_bind_int(sql.delete_way_pc, 1, w->id);
 669 return db_exec(db,sql.delete_way_pc);
 670 }
 671
 672 static void
 673 db_insert_way_names_nls_cb(gpointer key, gpointer value, gpointer user_data)
 674 {
 675 way *w=(way *)user_data;
 676
 677 sqlite3_bind_int(sql.insert_way_names_nls, 1, w->id);
 678 sqlite3_bind_text(sql.insert_way_names_nls, 2, (gchar *)key, -1, SQLITE_TRANSIENT);
 679 sqlite3_bind_text(sql.insert_way_names_nls, 3, (gchar *)value, -1, SQLITE_TRANSIENT);
 680 #ifdef TRANSLIT_NAMES
 681 {
 682         gchar *norm;
 683         norm=g_convert((gchar *value), -1, "ASCII//TRANSLIT//IGNORE", "utf8", NULL, NULL, NULL);
 684         if (norm && strcmp((gchar *)value, norm)!=0) {
 685                 sqlite3_bind_text(sql.insert_way_names_nls, 4, norm, -1, SQLITE_TRANSIENT);
 686         }
 687         if (norm)
 688                 g_free(norm);
 689 }
 690 #endif
 691 db_exec(db,sql.insert_way_names_nls);
 692 }
 693
 694 static void
 695 db_insert_way_names_nls(way *w)
 696 {
 697 if (!w->data)
 698         return;
 699 if (!w->data->names)
 700         return;
 701
 702 g_hash_table_foreach(w->data->names, db_insert_way_names_nls_cb, w);
 703 }
 704
 705 /**
 706  * Insert all data for the given way
 707  * - name
 708  * - ref
 709  * - nodes
 710  *
 711  */
 712 static gboolean
 713 db_insert_way(way *w)
 714 {
 715 GSList *iter;
 716 guint ncnt;
 717 node *wmn;
 718
 719 if (!w)
 720         return FALSE;
 721
 722 /* Skip things we don't use (yet) */
 723 if (w->type==WAY_UNWAYED || w->type>WAY_ROAD_END)
 724         return TRUE;
 725
 726 /* Insert nodes */
 727 for (iter=w->nodes; iter!=NULL; iter=iter->next) {
 728         if (!iter->next)
 729                 break;
 730         db_insert_way_n2n(w, iter->data, iter->next->data);
 731 }
 732
 733 if (w->id==0)
 734         return FALSE;
 735
 736 if (w->data) {
 737         w->data->isin_p=osm_find_way_place(w, NODE_PLACE_CITY);
 738         w->data->isin_c=osm_find_way_place(w, NODE_PLACE_COUNTRY);
 739 }
 740
 741 print_way(w);
 742
 743 sqlite3_bind_int(sql.insert_way_data, 1, w->id);
 744 sqlite3_bind_int(sql.insert_way_data, 2, w->ncnt);
 745 sqlite3_bind_int(sql.insert_way_data, 3, w->type);
 746 sqlite3_bind_int(sql.insert_way_data, 4, w->flags);
 747 if (w->data) {
 748         sqlite3_bind_int(sql.insert_way_data, 5, w->data->speed);
 749         sqlite3_bind_int(sql.insert_way_data, 6, w->data->isin_c);
 750         sqlite3_bind_int(sql.insert_way_data, 7, w->data->isin_p);
 751 }
 752
 753 /* Get middle node, use it as a rough way location */
 754 ncnt=g_slist_length(w->nodes);
 755 if (ncnt>1) {
 756         wmn=g_slist_nth_data(w->nodes, ncnt/2);
 757         if (wmn) {
 758                 sqlite3_bind_double(sql.insert_way_data, 8, wmn->lat);
 759                 sqlite3_bind_double(sql.insert_way_data, 9, wmn->lon);
 760         } else {
 761                 g_printerr("Failed to get way middlepoint node for location information!\n");
 762         }
 763 }
 764
 765 db_exec(db,sql.insert_way_data);
 766
 767 db_insert_way_ref(w);
 768 db_insert_way_name(w);
 769 db_insert_way_names_nls(w);
 770 db_insert_way_pc(w);
 771
 772 osm_free_way_data(w);
 773 return TRUE;
 774 }
 775
 776 /********************************************************************/
 777
 778 static gchar *
 779 get_attr_key_value(const gchar **p, gchar *key)
 780 {
 781 gchar **d;
 782
 783 d=p;
 784 while (*d!=NULL) {
 785         if (strncmp(*d, key, strlen(key))==0) {
 786                 d++;
 787                 return *d;
 788         }
 789         d++;
 790         d++;
 791 }
 792 return NULL;
 793 }
 794
 795 static tag_state_t
 796 check_tag(const gchar *tag)
 797 {
 798 if (strcmp(tag,"node")==0) return IN_NODE_TAG;
 799 else if (strcmp(tag,"nd")==0) return IN_WNODE_TAG;
 800 else if (strcmp(tag,"way")==0) return IN_WAY_TAG;
 801 else if (strcmp(tag,"tag")==0) return IN_KEY_TAG;
 802 else if (strcmp(tag,"osm")==0) return IN_OSM_TAG;
 803 else if (strcmp(tag,"bound")==0) return IN_BOUND_TAG;
 804 else if (strcmp(tag,"relation")==0) return IN_RELATION_TAG;
 805 else if (strcmp(tag,"member")==0) return IN_MEMBER_TAG;
 806 else return ERROR;
 807 }
 808
 809 static void
 810 find_nls_names(gpointer key, gpointer value, gpointer user_data)
 811 {
 812 gchar *k, *v;
 813 gchar *tmp;
 814 GHashTable *nls;
 815
 816 k=(gchar *)key;
 817 v=(gchar *)value;
 818 nls=(GHashTable *)user_data;
 819
 820 /* Check if it is a name key, return if not. */
 821 if (g_str_has_prefix(k, "name:")==FALSE)
 822         return;
 823
 824 tmp=g_strrstr(k, ":");
 825 if (!tmp)
 826         return;
 827 tmp++; /* skip : */
 828 if (*tmp==0)
 829         return;
 830 g_hash_table_insert(nls, g_strdup(tmp), g_strdup(v));
 831 #ifdef VERBOSE
 832 g_printf("NLS(%s): [%s]\n", tmp, v);
 833 #endif
 834 }
 835
 836 /********************************************************************/
 837
 838 static void
 839 node_print (node *n)
 840 {
 841 g_assert(n);
 842 if (n->data) {
 843         g_printf("N: %d [%f:%f][%s](%d)\n",
 844                 n->id, n->lat, n->lon,
 845                 n->data->name ? n->data->name : "-",
 846                 n->type);
 847 } else {
 848         g_printf("N: %d [%f:%f]\n",
 849                 n->id, n->lat, n->lon);
 850 }
 851 }
 852
 853 #ifdef DEBUG
 854 static void
 855 dump_array(const gchar **p)
 856 {
 857 char **d;
 858
 859 d=p;
 860 while (*d!=NULL) {
 861         g_printf("[%s]", *d);
 862         d++;
 863 }
 864 g_print("\n");
 865 }
 866 #endif
 867
 868 static inline gboolean
 869 osm_node_check_box(gdouble nlat, gdouble nlon)
 870 {
 871 if (use_bbox==FALSE)
 872         return TRUE;
 873 return (nlat > bbox.lat_min && nlat < bbox.lat_max && nlon > bbox.lon_min && nlon < bbox.lon_max) ? TRUE : FALSE;
 874 }
 875
 876 static void
 877 osm_new_node_data(node *n)
 878 {
 879 if (n==NULL)
 880         return;
 881 if (n->data!=NULL)
 882         return;
 883 n->data=g_slice_new(node_data);
 884 n->data->name=NULL;
 885 n->data->url=NULL;
 886 n->data->desc=NULL;
 887 n->data->postal_code=NULL;
 888 n->type=NODE_PLAIN;
 889 noded_cnt++;
 890 }
 891
 892 static void
 893 osm_free_node_data(node *n)
 894 {
 895 g_assert(n);
 896 g_assert(n->data);
 897 if (n->data->name)
 898         g_free(n->data->name);
 899 if (n->data->url)
 900         g_free(n->data->url);
 901 if (n->data->desc)
 902         g_free(n->data->desc);
 903 if (n->data->postal_code)
 904         g_free(n->data->postal_code);
 905 g_slice_free(node_data, n->data);
 906 n->data=NULL;
 907 noded_cnt--;
 908 }
 909
 910 static node *
 911 osm_new_node(gint id, gdouble lat, gdouble lon)
 912 {
 913 node *n=NULL;
 914
 915 n=g_slice_new(node);
 916 g_assert(n);
 917 n->id=id;
 918 n->lat=lat;
 919 n->lon=lon;
 920 n->data=(node_data *)NULL;
 921 return n;
 922 }
 923
 924 static void
 925 osm_free_node(node *n)
 926 {
 927 g_assert(n);
 928 g_slice_free(node, n);
 929 }
 930
 931 static node *
 932 osm_find_node(guint32 nid)
 933 {
 934 node *n;
 935
 936 g_assert(osm_nodes);
 937 return g_hash_table_lookup(osm_nodes, GINT_TO_POINTER(nid));
 938 }
 939
 940 static void
 941 osm_new_way_data(way *w)
 942 {
 943 if (w==NULL)
 944         return;
 945 if (w->data!=NULL)
 946         return;
 947
 948 w->data=g_slice_new(way_data);
 949 w->data->name=NULL;
 950 w->data->names=NULL;
 951 w->data->ref=NULL;
 952 w->data->int_ref=NULL;
 953 w->data->postal_code=NULL;
 954 w->data->layer=0;
 955 w->data->speed=0;
 956 }
 957
 958 static void
 959 osm_free_way_data(way *w)
 960 {
 961 g_assert(w);
 962 if (!w->data)
 963         return;
 964 if (w->data->name)
 965         g_free(w->data->name);
 966 if (w->data->ref)
 967         g_free(w->data->ref);
 968 if (w->data->int_ref)
 969         g_free(w->data->int_ref);
 970 g_slice_free(way_data, w->data);
 971 w->data=NULL;
 972 }
 973
 974 static way *
 975 osm_new_way(gint id)
 976 {
 977 way *w;
 978
 979 w=g_slice_new(way);
 980 g_assert(w);
 981 w->id=id;
 982 w->nodes=NULL;
 983 w->type=WAY_UNWAYED;
 984 w->data=NULL;
 985 w->ncnt=0;
 986 w->flags=0;
 987
 988 /* Add to list of ways */
 989 return w;
 990 }
 991
 992 static void
 993 osm_free_way(way *w)
 994 {
 995 if (w->nodes)
 996         g_slist_free(w->nodes);
 997 g_slice_free(way, w);
 998 }
 999
1000 static void
1001 osm_way_add_to_list(way *w)
1002 {
1003 g_assert(w);
1004 osm_ways=g_slist_prepend(osm_ways, w);
1005 }
1006
1007 static void
1008 osm_way_new_node(way *w, gint nid)
1009 {
1010 node *n;
1011
1012 g_assert(w);
1013 n=osm_find_node(nid);
1014 w->nodes=g_slist_prepend(w->nodes, n);
1015 w->ncnt++;
1016 }
1017
1018 /**
1019  * Search the place hash table for the location of the node.
1020  *
1021  */
1022 static guint32
1023 osm_find_node_place(node *n)
1024 {
1025 node *t;
1026 gchar **isin;
1027 gchar **place;
1028
1029 if (!n->data)
1030         return 0;
1031
1032 isin=g_hash_table_lookup(osm_node_isin, GINT_TO_POINTER(n->id));
1033
1034 if (!isin)
1035         return 0;
1036
1037 place=isin;
1038 while (*place!=NULL) {
1039         gchar *ps;
1040         ps=g_strstrip(*place);
1041 #ifdef VERBOSE
1042         g_printf("Checking (%d) [%s] in [%s]\n",n->type, n->data->name, ps);
1043 #endif
1044         switch (n->type) {
1045         case NODE_PLACE_CITY:
1046         case NODE_PLACE_TOWN:
1047         case NODE_PLACE_VILLAGE:
1048         case NODE_PLACE_HAMLET:
1049                 t=g_hash_table_lookup(osm_place_region, ps);
1050                 if (t)
1051                         return t->id;
1052                 t=g_hash_table_lookup(osm_place_country, ps);
1053                 if (t)
1054                         return t->id;
1055         break;
1056         case NODE_PLACE_SUBURB:
1057         case NODE_PLACE_LOCALITY:
1058                 t=g_hash_table_lookup(osm_place_city, ps);
1059                 if (t)
1060                         return t->id;
1061         break;
1062         case NODE_PLACE_ISLAND:
1063                 return 0;
1064         break;
1065         default:
1066                 t=g_hash_table_lookup(osm_place_city, ps);
1067                 if (t)
1068                         return t->id;
1069         break;
1070         }
1071         place++;
1072 }
1073
1074 return 0;
1075 }
1076
1077 static guint32
1078 osm_find_way_place(way *w, node_type_t nt)
1079 {
1080 gchar **isin;
1081 gchar **place;
1082
1083 isin=g_hash_table_lookup(osm_way_isin, GINT_TO_POINTER(w->id));
1084 if (!isin)
1085         return 0;
1086
1087 place=isin;
1088 while (*place!=NULL) {
1089         node *t;
1090         gchar *ps;
1091
1092         ps=g_strstrip(*place);
1093
1094 #ifdef VERBOSE
1095         g_printf("Checking (%d) in [%s]\n",w->id, ps);
1096 #endif
1097 switch (nt) {
1098         case NODE_PLACE_CITY:
1099         case NODE_PLACE_TOWN:
1100         case NODE_PLACE_VILLAGE:
1101         case NODE_PLACE_HAMLET:
1102         case NODE_PLACE_LOCALITY:
1103                 t=g_hash_table_lookup(osm_place_city, ps);
1104                 if (t)
1105                         return t->id;
1106         break;
1107         case NODE_PLACE_COUNTRY:
1108                 t=g_hash_table_lookup(osm_place_country, ps);
1109                 if (t)
1110                         return t->id;
1111         break;
1112         default:
1113                 g_assert_not_reached();
1114         break;
1115         }
1116         place++;
1117 }
1118
1119 return 0;
1120 }
1121
1122 /***********************************************************************/
1123
1124 static void
1125 osm_node_save_node(gint key, gpointer value, gpointer user_data)
1126 {
1127 node *n=(node *)value;
1128
1129 dbnode_cnt++;
1130 db_insert_node(n);
1131 if (dbnode_cnt % 20000==0)
1132         g_printf("\rNodes: %f%% (%u/%u)\n",
1133                 ((float)dbnode_cnt/(float)(node_cnt-node_skip_cnt))*100,
1134                 dbnode_cnt, node_cnt);
1135 }
1136
1137 /**
1138  * Check node type and insert as POI or Place
1139  * Discard extra data after insert.
1140  */
1141 static gboolean
1142 osm_node_save_poi(node *n, gpointer user_data)
1143 {
1144 if (!n) {
1145         g_printerr("ERROR: null poi\n");
1146         return FALSE;
1147 }
1148
1149 if (!n->data) {
1150         g_printerr("POI node with no data ?\n");
1151         return FALSE;
1152 }
1153
1154 n->data->isin_p=osm_find_node_place(n);
1155 n->data->isin_c=0;
1156
1157 if (n->type>NODE_POI_START && n->type<NODE_POI_END) {
1158         print_node(n);
1159         db_insert_poi(n);
1160         osm_free_node_data(n);
1161 } else if (n->type>NODE_PLACE_START && n->type<NODE_PLACE_END) {
1162         print_node(n);
1163         db_insert_place(n);
1164 } else {
1165         osm_free_node_data(n);
1166         return FALSE;
1167 }
1168
1169 return TRUE;
1170 }
1171
1172 static gboolean
1173 osm_planet_poi_clear_nodes(void)
1174 {
1175 g_print("Removing old OSM POIs...\n");
1176 db_transaction_begin(db);
1177 sqlite3_step(sql.delete_osm_poi);
1178 sqlite3_step(sql.delete_place);
1179 return db_transaction_commit(db);
1180 }
1181
1182 static gboolean
1183 osm_planet_poi_save_nodes(void)
1184 {
1185 g_print("Storing new POIs...\n");
1186 db_transaction_begin(db);
1187 g_slist_foreach(osm_poi, osm_node_save_poi, NULL);
1188 g_slist_free(osm_poi);
1189 return db_transaction_commit(db);
1190 }
1191
1192 /*********************************************************************/
1193
1194 static void
1195 osm_planet_clear_nodes(void)
1196 {
1197 g_print("Clearing old nodes\n");
1198 sqlite3_step(sql.delete_nodes);
1199 }
1200
1201 static gboolean
1202 osm_planet_save_nodes(void)
1203 {
1204 g_print("Storing nodes...\n");
1205 db_transaction_begin(db);
1206 g_hash_table_foreach(osm_nodes, osm_node_save_node, NULL);
1207 return db_transaction_commit(db);
1208 }
1209
1210 /*********************************************************************/
1211
1212 static void
1213 osm_way_save(way *value, gpointer user_data)
1214 {
1215 dbway_cnt++;
1216 db_insert_way(value);
1217 if (dbway_cnt % 15000==0 && dbway_cnt>0) {
1218                 g_printf("\rWays: %f%% (%u/%u)\n",
1219                         (((float)dbway_cnt/(float)way_cnt)*100),
1220                         dbway_cnt, way_cnt);
1221                 print_way(value);
1222 }
1223 }
1224
1225 static void
1226 osm_planet_clear_ways(void)
1227 {
1228 g_print("Clearing old data\n");
1229 sqlite3_step(sql.delete_way);
1230 sqlite3_step(sql.delete_way_name);
1231 sqlite3_step(sql.delete_way_ref);
1232 sqlite3_step(sql.delete_way_n2n);
1233 }
1234
1235 static gboolean
1236 osm_planet_save_ways(void)
1237 {
1238 g_print("Inserting new ways\n");
1239 db_transaction_begin(db);
1240 g_slist_foreach(osm_ways, osm_way_save, NULL);
1241 return db_transaction_commit(db);
1242 }
1243
1244 /*********************************************************************/
1245
1246 static void
1247 osm_planet_save_all_nodes(void)
1248 {
1249 g_printf("Saving planet nodes to database:\n");
1250
1251 osm_planet_poi_clear_nodes();
1252 osm_planet_poi_save_nodes();
1253
1254 if (!is_update) {
1255         osm_planet_clear_nodes();
1256         osm_planet_clear_ways();
1257 }
1258 osm_planet_save_nodes();
1259 }
1260
1261 static void
1262 osm_planet_save_all_ways(void)
1263 {
1264 g_printf("Saving planet way to database:\n");
1265
1266 osm_planet_save_ways();
1267 }
1268
1269 /***********************************************************************/
1270
1271 static void
1272 _osm_tag_start(void *userData, const char *name, const char **atts)
1273 {
1274 tag_state_t t;
1275 gchar *k, *v;
1276 guint32 id, ndref;
1277 gdouble nlat, nlon;
1278
1279 t=check_tag(name);
1280 switch (t) {
1281         case IN_OSM_TAG:
1282                 g_printf("Starting...\n");
1283         break;
1284         case IN_NODE_TAG:
1285                 tag_parent=IS_NODE;
1286                 node_cnt++;
1287
1288                 id=atoi(get_attr_key_value(atts, "id"));
1289                 nlat=atof(get_attr_key_value(atts, "lat"));
1290                 nlon=atof(get_attr_key_value(atts, "lon"));
1291
1292                 cnode=osm_new_node(id, nlat, nlon);
1293                 osm_node_tags=g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free);
1294         break;
1295         case IN_WAY_TAG:
1296                 tag_parent=IS_WAY;
1297                 way_cnt++;
1298                 id=atoi(get_attr_key_value(atts, "id"));
1299                 cway=osm_new_way(id);
1300                 osm_way_tags=g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free);
1301         break;
1302         case IN_WNODE_TAG:
1303                 ndref=atoi(get_attr_key_value(atts, "ref"));
1304                 if (use_bbox==TRUE) {
1305                         if (osm_find_node(ndref)==NULL) {
1306                                 cway->id=0;
1307                                 return;
1308                         }
1309                 }
1310                 osm_way_new_node(cway, ndref);
1311         break;
1312         case IN_KEY_TAG:
1313                 k=get_attr_key_value(atts, "k");
1314                 if (strcmp(k,"created_by")==0)
1315                         return;
1316                 if (strcmp(k,"source")==0)
1317                         return;
1318
1319                 v=get_attr_key_value(atts, "v");
1320 #ifdef VERBOSE_KEYS
1321                 g_printf("TAG: K=[%s] V=[%s]\n", k, v);
1322 #endif
1323
1324                 switch (tag_parent) {
1325                 case IS_NONE:
1326                         g_printf("Tag key/value pair but unknown owner\n");
1327                 break;
1328                 case IS_NODE:
1329                 {
1330                         if (!osm_node_tags)
1331                                 return;
1332
1333                         /* Insert key/value pairs into hash table */
1334                         if (cnode==NULL) {
1335                                 g_printerr("In node tags but node is NULL!\n");
1336                                 return;
1337                         }
1338                         g_hash_table_insert(osm_node_tags, g_strdup(k), g_strdup(v));
1339                 }
1340                 break;
1341                 case IS_WAY:
1342                 {
1343                         if (cway==NULL) {
1344                                 g_printerr("In way tags but way is NULL!\n");
1345                                 return;
1346                         }
1347                         g_hash_table_insert(osm_way_tags, g_strdup(k), g_strdup(v));
1348                         osm_new_way_data(cway);
1349                 }
1350                 break;
1351                 case IS_RELATION:
1352
1353                 break;
1354                 }
1355         break;
1356         case IN_BOUND_TAG:
1357                 /* Ignore for now */
1358                 g_printf("Ignoring bound tag\n");
1359         break;
1360         case IN_RELATION_TAG:
1361                 tag_parent=IS_RELATION;
1362
1363         break;
1364         case IN_MEMBER_TAG:
1365
1366         break;
1367         default:
1368                 tag_parent=IS_NONE;
1369                 g_printf("Unknown tag: %s\n", name);
1370         break;
1371 }
1372 }
1373
1374 static void
1375 _osm_tag_end(void *userData, const char *name)
1376 {
1377 tag_state_t t;
1378 gchar *v;
1379 guint i;
1380 t=check_tag(name);
1381 switch (t) {
1382         case IN_NODE_TAG:
1383
1384                 if (node_cnt % 25000==0) {
1385                         g_printf("Nodes: %u of %u, POIs: %u, Outside box: %u\n",
1386                                 node_cnt-node_skip_cnt, node_cnt, noded_cnt, node_skip_cnt);
1387                 }
1388
1389                 if (!osm_node_tags)
1390                         return;
1391
1392                 osm_new_node_data(cnode);
1393
1394                 for (i=0; nodeinfo[i].k; i++) {
1395                         v=g_hash_table_lookup(osm_node_tags, nodeinfo[i].k);
1396                         if (!v)
1397                                 continue;
1398                         if (strcasecmp (v, nodeinfo[i].v)==0) {
1399                                 cnode->type=nodeinfo[i].type;
1400                                 break;
1401                         }
1402                 }
1403
1404                 /* Check if node is inside bounding box, if not skip it.
1405                  * But keep it if it's something we might need for other nodes:
1406                  * - Places (for is_in)
1407                  * - ...
1408                  */
1409                 if ((osm_node_check_box(cnode->lat, cnode->lon)==FALSE) &&
1410                                 (cnode->type<NODE_PLACE_START)) {
1411                         osm_free_node_data(cnode);
1412                         osm_free_node(cnode);
1413                         g_hash_table_destroy(osm_node_tags);
1414                         node_skip_cnt++;
1415                         return;
1416                 }
1417
1418                 g_hash_table_insert(osm_nodes, GINT_TO_POINTER(cnode->id), cnode);
1419
1420                 if (cnode->type!=NODE_PLAIN) {
1421                         cnode->data->name=NULL;
1422                         v=g_hash_table_lookup(osm_node_tags, "name");
1423                         if (v)
1424                                 cnode->data->name=g_strstrip(g_utf8_normalize(v, -1, G_NORMALIZE_ALL_COMPOSE));
1425                         v=g_hash_table_lookup(osm_node_tags, "note");
1426                         if (v)
1427                                 cnode->data->desc=g_strstrip(g_strdup(v));
1428                         v=g_hash_table_lookup(osm_node_tags, "postal_code");
1429                         if (v)
1430                                 cnode->data->postal_code=g_strstrip(g_strdup(v));
1431
1432                         /* Links */
1433                         v=g_hash_table_lookup(osm_node_tags, "url");
1434                         if (v) {
1435                                 cnode->data->url=g_strstrip(g_strdup(v));
1436                         } else {
1437                                 v=g_hash_table_lookup(osm_node_tags, "wikipedia");
1438                                 if (v && strncmp(v,"http:", 5)==0)
1439                                         cnode->data->url=g_strstrip(g_strdup(v));
1440                         }
1441                 }
1442
1443                 cnode->data->isin_c=0;
1444                 cnode->data->isin_p=0;
1445                 v=g_hash_table_lookup(osm_node_tags, "is_in");
1446                 if (v) {
1447                         gchar **isin;
1448                         isin=g_strsplit(v, ",", 10);
1449                         g_hash_table_insert(osm_node_isin, GINT_TO_POINTER(cnode->id), isin);
1450                 }
1451
1452                 if (cnode->type==NODE_PLAIN) {
1453                         osm_free_node_data(cnode);
1454                 } else {
1455                         osm_poi=g_slist_prepend(osm_poi, cnode);
1456                         if (cnode->data->name) {
1457                                 switch (cnode->type) {
1458                                 case NODE_PLACE_COUNTRY:
1459                                         g_hash_table_insert(osm_place_country, cnode->data->name, cnode);
1460                                 break;
1461                                 case NODE_PLACE_CITY:
1462                                 case NODE_PLACE_TOWN:
1463                                         g_hash_table_insert(osm_place_city, cnode->data->name, cnode);
1464                                 break;
1465                                 case NODE_PLACE_SUBURB:
1466                                         g_hash_table_insert(osm_place_suburb, cnode->data->name, cnode);
1467                                 break;
1468                                 case NODE_PLACE_VILLAGE:
1469                                 case NODE_PLACE_HAMLET:
1470                                 case NODE_PLACE_LOCALITY:
1471                                         g_hash_table_insert(osm_place_village, cnode->data->name, cnode);
1472                                 break;
1473                                 case NODE_PLACE_ISLAND:
1474                                         /* Ignore for now */
1475                                 break;
1476                                 default:;
1477                                 }
1478                         }
1479                 }
1480                 g_hash_table_destroy(osm_node_tags);
1481                 cnode=NULL;
1482         break;
1483         case IN_WAY_TAG:
1484                 if (way_cnt % 1024==0) {
1485                         g_printf("\rWays: %d\n", way_cnt);
1486                 }
1487
1488                 cway->nodes=g_slist_reverse(cway->nodes);
1489
1490                 for (i=0; wayinfo[i].k; i++) {
1491                         v=g_hash_table_lookup(osm_way_tags, wayinfo[i].k);
1492                         if (!v)
1493                                 continue;
1494                         if (strcasecmp (v, wayinfo[i].v)==0) {
1495                                 if (wayinfo[i].link==TRUE)
1496                                         cway->flags|=W_LINK;
1497                                 if (wayinfo[i].area==TRUE)
1498                                         cway->flags|=W_AREA;
1499                                 if (wayinfo[i].oneway==TRUE)
1500                                         cway->flags|=W_ONEWAY;
1501                                 cway->type=wayinfo[i].type;
1502                                 if (cway->data->speed==0)
1503                                         cway->data->speed=wayinfo[i].defspeed;
1504                                 break;
1505                         }
1506                 }
1507
1508                 v=g_hash_table_lookup(osm_way_tags, "name");
1509                 if (v) {
1510                         cway->data->name=g_utf8_normalize(v, -1, G_NORMALIZE_ALL_COMPOSE);
1511                         /* Try to find other language names */
1512                         cway->data->names=g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free);
1513                         g_hash_table_foreach(osm_way_tags, find_nls_names, cway->data->names);
1514                         if (g_hash_table_size(cway->data->names)==0) {
1515                                 g_hash_table_destroy(cway->data->names);
1516                                 cway->data->names=NULL;
1517                         }
1518                 }
1519
1520                 v=g_hash_table_lookup(osm_way_tags, "ref");
1521                 if (v)
1522                         cway->data->ref=g_strdup(v);
1523                 v=g_hash_table_lookup(osm_way_tags, "int_ref");
1524                 if (v)
1525                         cway->data->int_ref=g_strdup(v);
1526
1527                 v=g_hash_table_lookup(osm_way_tags, "postal_code");
1528                 if (v)
1529                         cway->data->postal_code=g_strdup(v);
1530
1531                 v=g_hash_table_lookup(osm_way_tags, "oneway");
1532                 if (v) {
1533                         cway->flags|=W_ONEWAY;
1534                         if (strcmp(v, "-1")==0)
1535                                 cway->nodes=g_slist_reverse(cway->nodes);
1536                 }
1537
1538                 v=g_hash_table_lookup(osm_way_tags, "noexit");
1539                 if (v)
1540                         cway->flags|=W_NOEXIT;
1541
1542                 v=g_hash_table_lookup(osm_way_tags, "speedlimit");
1543                 if (v)
1544                         cway->data->speed=atoi(v);
1545                 v=g_hash_table_lookup(osm_way_tags, "maxspeed");
1546                 if (v)
1547                         cway->data->speed=atoi(v);
1548
1549                 v=g_hash_table_lookup(osm_way_tags, "layer");
1550                 if (v)
1551                         cway->data->layer=atoi(v);
1552
1553                 v=g_hash_table_lookup(osm_way_tags, "junction");
1554                 if (v && strcasecmp(v,"roundabout")==0) {
1555                         cway->flags|=W_ROUNDABOUT;
1556                         cway->flags|=W_ONEWAY;
1557                 } else if (v && strcasecmp(v,"mini_roundabout")==0) {
1558                         cway->flags|=W_ROUNDABOUT;
1559                         cway->flags|=W_ONEWAY;
1560                 }
1561
1562                 /* XXX: Should check keys */
1563                 v=g_hash_table_lookup(osm_way_tags, "access");
1564                 if (v && (strcasecmp(v, "private")==0)) {
1565                         cway->flags|=W_NOACCESS;
1566                 }
1567
1568                 print_way(cway);
1569
1570                 v=g_hash_table_lookup(osm_way_tags, "is_in");
1571                 if (v) {
1572                         gchar **isin;
1573                         isin=g_strsplit(v, ",", 10);
1574                         g_hash_table_insert(osm_way_isin, GINT_TO_POINTER(cway->id), isin);
1575                 }
1576
1577                 if (cway->data && cway->data->name==NULL && cway->data->ref==NULL &&
1578                         cway->data->int_ref==NULL && cway->data->layer==0 && cway->data->speed==0)
1579                         osm_free_way_data(cway);
1580
1581                 if (cway->id!=0)
1582                         osm_way_add_to_list(cway);
1583                 else
1584                         osm_free_way(cway);
1585
1586                 cway=NULL;
1587                 g_hash_table_destroy(osm_way_tags);
1588         break;
1589         case IN_BOUND_TAG:
1590                 /* */
1591         break;
1592         case IN_OSM_TAG:
1593                 g_printf("\nPlanet loaded.\n");
1594         break;
1595         default:;
1596 }
1597 }
1598
1599 /************************************************************************/
1600
1601 static void
1602 storage_init(void)
1603 {
1604 osm_nodes=g_hash_table_new(g_direct_hash, g_direct_equal);
1605
1606 osm_place_country=g_hash_table_new(g_str_hash, g_str_equal);
1607 osm_place_city=g_hash_table_new(g_str_hash, g_str_equal);
1608 osm_place_suburb=g_hash_table_new(g_str_hash, g_str_equal);
1609 osm_place_village=g_hash_table_new(g_str_hash, g_str_equal);
1610 osm_place_region=g_hash_table_new(g_str_hash, g_str_equal);
1611 osm_node_isin=g_hash_table_new(g_direct_hash, g_direct_equal);
1612 osm_way_isin=g_hash_table_new(g_direct_hash, g_direct_equal);
1613 }
1614
1615 static void
1616 storage_free(void)
1617 {
1618 g_hash_table_destroy(osm_nodes);
1619
1620 g_hash_table_destroy(osm_place_country);
1621 g_hash_table_destroy(osm_place_city);
1622 g_hash_table_destroy(osm_place_suburb);
1623 g_hash_table_destroy(osm_place_village);
1624 g_hash_table_destroy(osm_place_region);
1625 g_hash_table_destroy(osm_node_isin);
1626 }
1627
1628 /************************************************************************/
1629
1630 static gint
1631 print_fail(const gchar *msg, gint ret)
1632 {
1633 g_printerr("ERROR: %s\n", msg);
1634 return ret;
1635 }
1636
1637 /************************************************************************/
1638
1639 static void
1640 print_memory_usage(void)
1641 {
1642 g_print("Memory usage per item:\n");
1643 g_printf("Node  size: %d\n", (gint)sizeof(node));
1644 g_printf("NodeD size: %d\n", (gint)sizeof(node_data));
1645 g_printf("Way   size: %d\n", (gint)sizeof(way));
1646 g_printf("WayD  size: %d\n", (gint)sizeof(way_data));
1647 }
1648
1649 /************************************************************************
1650  * Public inteface
1651  ************************************************************************/
1652
1653 void
1654 osm_planet_parser_init(void)
1655 {
1656 xp=XML_ParserCreate(NULL);
1657 XML_SetElementHandler(xp, _osm_tag_start, _osm_tag_end);
1658 storage_init();
1659 }
1660
1661 void
1662 osm_planet_parser_deinit(void)
1663 {
1664 XML_ParserFree(xp);
1665 storage_free();
1666 }
1667
1668 gboolean
1669 osm_planet_parse_buffer(const gchar *buffer, size_t r)
1670 {
1671 if (XML_Parse(xp, buffer, r, r>0 ? 0:1) == XML_STATUS_ERROR) {
1672         g_printerr("Parse error at line %d:\n%s\n",
1673                 (gint)XML_GetCurrentLineNumber(xp),
1674                 XML_ErrorString(XML_GetErrorCode(xp)));
1675         return FALSE;
1676 }
1677 return TRUE;
1678 }
1679
1680 gboolean
1681 osm_planet_parse_file(gchar *pfile)
1682 {
1683 FILE *f;
1684 BZFILE *b;
1685 int bzerror;
1686 int r;
1687 gchar buffer[FILE_BUFFER];
1688 gboolean res=TRUE;
1689
1690 f=fopen(pfile, "r");
1691 if (!f) {
1692         perror("fopen failed\n");
1693         return FALSE;
1694 }
1695
1696 b=BZ2_bzReadOpen(&bzerror, f, 0, 0, NULL, 0);
1697 if (bzerror != BZ_OK) {
1698         g_printf("BZ2_bzReadOpen failed\n");
1699         BZ2_bzReadClose(&bzerror, b);
1700         return FALSE;
1701 }
1702
1703 do {
1704         r=BZ2_bzRead(&bzerror, b, buffer, FILE_BUFFER);
1705         if ((bzerror!=BZ_STREAM_END) && (bzerror!=BZ_OK)) {
1706                 res=FALSE;
1707                 break;
1708         }
1709         if (!osm_planet_parse_buffer(buffer, r)) {
1710                 res=FALSE;
1711                 break;
1712         }
1713 } while (bzerror==BZ_OK);
1714
1715 BZ2_bzReadClose(&bzerror, b);
1716 fclose(f);
1717 return res;
1718 }
1719
1720 /**
1721  * Set up bounding box for import.
1722  *
1723  */
1724 void
1725 osm_import_set_bbox(gboolean use_bb, gdouble latmin, gdouble lonmin, gdouble latmax, gdouble lonmax)
1726 {
1727 use_bbox=use_bb;
1728 bbox.lat_min=latmin;
1729 bbox.lon_min=lonmin;
1730 bbox.lat_max=latmax;
1731 bbox.lon_max=lonmax;
1732 g_printf("Skipping data outside of box: %f,%f - %f,%f\n",
1733         bbox.lat_min, bbox.lon_min,     bbox.lat_max, bbox.lon_max);
1734 }
1735
1736 static void
1737 osm_print_import_stats(void)
1738 {
1739 g_printf("Total nodes %d, POIs: %d and Ways %d.\n",     node_cnt, noded_cnt, way_cnt);
1740 g_printf("Cities/Towns: %d\n", g_hash_table_size(osm_place_city));
1741 g_printf("Villages/Hamlets: %d\n", g_hash_table_size(osm_place_village));
1742 g_printf("Suburbs: %d\n", g_hash_table_size(osm_place_suburb));
1743 g_printf("Nodes: %d\n", g_hash_table_size(osm_nodes));
1744 }
1745
1746 /**
1747  * Simple helper to do all preparations and importing from planet -> database
1748  *
1749  */
1750 gboolean
1751 osm_import(const gchar *planet, const gchar *database)
1752 {
1753 if (db_connect(&db, database)!=TRUE) {
1754         g_printerr("Database open failed: %s", database);
1755         return FALSE;
1756 }
1757
1758 if (!osm_db_create(db)) {
1759         g_printerr("Failed to create osm tables or indexes\n");
1760         return FALSE;
1761 }
1762
1763 if (!osm_db_prepare(db)) {
1764         g_printerr("Failed to prepare sql statements\n");
1765         return FALSE;
1766 }
1767
1768 osm_planet_parser_init();
1769
1770 if (osm_planet_parse_file(planet)==FALSE) {
1771         g_printerr("Failed to parse file: %s\n", planet);
1772         return FALSE;
1773 }
1774
1775 osm_print_import_stats();
1776
1777 osm_planet_save_all_nodes();
1778 osm_planet_save_all_ways();
1779 osm_planet_parser_deinit();
1780 db_finalize();
1781 db_close(&db);
1782 g_printf("All done.\n");
1783 return TRUE;
1784 }
1785
1786 static gpointer
1787 osm_import_thread(gpointer user_data)
1788 {
1789 gboolean r;
1790 osm_import_data_req *req=(osm_import_data_req *)user_data;
1791
1792 g_assert(req);
1793 g_assert(req->planet);
1794 g_assert(req->db);
1795
1796 osm_import_progress_cb=req->progress_cb!=NULL ? req->progress_cb : NULL;
1797
1798 r=osm_import(req->planet, req->db);
1799 g_debug("OSM import result: %d", r);
1800
1801 g_free(req->planet);
1802 g_free(req->db);
1803
1804 if (req->done_cb!=NULL)
1805         g_idle_add(req->done_cb, GINT_TO_POINTER(r==TRUE ? 0 : 1));
1806
1807 return r==TRUE ? 0 : 1;
1808 }
1809
1810 /**
1811  * Helper to start an import in the background using a thread.
1812  *
1813  * Two callback can be given, one for progress feedback and one when the operation is done.
1814  * Done callback must call the join function.
1815  * Only one import thread can run at a time.
1816  *
1817  */
1818 gboolean
1819 osm_import_bg(const gchar *planet, const gchar *database, GSourceFunc progress_cb, GSourceFunc done_cb)
1820 {
1821 GError *error=NULL;
1822
1823 g_return_val_if_fail(import_thread==NULL, FALSE);
1824
1825 osm_import_req.planet=g_strdup(planet);
1826 osm_import_req.db=g_strdup(database);
1827 osm_import_req.progress_cb=progress_cb;
1828 osm_import_req.done_cb=done_cb;
1829
1830 import_thread=g_thread_create(osm_import_thread, &osm_import_req, TRUE, &error);
1831 if (import_thread==NULL) {
1832         g_free(osm_import_req.planet);
1833         g_free(osm_import_req.db);
1834         g_printerr("Import thread creation failed.\n");
1835         return FALSE;
1836 }
1837 if (osm_import_progress_cb!=NULL)
1838         import_sid=g_timeout_add(1000, osm_import_progress_cb, NULL);
1839 return TRUE;
1840 }
1841
1842 gint
1843 osm_import_join_bg(void)
1844 {
1845 gint ret;
1846 g_assert(import_thread!=NULL);
1847
1848 if (import_sid!=0)
1849         g_source_remove(import_sid);
1850 ret=g_thread_join(import_thread);
1851 import_thread=NULL;
1852 return ret;
1853 }