err.no Git - mapper/blob - src/osm-db-import.c

   1 /*
   2  * This file is part of mapper
   3  *
   4  * Copyright (C) 2007 Kaj-Michael Lang
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along
  17  * with this program; if not, write to the Free Software Foundation, Inc.,
  18  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  19  */
  20
  21 /*
  22  * Routines to read OSM planet XML file and store it in a sqlite3 database.
  23  * Reads in all nodes (if used, skips nodes outside bounding box)
  24  * Special POI nodes are stored in POI table.
  25  * Place POI nodes are stored in place table.
  26  *
  27  * Ways are read in and their data (name, type, etc) are stored
  28  * in way, way_name and way_ref tables.
  29  *
  30  * Nodes used by they ways are stored in way_n2n table.
  31  *
  32  */
  33
  34 #include <stdio.h>
  35 #include <unistd.h>
  36 #include <string.h>
  37 #include <strings.h>
  38 #include <sys/types.h>
  39 #include <sys/stat.h>
  40 #include <fcntl.h>
  41 #include <math.h>
  42 #include <glib.h>
  43 #include <glib/gstdio.h>
  44 #include <sqlite3.h>
  45 #include <expat.h>
  46 #include <bzlib.h>
  47
  48 #include "osm.h"
  49 #include "latlon.h"
  50 #include "db.h"
  51 #include "osm-db-import.h"
  52
  53 #if 0
  54 #define VERBOSE
  55 #endif
  56 /* #define VERBOSE_KEYS */
  57
  58
  59 /* Use g_convert to transliterate names.. my iconv seems to be fucked so this didn't work... */
  60 /* #define TRANSLIT_NAMES */
  61
  62 #define FILE_BUFFER 65535
  63
  64 static guint node_cnt=0;
  65 static guint node_skip_cnt=0;
  66 static guint noded_cnt=0;
  67 static guint way_cnt=0;
  68 static guint way_names=0;
  69 static guint way_refs=0;
  70
  71 static guint dbnode_cnt=0;
  72 static guint dbnoded_cnt=0;
  73 static guint dbway_cnt=0;
  74
  75 static gboolean is_update=FALSE;
  76 static XML_Parser xp;
  77
  78
  79 /* XML tag IDs */
  80 typedef enum {
  81         START,
  82         IN_OSM_TAG,
  83         IN_NODE_TAG,
  84         IN_WNODE_TAG,
  85         IN_WAY_TAG,
  86         IN_KEY_TAG,
  87         IN_BOUND_TAG,
  88         IN_RELATION_TAG,
  89         IN_MEMBER_TAG,
  90         END,
  91         ERROR
  92 } tag_state_t;
  93
  94 /* Parent tag type */
  95 typedef enum {
  96         IS_NONE,
  97         IS_NODE,
  98         IS_WAY,
  99         IS_RELATION
 100 } tag_parent_t;
 101
 102 /* Node types table */
 103 /* XXX: Add support for parent category */
 104 struct _nodeinfo {
 105         gchar *k, *v;
 106         node_type_t type;
 107 } nodeinfo[] = {
 108         { "amenity", "fuel",            NODE_AMENITY_FUEL },
 109         { "amenity", "parking",         NODE_AMENITY_PARKING },
 110
 111         { "amenity", "pub",                     NODE_AMENITY_PUB },
 112         { "amenity", "nightclub",       NODE_AMENITY_NIGHTCLUB },
 113         { "amenity", "biergarten",      NODE_AMENITY_PUB },
 114         { "amenity", "cafe",            NODE_AMENITY_CAFE },
 115         { "amenity", "fast_food",       NODE_AMENITY_FOOD },
 116         { "amenity", "restaurant",      NODE_AMENITY_FOOD },
 117
 118         { "amenity", "telephone",       NODE_AMENITY_TELEPHONE },
 119         { "amenity", "toilets",         NODE_AMENITY_WC },
 120
 121         { "amenity", "hospital",        NODE_AMENITY_HOSPITAL },
 122         { "amenity", "doctors",         NODE_AMENITY_HOSPITAL },
 123         { "amenity", "pharmacy",        NODE_AMENITY_PHARMACY },
 124
 125         { "amenity", "post_office",     NODE_AMENITY_POST },
 126         { "amenity", "post_box",        NODE_AMENITY_POST_BOX },
 127
 128         { "amenity", "cinema",          NODE_AMENITY_CINEMA },
 129         { "amenity", "theatre",         NODE_AMENITY_THEATRE },
 130
 131         { "amenity", "atm",                     NODE_AMENITY_ATM },
 132         { "amenity", "bank",            NODE_AMENITY_BANK },
 133
 134         { "amenity", "police",          NODE_AMENITY_POLICE },
 135         { "amenity", "speed_trap",      NODE_AMENITY_SPEEDCAM },
 136         { "amenity", "speed_camera",    NODE_AMENITY_SPEEDCAM },
 137         { "amenity", "speed camera",    NODE_AMENITY_SPEEDCAM },
 138
 139         { "amenity", "place_of_worship",NODE_AMENITY_POW },
 140
 141         { "amenity", "school",          NODE_AMENITY_SCHOOL },
 142         { "amenity", "college",         NODE_AMENITY_COLLEGE },
 143         { "amenity", "university",      NODE_AMENITY_COLLEGE },
 144
 145         { "amenity", "library", NODE_AMENITY_LIBRARY },
 146         { "amenity", "townhall",        NODE_AMENITY_TOWNHALL },
 147
 148         { "amenity", "supermarket",     NODE_AMENITY_SHOP },
 149         { "amenity", "shopping_centre", NODE_AMENITY_SHOP },
 150         { "amenity", "shop",            NODE_AMENITY_SHOP },
 151         { "amenity", "shops",           NODE_AMENITY_SHOP },
 152         { "amenity", "shopping",        NODE_AMENITY_SHOP },
 153         { "amenity", "shopping_mall",NODE_AMENITY_SHOP },
 154         { "amenity", "cycle_shop",      NODE_AMENITY_SHOP },
 155         { "amenity", "bike_shop",       NODE_AMENITY_SHOP },
 156         { "amenity", "coffee_shop",     NODE_AMENITY_SHOP },
 157         { "amenity", "indoor_shopping_centre",  NODE_AMENITY_SHOP },
 158         { "amenity", "farm_shop",       NODE_AMENITY_SHOP },
 159         { "amenity", "tea_shop",        NODE_AMENITY_SHOP },
 160
 161         /* Shops */
 162         { "shop",        "supermarket", NODE_AMENITY_SHOP },
 163         { "shop",        "bakery",              NODE_AMENITY_SHOP },
 164         { "shop",        "alcohol",             NODE_AMENITY_SHOP },
 165         { "shop",        "butcher",             NODE_AMENITY_SHOP },
 166         { "shop",        "flowers",             NODE_AMENITY_SHOP },
 167         { "shop",        "clothing",    NODE_AMENITY_SHOP },
 168         { "shop",        "souvenir",    NODE_AMENITY_SHOP },
 169         { "shop",        "bicycles",    NODE_AMENITY_SHOP },
 170         { "shop",        "grocers",             NODE_AMENITY_SHOP },
 171         { "shop",        "newsagents",  NODE_AMENITY_SHOP },
 172         { "shop",        "convenience", NODE_AMENITY_SHOP },
 173         { "shop",        "bakers",              NODE_AMENITY_SHOP },
 174         { "shop",        "garden_centre",NODE_AMENITY_SHOP },
 175         { "shop",        "photography", NODE_AMENITY_SHOP },
 176         { "shop",        "general_store",NODE_AMENITY_SHOP },
 177         { "shop",        "food",                NODE_AMENITY_SHOP },
 178         { "shop",        "drinks",              NODE_AMENITY_SHOP },
 179         { "shop",        "pharmacy",    NODE_AMENITY_PHARMACY },
 180
 181         /* Sport */
 182         { "sport"  , "swimming",        NODE_SPORT_SWIMMING },
 183         { "sport"  , "golf",            NODE_SPORT_GOLF },
 184         { "sport"  , "tennis",          NODE_SPORT_TENNIS },
 185         { "sport"  , "football",        NODE_SPORT_FOOTBALL },
 186         { "sport"  , "soccer",          NODE_SPORT_SOCCER },
 187         { "sport"  , "baskteball",      NODE_SPORT_BASKETBALL },
 188         { "sport"  , "rugby",           NODE_SPORT_RUGBY },
 189         { "sport"  , "skating",         NODE_SPORT_SKATING },
 190         { "sport"  , "hockey",          NODE_SPORT_HOCKEY },
 191         { "sport"  , "skateboard",      NODE_SPORT_SKATEBOARD },
 192         { "sport"  , "bowling",         NODE_SPORT_BOWLING },
 193         { "sport"  , "10pin",           NODE_SPORT_BOWLING },
 194         { "sport"  , "motor",           NODE_SPORT_MOTOR },
 195         { "sport"  , "shooting_range",NODE_SPORT_SHOOTING },
 196         { "sport"  , "paintball",       NODE_SPORT_PAINTBALL },
 197         { "sport"  , "horse_racing",NODE_SPORT_HORSES },
 198         { "sport"  , "horse",           NODE_SPORT_HORSES },
 199         { "sport"  , "horses",          NODE_SPORT_HORSES },
 200         { "sport"  , "dog_racing",      NODE_SPORT_DOG },
 201         { "sport"  , "pelota",          NODE_SPORT_PELOTA },
 202         { "sport"  , "racquet",         NODE_SPORT_RACQUET },
 203         { "sport"  , "equestrian",      NODE_SPORT_HORSES },
 204         { "sport"  , "baseball",        NODE_SPORT_BASEBALL },
 205         { "sport"  , "cricket",         NODE_SPORT_CRICKET },
 206         { "sport"  , "croquet",         NODE_SPORT_CROQUET },
 207         { "sport"  , "cycling",         NODE_SPORT_CYCLING },
 208         { "sport"  , "bowls",           NODE_SPORT_BOWLS },
 209         { "sport"  , "athletics",       NODE_SPORT_ATHLETICS },
 210         { "sport"  , "gymnastics",      NODE_SPORT_GYMNASTICS },
 211         { "sport"  , "multi",           NODE_SPORT_OTHER },
 212         { "leisure", "sport_centre",NODE_SPORT_CENTER },
 213
 214         /* Tourism */
 215         { "tourism", "information",     NODE_TOURISM_INFO },
 216         { "tourism", "camp_site",       NODE_TOURISM_CAMP_SITE },
 217         { "tourism", "caravan_site",NODE_TOURISM_CARAVAN_SITE },
 218         { "tourism", "picnic_site",     NODE_TOURISM_PICNIC_SITE },
 219         { "tourism", "theme_park",      NODE_TOURISM_THEME_PARK },
 220         { "tourism", "hotel",           NODE_TOURISM_HOTEL },
 221         { "tourism", "motel",           NODE_TOURISM_MOTEL },
 222         { "tourism", "hostel",          NODE_TOURISM_HOSTEL },
 223         { "tourism", "attraction",      NODE_TOURISM_ATTRACTION },
 224         { "tourism", "zoo",                     NODE_TOURISM_ATTRACTION },
 225
 226         { "historic", "ruins",          NODE_TOURISM_ATTRACTION },
 227         { "historic", "monument",       NODE_TOURISM_ATTRACTION },
 228         { "historic", "memorial",       NODE_TOURISM_ATTRACTION },
 229         { "historic", "museum",         NODE_HISTORIC_MUSEUM },
 230         { "historic", "castle",         NODE_HISTORIC_CASTLE },
 231
 232         { "railway", "station",         NODE_RAILWAY_STATION },
 233         { "railway", "halt",            NODE_RAILWAY_HALT },
 234
 235         { "aeroway", "terminal",        NODE_AIRPORT_TERMINAL },
 236
 237         /* Places */
 238         { "place", "city",                      NODE_PLACE_CITY },
 239         { "place", "town",                      NODE_PLACE_TOWN },
 240         { "place", "village",           NODE_PLACE_VILLAGE },
 241         { "place", "hamlet",            NODE_PLACE_HAMLET },
 242         { "place", "locality",          NODE_PLACE_LOCALITY },
 243         { "place", "suburb",            NODE_PLACE_SUBURB },
 244         { "place", "island",            NODE_PLACE_ISLAND },
 245
 246         { "highway", "traffic_signals", NODE_TRAFFIC_SIGNALS },
 247         { "highway", "motorway_junction",       NODE_JUNCTION },
 248         { "highway", "services",        NODE_AMENITY_PARKING },
 249         { "highway", "toll_booth",      NODE_TOLLBOOTH },
 250         { "highway", "gate",            NODE_GATE },
 251
 252         { NULL, NULL, NODE_PLAIN }
 253 };
 254
 255 /* Array to get id number and defaults for ways of different types */
 256 struct _wayinfo {
 257         gchar *k, *v;
 258         guint defspeed;
 259         way_type_t type;
 260         gboolean oneway, link, area, car, foot;
 261 } wayinfo[] = {
 262         { "highway", "motorway",120,WAY_MOTORWAY,               TRUE, FALSE, FALSE, TRUE, FALSE },
 263         { "highway", "motorway_link",120,WAY_MOTORWAY,  TRUE, TRUE, FALSE, TRUE, FALSE },
 264         { "highway", "trunk",100,WAY_TRUNK,                             FALSE, FALSE, FALSE, TRUE, FALSE },
 265         { "highway", "trunk_link",100,WAY_TRUNK,                FALSE, TRUE, FALSE, TRUE, FALSE },
 266         { "highway", "primary",80,WAY_PRIMARY,                  FALSE, FALSE, FALSE, TRUE, TRUE },
 267         { "highway", "primary_link",60,WAY_PRIMARY,             FALSE, TRUE, FALSE, TRUE, TRUE },
 268         { "highway", "secondary",80,WAY_SECONDARY,              FALSE, FALSE, FALSE, TRUE, TRUE },
 269         { "highway", "secondary_link",60,WAY_SECONDARY, FALSE, TRUE, FALSE, TRUE, TRUE },
 270         { "highway", "tertiary",60,WAY_TERTIARY,                FALSE, FALSE, FALSE, TRUE, TRUE },
 271         { "highway", "unclassified",50,WAY_UNCLASSIFIED,        FALSE, FALSE, FALSE, TRUE, TRUE },
 272         { "highway", "byway",40,WAY_UNCLASSIFIED,       FALSE, FALSE, FALSE, TRUE, TRUE },
 273         { "highway", "residential",40,WAY_RESIDENTIAL,  FALSE, FALSE, FALSE, TRUE, TRUE },
 274         { "highway", "service",20,WAY_SERVICE,                  FALSE, FALSE, FALSE, TRUE, TRUE },
 275         { "highway", "track",20,WAY_TRACK,                              FALSE, FALSE, FALSE, TRUE, TRUE },
 276         { "highway", "unsurfaced",60,WAY_TRACK,                 FALSE, FALSE, FALSE, TRUE, TRUE },
 277         { "highway", "minor",60,WAY_TRACK,                      FALSE, FALSE, FALSE, TRUE, TRUE },
 278         { "highway", "pedestrian",20,WAY_FOOTWAY,               FALSE, FALSE, FALSE, FALSE, TRUE },
 279         { "highway", "footway",1,WAY_FOOTWAY,                   FALSE, FALSE, FALSE, FALSE, TRUE },
 280         { "highway", "steps",0,WAY_FOOTWAY,                             FALSE, FALSE, FALSE, FALSE, TRUE},
 281         { "highway", "bridleway",10,WAY_FOOTWAY,                FALSE, FALSE, FALSE, FALSE, TRUE },
 282         { "highway", "cycleway",10,WAY_CYCLEWAY,                FALSE, FALSE, FALSE, FALSE, TRUE },
 283         { "railway", "rail",0,WAY_RAIL,                                 FALSE, FALSE, FALSE, FALSE, FALSE },
 284         { "aeroway", "runway",0,WAY_RUNWAY,                             FALSE, FALSE, FALSE, FALSE, FALSE },
 285         { "aeroway", "taxiway",0,WAY_TAXIWAY,                   FALSE, FALSE, FALSE, FALSE, FALSE },
 286         { "natural", "water",0,WAY_WATER,                               FALSE, FALSE, TRUE, FALSE, FALSE },
 287         { "waterway", "river",0,WAY_WATER,                              FALSE, FALSE, FALSE, FALSE, FALSE },
 288         { "waterway", "canal",0,WAY_WATER,                              FALSE, FALSE, FALSE, FALSE, FALSE },
 289         { "waterway", "stream",0,WAY_WATER,                             FALSE, FALSE, FALSE, FALSE, FALSE },
 290         { "building", "*",0,WAY_UNWAYED,                                FALSE, FALSE, TRUE, FALSE, FALSE },
 291         { NULL, NULL, 0, WAY_UNWAYED, FALSE, FALSE, FALSE, FALSE, FALSE }
 292 };
 293
 294 static sqlite3 *db;
 295 tag_parent_t tag_parent=IS_NONE;
 296
 297 static GHashTable *osm_nodes;
 298 static GHashTable *osm_node_tags;
 299 static GHashTable *osm_way_tags;
 300 static GSList *osm_ways;
 301 static GSList *osm_poi;
 302
 303 static GHashTable *osm_place_country;
 304 static GHashTable *osm_place_region;
 305 static GHashTable *osm_place_city;
 306 static GHashTable *osm_place_suburb;
 307 static GHashTable *osm_place_village;
 308 static GHashTable *osm_node_isin;
 309 static GHashTable *osm_way_isin;
 310
 311 static node *cnode=NULL;
 312 static way *cway=NULL;
 313
 314 struct sql_stmt {
 315         sqlite3_stmt *insert_poi;
 316         sqlite3_stmt *delete_osm_poi;
 317
 318         sqlite3_stmt *insert_node;
 319         sqlite3_stmt *delete_nodes;
 320         sqlite3_stmt *select_node;
 321         sqlite3_stmt *update_node;
 322
 323         sqlite3_stmt *insert_way_data;
 324         sqlite3_stmt *insert_way_ref;
 325         sqlite3_stmt *insert_way_pc;
 326         sqlite3_stmt *insert_way_name;
 327         sqlite3_stmt *insert_way_names_nls;
 328         sqlite3_stmt *insert_way_n2n;
 329         sqlite3_stmt *delete_way;
 330         sqlite3_stmt *delete_way_n2n;
 331         sqlite3_stmt *delete_way_name;
 332         sqlite3_stmt *delete_way_names_nls;
 333         sqlite3_stmt *delete_way_ref;
 334         sqlite3_stmt *delete_way_pc;
 335
 336         sqlite3_stmt *insert_place;
 337         sqlite3_stmt *delete_place;
 338 };
 339 static struct sql_stmt sql;
 340
 341
 342 static struct map_bbox bbox;
 343 static gboolean use_bbox;
 344
 345 void osm_free_way_data(way *w);
 346 void print_way(way *w);
 347
 348 void db_prepare(sqlite3 *db);
 349 gboolean db_insert_node(node *n);
 350 guint32 osm_find_way_place(way *w, node_type_t nt);
 351
 352 /****************************************************/
 353 /* Functions */
 354 /****************************************************/
 355
 356 static void
 357 db_finalize(void)
 358 {
 359 sqlite3_finalize(sql.insert_poi);
 360 sqlite3_finalize(sql.delete_osm_poi);
 361
 362 sqlite3_finalize(sql.insert_node);
 363 sqlite3_finalize(sql.select_node);
 364 sqlite3_finalize(sql.delete_nodes);
 365 sqlite3_finalize(sql.update_node);
 366
 367 sqlite3_finalize(sql.insert_place);
 368 sqlite3_finalize(sql.delete_place);
 369
 370 sqlite3_finalize(sql.delete_way);
 371 sqlite3_finalize(sql.insert_way_data);
 372
 373 sqlite3_finalize(sql.delete_way_name);
 374 sqlite3_finalize(sql.insert_way_name);
 375
 376 sqlite3_finalize(sql.delete_way_n2n);
 377 sqlite3_finalize(sql.insert_way_n2n);
 378
 379 sqlite3_finalize(sql.delete_way_pc);
 380 sqlite3_finalize(sql.insert_way_pc);
 381
 382 sqlite3_finalize(sql.delete_way_names_nls);
 383 sqlite3_finalize(sql.insert_way_names_nls);
 384 }
 385
 386 void
 387 db_prepare(sqlite3 *db)
 388 {
 389 /* Way nodes */
 390 sqlite3_prepare_v2(db, "insert or replace into nodes (nid,ilat,ilon,rlat,rlon,l,f) values (?,?,?,?,?,0,?)", -1, &sql.insert_node, NULL);
 391 sqlite3_prepare_v2(db, "select ilat,ilon,l from nodes where nid=?", -1, &sql.select_node, NULL);
 392 sqlite3_prepare_v2(db, "delete from nodes", -1, &sql.delete_nodes, NULL);
 393 sqlite3_prepare_v2(db, "update nodes set l=l+1 where nid=?", -1, &sql.update_node, NULL);
 394
 395 /* Places */
 396 sqlite3_prepare_v2(db, "insert or replace into places (nid,type,name,isin_c,isin_p) values (?, ?, ?, ?, ?)", -1, &sql.insert_place, NULL);
 397 sqlite3_prepare_v2(db, "delete from places", -1, &sql.delete_place, NULL);
 398
 399 /* POI nodes */
 400 if (sqlite3_prepare_v2(db, "insert or replace into poi (osm_id, lat, lon, label, cat_id, public, source, priority, isin_c, isin_p, desc, url, postal_code) "
 401                                            " values (?, ?, ?, ?, ?, 1, 1, ?, ?, ?, ?, ?, ?)", -1, &sql.insert_poi, NULL)!=SQLITE_OK)
 402         g_printf("SQL: %s\n", sqlite3_errmsg(db));
 403
 404 sqlite3_prepare_v2(db, "delete from poi where osm_id>0 and source=1", -1, &sql.delete_osm_poi, NULL);
 405
 406 /* Ways */
 407 sqlite3_prepare_v2(db, "insert or replace into way (wid,nodes,type,flags,speed,isin_c,isin_p,lat,lon) values (?, ?, ?, ?, ?, ?, ?, ?, ?)", -1, &sql.insert_way_data, NULL);
 408 sqlite3_prepare_v2(db, "delete from way", -1, &sql.delete_way, NULL);
 409
 410 /* Way nodes */
 411 sqlite3_prepare_v2(db, "insert into way_n2n (wid,f,t) values (?,?,?)", -1, &sql.insert_way_n2n, NULL);
 412 sqlite3_prepare_v2(db, "delete from way_n2n where wid=?", -1, &sql.delete_way_n2n, NULL);
 413
 414 /* Way names */
 415 sqlite3_prepare_v2(db, "insert or replace into way_names (wid,name,norm) values (?, ?, ?)",  -1, &sql.insert_way_name, NULL);
 416 sqlite3_prepare_v2(db, "delete from way_names", -1, &sql.delete_way_name, NULL);
 417
 418 /* Way postal codes */
 419 sqlite3_prepare_v2(db, "insert or replace into way_pc (wid,pc) values (?, ?)",  -1, &sql.insert_way_pc, NULL);
 420 sqlite3_prepare_v2(db, "delete from way_pc", -1, &sql.delete_way_pc, NULL);
 421
 422 /* Other language names for ways */
 423 sqlite3_prepare_v2(db, "insert into way_names_nls (wid,lang,name, norm) values (?, ?, ?, ?)",  -1, &sql.insert_way_names_nls, NULL);
 424 sqlite3_prepare_v2(db, "delete from way_names_nls where wid=?", -1, &sql.delete_way_names_nls, NULL);
 425
 426 /* Way ref and int_ref */
 427 sqlite3_prepare_v2(db, "insert or replace into way_ref (rid,ref,int_ref) values (?, ?, ?)", -1, &sql.insert_way_ref, NULL);
 428 sqlite3_prepare_v2(db, "delete from way_ref", -1, &sql.delete_way_ref, NULL);
 429 }
 430
 431 /********************************************************************/
 432
 433 static void
 434 print_way(way *w)
 435 {
 436 #ifdef VERBOSE
 437 g_assert(w);
 438 g_printf("Way #%d(N:%d T:%d S:%d IS: %d/%d): %s [%s:%s:%s]\n",
 439                 w->id,
 440                 g_slist_length(w->nodes),
 441                 w->type,
 442                 w->data ? w->data->speed : 0,
 443                 w->data ? w->data->isin_c : -1,
 444                 w->data ? w->data->isin_p : -1,
 445                 w->data ? w->data->name ? w->data->name : "" : "",
 446                 w->flags & W_ONEWAY ? "-" : "=",
 447                 w->flags & W_ROUNDABOUT ? "O" : "-",
 448                 w->flags & W_LINK ? "|" : " ");
 449 #endif
 450 }
 451
 452 static void
 453 print_node(node *n)
 454 {
 455 #ifdef VERBOSE
 456 g_assert(n);
 457 g_printf("Node #%d: T:%d IS: %d/%d [%s]\n",
 458         n->id,
 459         n->type,
 460         n->data ? n->data->isin_c : -1,
 461         n->data ? n->data->isin_p : -1,
 462         n->data ? n->data->name : "");
 463 #endif
 464 }
 465
 466 /********************************************************************/
 467
 468 static gboolean
 469 db_insert_node(node *n)
 470 {
 471 gint32 lat, lon;
 472
 473 g_assert(n);
 474
 475 lat=lat2mp_int(n->lat);
 476 lon=lon2mp_int(n->lon);
 477
 478 sqlite3_bind_int(sql.insert_node, 1, n->id);
 479
 480 /* Projected and integerized lat/lot */
 481 sqlite3_bind_int(sql.insert_node, 2, lat);
 482 sqlite3_bind_int(sql.insert_node, 3, lon);
 483 /* Original */
 484 sqlite3_bind_double(sql.insert_node, 4, n->lat);
 485 sqlite3_bind_double(sql.insert_node, 5, n->lon);
 486 sqlite3_bind_int(sql.insert_node, 6, n->type);
 487
 488 db_exec(db, sql.insert_node);
 489
 490 return TRUE;
 491 }
 492
 493 static gboolean
 494 db_insert_place(node *n)
 495 {
 496 g_assert(n);
 497 if (!n->data)
 498         return FALSE;
 499 if (!n->data->name)
 500         return FALSE;
 501 sqlite3_bind_int(sql.insert_place, 1, n->id);
 502 sqlite3_bind_int(sql.insert_place, 2, n->type);
 503 sqlite3_bind_text(sql.insert_place, 3, n->data->name, -1, SQLITE_TRANSIENT);
 504 sqlite3_bind_int(sql.insert_place, 4, n->data->isin_p);
 505 sqlite3_bind_int(sql.insert_place, 5, n->data->isin_c);
 506
 507 return db_exec(db,sql.insert_place);
 508 }
 509
 510 static gboolean
 511 db_insert_poi(node *n)
 512 {
 513 g_assert(n);
 514 sqlite3_bind_int(sql.insert_poi, 1, n->id);
 515 sqlite3_bind_double(sql.insert_poi, 2, n->lat);
 516 sqlite3_bind_double(sql.insert_poi, 3, n->lon);
 517 if (n->data->name)
 518         sqlite3_bind_text(sql.insert_poi, 4, n->data->name, -1, SQLITE_TRANSIENT);
 519 else
 520         sqlite3_bind_text(sql.insert_poi, 4, "", -1, SQLITE_TRANSIENT);
 521 sqlite3_bind_int(sql.insert_poi, 5, n->type);
 522 sqlite3_bind_int(sql.insert_poi, 6, n->type/100);
 523 sqlite3_bind_int(sql.insert_poi, 7, n->data->isin_c);
 524 sqlite3_bind_int(sql.insert_poi, 8, n->data->isin_p);
 525
 526 if (n->data->desc)
 527         sqlite3_bind_text(sql.insert_poi, 9, n->data->desc, -1, SQLITE_TRANSIENT);
 528 if (n->data->url)
 529         sqlite3_bind_text(sql.insert_poi, 10, n->data->url, -1, SQLITE_TRANSIENT);
 530 if (n->data->postal_code)
 531         sqlite3_bind_text(sql.insert_poi, 11, n->data->postal_code, -1, SQLITE_TRANSIENT);
 532
 533 return db_exec(db,sql.insert_poi);
 534 }
 535
 536 /**
 537  * Update node usage count
 538  */
 539 static gboolean
 540 db_update_node_links(node *n)
 541 {
 542 g_assert(n);
 543 sqlite3_bind_int(sql.update_node, 1, n->id);
 544
 545 return db_exec(db,sql.update_node);
 546 }
 547
 548 /**
 549  * Insert way,node1,node2 triplet
 550  */
 551 static gboolean
 552 db_insert_way_n2n(way *w, node *nf, node *nt)
 553 {
 554 if (!w) {
 555         g_printf("NULL WAY\n");
 556         return FALSE;
 557 }
 558
 559 if (!nf) {
 560         g_printf("NULL NODE 1\n");
 561         return FALSE;
 562 }
 563
 564 if (!nt) {
 565         g_printf("NULL NODE 2\n");
 566         return FALSE;
 567 }
 568
 569 sqlite3_bind_int(sql.insert_way_n2n, 1, w->id);
 570 sqlite3_bind_int(sql.insert_way_n2n, 2, nf->id);
 571 sqlite3_bind_int(sql.insert_way_n2n, 3, nt->id);
 572
 573 #ifdef VERBOSE_N2N
 574 g_printf("%d [%d - %d]\n", w->id, nf->id, nt->id);
 575 #endif
 576
 577 db_exec(db,sql.insert_way_n2n);
 578 db_update_node_links(nf);
 579 db_update_node_links(nt);
 580 return TRUE;
 581 }
 582
 583 /**
 584  * Insert way ref and int_ref
 585  */
 586 static void
 587 db_insert_way_ref(way *w)
 588 {
 589 if (!w->data)
 590         return;
 591
 592 if (!w->data->ref && !w->data->int_ref)
 593         return;
 594
 595 way_refs++;
 596
 597 sqlite3_bind_int(sql.insert_way_ref, 1, w->id);
 598 if (w->data->ref)
 599         sqlite3_bind_text(sql.insert_way_ref, 2, w->data->ref, -1, SQLITE_TRANSIENT);
 600 if (w->data->int_ref)
 601         sqlite3_bind_text(sql.insert_way_ref, 3, w->data->int_ref, -1, SQLITE_TRANSIENT);
 602
 603 db_exec(db,sql.insert_way_ref);
 604 }
 605
 606 /**
 607  * Insert way name
 608  */
 609 static void
 610 db_insert_way_name(way *w)
 611 {
 612 gchar *norm;
 613
 614 if (!w->data)
 615         return;
 616 if (!w->data->name)
 617         return;
 618
 619 way_names++;
 620
 621 sqlite3_bind_int(sql.insert_way_name, 1, w->id);
 622 sqlite3_bind_text(sql.insert_way_name, 2, w->data->name, -1, SQLITE_TRANSIENT);
 623
 624 #ifdef TRANSLIT_NAMES
 625 norm=g_convert(w->data->name, -1, "ASCII//TRANSLIT//IGNORE", "utf8", NULL, NULL, NULL);
 626 if (norm && strcmp(w->data->name, norm)!=0) {
 627         sqlite3_bind_text(sql.insert_way_name, 3, norm, -1, SQLITE_TRANSIENT);
 628 }
 629 if (norm)
 630         g_free(norm);
 631 #endif
 632
 633 db_exec(db,sql.insert_way_name);
 634 }
 635
 636 static void
 637 db_delete_way_names_nls(way *w)
 638 {
 639 sqlite3_bind_int(sql.delete_way_names_nls, 1, w->id);
 640 db_exec(db,sql.delete_way_names_nls);
 641 }
 642
 643 static void
 644 db_insert_way_pc(way *w)
 645 {
 646 if (!w->data)
 647         return;
 648 if (!w->data->postal_code)
 649         return;
 650
 651 sqlite3_bind_int(sql.insert_way_pc, 1, w->id);
 652 sqlite3_bind_text(sql.insert_way_pc, 2, w->data->postal_code, -1, SQLITE_TRANSIENT);
 653
 654 db_exec(db,sql.insert_way_pc);
 655 }
 656
 657 static void
 658 db_delete_way_pc(way *w)
 659 {
 660 sqlite3_bind_int(sql.delete_way_pc, 1, w->id);
 661 db_exec(db,sql.delete_way_pc);
 662 }
 663
 664 static void
 665 db_insert_way_names_nls_cb(gpointer key, gpointer value, gpointer user_data)
 666 {
 667 gchar *norm;
 668
 669 way *w=(way *)user_data;
 670
 671 sqlite3_bind_int(sql.insert_way_names_nls, 1, w->id);
 672 sqlite3_bind_text(sql.insert_way_names_nls, 2, (gchar *)key, -1, SQLITE_TRANSIENT);
 673 sqlite3_bind_text(sql.insert_way_names_nls, 3, (gchar *)value, -1, SQLITE_TRANSIENT);
 674 #ifdef TRANSLIT_NAMES
 675 norm=g_convert((gchar *value), -1, "ASCII//TRANSLIT//IGNORE", "utf8", NULL, NULL, NULL);
 676 if (norm && strcmp((gchar *)value, norm)!=0) {
 677         sqlite3_bind_text(sql.insert_way_names_nls, 4, norm, -1, SQLITE_TRANSIENT);
 678 }
 679 if (norm)
 680         g_free(norm);
 681 #endif
 682 db_exec(db,sql.insert_way_names_nls);
 683 }
 684
 685 static void
 686 db_insert_way_names_nls(way *w)
 687 {
 688 if (!w->data)
 689         return;
 690 if (!w->data->names)
 691         return;
 692
 693 g_hash_table_foreach(w->data->names, db_insert_way_names_nls_cb, w);
 694 }
 695
 696 /**
 697  * Insert all data for the given way
 698  * - name
 699  * - ref
 700  * - nodes
 701  *
 702  */
 703 static gboolean
 704 db_insert_way(way *w)
 705 {
 706 GSList *iter;
 707 guint ncnt;
 708 node *wmn;
 709
 710 if (!w)
 711         return FALSE;
 712
 713 /* Skip things we don't use (yet) */
 714 if (w->type==WAY_UNWAYED || w->type>WAY_ROAD_END)
 715         return TRUE;
 716
 717 /* Insert nodes */
 718 for (iter=w->nodes; iter!=NULL; iter=iter->next) {
 719         if (!iter->next)
 720                 break;
 721         db_insert_way_n2n(w, iter->data, iter->next->data);
 722 }
 723
 724 if (w->id==0)
 725         return FALSE;
 726
 727 if (w->data) {
 728         w->data->isin_p=osm_find_way_place(w, NODE_PLACE_CITY);
 729         w->data->isin_c=osm_find_way_place(w, NODE_PLACE_COUNTRY);
 730 }
 731
 732 print_way(w);
 733
 734 /* Get middle node, use it as way location */
 735 ncnt=g_slist_length(w->nodes);
 736 if (ncnt>1)
 737         wmn=g_slist_nth_data(w->nodes, ncnt/2);
 738 else
 739         wmn=0;
 740
 741 sqlite3_bind_int(sql.insert_way_data, 1, w->id);
 742 sqlite3_bind_int(sql.insert_way_data, 2, w->ncnt);
 743 sqlite3_bind_int(sql.insert_way_data, 3, w->type);
 744 sqlite3_bind_int(sql.insert_way_data, 4, w->flags);
 745 if (w->data) {
 746         sqlite3_bind_int(sql.insert_way_data, 5, w->data->speed);
 747         sqlite3_bind_int(sql.insert_way_data, 6, w->data->isin_c);
 748         sqlite3_bind_int(sql.insert_way_data, 7, w->data->isin_p);
 749 }
 750 if (wmn) {
 751         sqlite3_bind_double(sql.insert_way_data, 8, wmn->lat);
 752         sqlite3_bind_double(sql.insert_way_data, 9, wmn->lon);
 753 } else {
 754         g_printerr("Failed to get way location node!\n");
 755 }
 756
 757 db_exec(db,sql.insert_way_data);
 758
 759 db_insert_way_ref(w);
 760 db_insert_way_name(w);
 761 db_insert_way_names_nls(w);
 762 db_insert_way_pc(w);
 763
 764 osm_free_way_data(w);
 765 return TRUE;
 766 }
 767
 768 /********************************************************************/
 769
 770 static gchar *
 771 get_attr_key_value(const gchar **p, gchar *key)
 772 {
 773 gchar **d;
 774
 775 d=p;
 776 while (*d!=NULL) {
 777         if (strncmp(*d, key, strlen(key))==0) {
 778                 d++;
 779                 return *d;
 780         }
 781         d++;
 782         d++;
 783 }
 784 return NULL;
 785 }
 786
 787 static tag_state_t
 788 check_tag(const gchar *tag)
 789 {
 790 if (strcmp(tag,"node")==0) return IN_NODE_TAG;
 791 else if (strcmp(tag,"nd")==0) return IN_WNODE_TAG;
 792 else if (strcmp(tag,"way")==0) return IN_WAY_TAG;
 793 else if (strcmp(tag,"tag")==0) return IN_KEY_TAG;
 794 else if (strcmp(tag,"osm")==0) return IN_OSM_TAG;
 795 else if (strcmp(tag,"bound")==0) return IN_BOUND_TAG;
 796 else if (strcmp(tag,"relation")==0) return IN_RELATION_TAG;
 797 else if (strcmp(tag,"member")==0) return IN_MEMBER_TAG;
 798 else return ERROR;
 799 }
 800
 801 static void
 802 find_nls_names(gpointer key, gpointer value, gpointer user_data)
 803 {
 804 gchar *k, *v;
 805 gchar *tmp;
 806 GHashTable *nls;
 807
 808 k=(gchar *)key;
 809 v=(gchar *)value;
 810 nls=(GHashTable *)user_data;
 811
 812 /* Check if it is a name key, return if not. */
 813 if (g_str_has_prefix(k, "name:")==FALSE)
 814         return;
 815
 816 tmp=g_strrstr(k, ":");
 817 if (!tmp)
 818         return;
 819 tmp++; /* skip : */
 820 if (*tmp==0)
 821         return;
 822 g_hash_table_insert(nls, g_strdup(tmp), g_strdup(v));
 823 #ifdef VERBOSE
 824 g_printf("NLS(%s): [%s]\n", tmp, v);
 825 #endif
 826 }
 827
 828 /********************************************************************/
 829
 830 static void
 831 node_print (node *n)
 832 {
 833 g_assert(n);
 834 if (n->data) {
 835         g_printf("N: %d [%f:%f][%s](%d)\n",
 836                 n->id, n->lat, n->lon,
 837                 n->data->name ? n->data->name : "-",
 838                 n->type);
 839 } else {
 840         g_printf("N: %d [%f:%f]\n",
 841                 n->id, n->lat, n->lon);
 842 }
 843 }
 844
 845 #ifdef DEBUG
 846 static void
 847 dump_array(const gchar **p)
 848 {
 849 char **d;
 850
 851 d=p;
 852 while (*d!=NULL) {
 853         g_printf("[%s]", *d);
 854         d++;
 855 }
 856 g_print("\n");
 857 }
 858 #endif
 859
 860 static inline gboolean
 861 osm_node_check_box(gdouble nlat, gdouble nlon)
 862 {
 863 if (use_bbox==FALSE)
 864         return TRUE;
 865 return (nlat > bbox.lat_min && nlat < bbox.lat_max && nlon > bbox.lon_min && nlon < bbox.lon_max) ? TRUE : FALSE;
 866 }
 867
 868 static void
 869 osm_new_node_data(node *n)
 870 {
 871 if (n==NULL)
 872         return;
 873 if (n->data!=NULL)
 874         return;
 875 n->data=g_slice_new(node_data);
 876 n->data->name=NULL;
 877 n->data->url=NULL;
 878 n->data->desc=NULL;
 879 n->data->postal_code=NULL;
 880 n->type=NODE_PLAIN;
 881 noded_cnt++;
 882 }
 883
 884 static void
 885 osm_free_node_data(node *n)
 886 {
 887 g_assert(n);
 888 g_assert(n->data);
 889 if (n->data->name)
 890         g_free(n->data->name);
 891 if (n->data->url)
 892         g_free(n->data->url);
 893 if (n->data->desc)
 894         g_free(n->data->desc);
 895 if (n->data->postal_code)
 896         g_free(n->data->postal_code);
 897 g_slice_free(node_data, n->data);
 898 n->data=NULL;
 899 noded_cnt--;
 900 }
 901
 902 static node *
 903 osm_new_node(gint id, gdouble lat, gdouble lon)
 904 {
 905 node *n=NULL;
 906
 907 n=g_slice_new(node);
 908 g_assert(n);
 909 n->id=id;
 910 n->lat=lat;
 911 n->lon=lon;
 912 n->data=(node_data *)NULL;
 913 return n;
 914 }
 915
 916 static void
 917 osm_free_node(node *n)
 918 {
 919 g_assert(n);
 920 g_slice_free(node, n);
 921 }
 922
 923 static node *
 924 osm_find_node(guint32 nid)
 925 {
 926 node *n;
 927
 928 g_assert(osm_nodes);
 929 n=g_hash_table_lookup(osm_nodes, GINT_TO_POINTER(nid));
 930 #if 0
 931 if (!n)
 932         g_printerr("ERROR: Node %d not found!\n", nid);
 933 #endif
 934 return n;
 935 }
 936
 937 static void
 938 osm_new_way_data(way *w)
 939 {
 940 if (w==NULL)
 941         return;
 942 if (w->data!=NULL)
 943         return;
 944
 945 w->data=g_slice_new(way_data);
 946 w->data->name=NULL;
 947 w->data->names=NULL;
 948 w->data->ref=NULL;
 949 w->data->int_ref=NULL;
 950 w->data->postal_code=NULL;
 951 w->data->layer=0;
 952 w->data->speed=0;
 953 }
 954
 955 static void
 956 osm_free_way_data(way *w)
 957 {
 958 g_assert(w);
 959 if (!w->data)
 960         return;
 961 if (w->data->name)
 962         g_free(w->data->name);
 963 if (w->data->ref)
 964         g_free(w->data->ref);
 965 if (w->data->int_ref)
 966         g_free(w->data->int_ref);
 967 g_slice_free(way_data, w->data);
 968 w->data=NULL;
 969 }
 970
 971 static way *
 972 osm_new_way(gint id)
 973 {
 974 way *w;
 975
 976 w=g_slice_new(way);
 977 g_assert(w);
 978 w->id=id;
 979 w->nodes=NULL;
 980 w->type=WAY_UNWAYED;
 981 w->data=NULL;
 982 w->ncnt=0;
 983 w->flags=0;
 984
 985 /* Add to list of ways */
 986 return w;
 987 }
 988
 989 static void
 990 osm_way_add_to_list(way *w)
 991 {
 992 g_assert(w);
 993 osm_ways=g_slist_prepend(osm_ways, w);
 994 }
 995
 996 static void
 997 osm_way_new_node(way *w, gint nid)
 998 {
 999 node *n;
1000
1001 g_assert(w);
1002 n=osm_find_node(nid);
1003 w->nodes=g_slist_prepend(w->nodes, n);
1004 w->ncnt++;
1005 }
1006
1007 /**
1008  * Search the place hash table for the location of the node.
1009  *
1010  */
1011 static guint32
1012 osm_find_node_place(node *n)
1013 {
1014 node *t;
1015 gchar **isin;
1016 gchar **place;
1017
1018 if (!n->data)
1019         return 0;
1020
1021 isin=g_hash_table_lookup(osm_node_isin, GINT_TO_POINTER(n->id));
1022
1023 if (!isin)
1024         return 0;
1025
1026 place=isin;
1027 while (*place!=NULL) {
1028         gchar *ps;
1029         ps=g_strstrip(*place);
1030 #ifdef VERBOSE
1031         g_printf("Checking (%d) [%s] in [%s]\n",n->type, n->data->name, ps);
1032 #endif
1033         switch (n->type) {
1034         case NODE_PLACE_CITY:
1035         case NODE_PLACE_TOWN:
1036         case NODE_PLACE_VILLAGE:
1037         case NODE_PLACE_HAMLET:
1038                 t=g_hash_table_lookup(osm_place_region, ps);
1039                 if (t)
1040                         return t->id;
1041                 t=g_hash_table_lookup(osm_place_country, ps);
1042                 if (t)
1043                         return t->id;
1044         break;
1045         case NODE_PLACE_SUBURB:
1046         case NODE_PLACE_LOCALITY:
1047                 t=g_hash_table_lookup(osm_place_city, ps);
1048                 if (t)
1049                         return t->id;
1050         break;
1051         case NODE_PLACE_ISLAND:
1052                 return 0;
1053         break;
1054         default:
1055                 t=g_hash_table_lookup(osm_place_city, ps);
1056                 if (t)
1057                         return t->id;
1058         break;
1059         }
1060         place++;
1061 }
1062
1063 return 0;
1064 }
1065
1066 static guint32
1067 osm_find_way_place(way *w, node_type_t nt)
1068 {
1069 gchar **isin;
1070 gchar **place;
1071
1072 isin=g_hash_table_lookup(osm_way_isin, GINT_TO_POINTER(w->id));
1073 if (!isin)
1074         return 0;
1075
1076 place=isin;
1077 while (*place!=NULL) {
1078         node *t;
1079         gchar *ps;
1080
1081         ps=g_strstrip(*place);
1082
1083 #ifdef VERBOSE
1084         g_printf("Checking (%d) in [%s]\n",w->id, ps);
1085 #endif
1086 switch (nt) {
1087         case NODE_PLACE_CITY:
1088         case NODE_PLACE_TOWN:
1089         case NODE_PLACE_VILLAGE:
1090         case NODE_PLACE_HAMLET:
1091         case NODE_PLACE_LOCALITY:
1092                 t=g_hash_table_lookup(osm_place_city, ps);
1093                 if (t)
1094                         return t->id;
1095         break;
1096         case NODE_PLACE_COUNTRY:
1097                 t=g_hash_table_lookup(osm_place_country, ps);
1098                 if (t)
1099                         return t->id;
1100         break;
1101         default:
1102                 g_assert_not_reached();
1103         break;
1104         }
1105         place++;
1106 }
1107
1108 return 0;
1109 }
1110
1111 /***********************************************************************/
1112
1113 static void
1114 osm_node_save_node(gint key, gpointer value, gpointer user_data)
1115 {
1116 node *n=(node *)value;
1117
1118 dbnode_cnt++;
1119 db_insert_node(n);
1120 if (dbnode_cnt % 26214==0)
1121         g_printf("\rNodes: %f%%\n",((float)dbnode_cnt/(float)node_cnt)*100);
1122 }
1123
1124 /**
1125  * Check node type and insert as POI or Place
1126  * Discard extra data after insert.
1127  */
1128 static gboolean
1129 osm_node_save_poi(node *n, gpointer user_data)
1130 {
1131 if (!n) {
1132         g_printerr("ERROR: null poi\n");
1133         return FALSE;
1134 }
1135
1136 if (!n->data) {
1137         g_printerr("POI node with no data ?\n");
1138         return FALSE;
1139 }
1140
1141 n->data->isin_p=osm_find_node_place(n);
1142 n->data->isin_c=0;
1143
1144 if (n->type>NODE_POI_START && n->type<NODE_POI_END) {
1145         print_node(n);
1146         db_insert_poi(n);
1147         osm_free_node_data(n);
1148 } else if (n->type>NODE_PLACE_START && n->type<NODE_PLACE_END) {
1149         print_node(n);
1150         db_insert_place(n);
1151 } else {
1152         osm_free_node_data(n);
1153         return FALSE;
1154 }
1155
1156 return TRUE;
1157 }
1158
1159 static gboolean
1160 osm_planet_poi_clear_nodes(void)
1161 {
1162 g_print("Removing old OSM POIs...\n");
1163 db_transaction_begin(db);
1164 sqlite3_step(sql.delete_osm_poi);
1165 sqlite3_step(sql.delete_place);
1166 return db_transaction_commit(db);
1167 }
1168
1169 static gboolean
1170 osm_planet_poi_save_nodes(void)
1171 {
1172 g_print("Storing new POIs...\n");
1173 db_transaction_begin(db);
1174 g_slist_foreach(osm_poi, osm_node_save_poi, NULL);
1175 g_slist_free(osm_poi);
1176 return db_transaction_commit(db);
1177 }
1178
1179 /*********************************************************************/
1180
1181 static void
1182 osm_planet_clear_nodes(void)
1183 {
1184 g_print("Clearing old nodes\n");
1185 sqlite3_step(sql.delete_nodes);
1186 }
1187
1188 static gboolean
1189 osm_planet_save_nodes(void)
1190 {
1191 g_print("Storing nodes\n");
1192 db_transaction_begin(db);
1193 g_hash_table_foreach(osm_nodes, osm_node_save_node, NULL);
1194 return db_transaction_commit(db);
1195 }
1196
1197 /*********************************************************************/
1198
1199 static void
1200 osm_way_save(way *value, gpointer user_data)
1201 {
1202 dbway_cnt++;
1203 db_insert_way(value);
1204 if (dbway_cnt % 16384==0 && dbway_cnt>0) {
1205                 g_printf("\rWays: %f%%\n",(((float)dbway_cnt/(float)way_cnt)*100));
1206                 print_way(value);
1207 }
1208 }
1209
1210 static void
1211 osm_planet_clear_ways(void)
1212 {
1213 g_print("Clearing old data\n");
1214 sqlite3_step(sql.delete_way);
1215 sqlite3_step(sql.delete_way_name);
1216 sqlite3_step(sql.delete_way_ref);
1217 sqlite3_step(sql.delete_way_n2n);
1218 }
1219
1220 static gboolean
1221 osm_planet_save_ways(void)
1222 {
1223 g_print("Inserting new ways\n");
1224 db_transaction_begin(db);
1225 g_slist_foreach(osm_ways, osm_way_save, NULL);
1226 return db_transaction_commit(db);
1227 }
1228
1229 /*********************************************************************/
1230
1231 static void
1232 osm_planet_save_all_nodes(void)
1233 {
1234 g_printf("Saving planet nodes to database:\n");
1235
1236 osm_planet_poi_clear_nodes();
1237 osm_planet_poi_save_nodes();
1238
1239 if (!is_update) {
1240         osm_planet_clear_nodes();
1241         osm_planet_clear_ways();
1242 }
1243 osm_planet_save_nodes();
1244 }
1245
1246 static void
1247 osm_planet_save_all_ways(void)
1248 {
1249 g_printf("Saving planet way to database:\n");
1250
1251 osm_planet_save_ways();
1252 }
1253
1254 /***********************************************************************/
1255
1256 static void
1257 _osm_tag_start(void *userData, const char *name, const char **atts)
1258 {
1259 tag_state_t t;
1260 gchar *k, *v;
1261 guint32 id, ndref;
1262 gdouble nlat, nlon;
1263
1264 t=check_tag(name);
1265 switch (t) {
1266         case IN_OSM_TAG:
1267                 g_printf("Starting...\n");
1268         break;
1269         case IN_NODE_TAG:
1270                 tag_parent=IS_NODE;
1271                 node_cnt++;
1272
1273                 id=atoi(get_attr_key_value(atts, "id"));
1274                 nlat=atof(get_attr_key_value(atts, "lat"));
1275                 nlon=atof(get_attr_key_value(atts, "lon"));
1276
1277                 cnode=osm_new_node(id, nlat, nlon);
1278                 osm_node_tags=g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free);
1279         break;
1280         case IN_WAY_TAG:
1281                 tag_parent=IS_WAY;
1282                 way_cnt++;
1283                 id=atoi(get_attr_key_value(atts, "id"));
1284                 cway=osm_new_way(id);
1285                 osm_way_tags=g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free);
1286         break;
1287         case IN_WNODE_TAG:
1288                 ndref=atoi(get_attr_key_value(atts, "ref"));
1289                 if (use_bbox==TRUE) {
1290                         if (osm_find_node(ndref)==NULL) {
1291                                 cway->id=0;
1292                                 return;
1293                         }
1294                 }
1295                 osm_way_new_node(cway, ndref);
1296         break;
1297         case IN_KEY_TAG:
1298                 k=get_attr_key_value(atts, "k");
1299                 if (strcmp(k,"created_by")==0)
1300                         return;
1301                 if (strcmp(k,"source")==0)
1302                         return;
1303
1304                 v=get_attr_key_value(atts, "v");
1305 #ifdef VERBOSE_KEYS
1306                 g_printf("TAG: K=[%s] V=[%s]\n", k, v);
1307 #endif
1308
1309                 switch (tag_parent) {
1310                 case IS_NONE:
1311                         g_printf("Tag key/value pair but unknown owner\n");
1312                 break;
1313                 case IS_NODE:
1314                 {
1315                         if (!osm_node_tags)
1316                                 return;
1317
1318                         /* Insert key/value pairs into hash table */
1319                         if (cnode==NULL) {
1320                                 g_printerr("In node tags but node is NULL!\n");
1321                                 return;
1322                         }
1323                         g_hash_table_insert(osm_node_tags, g_strdup(k), g_strdup(v));
1324                 }
1325                 break;
1326                 case IS_WAY:
1327                 {
1328                         if (cway==NULL) {
1329                                 g_printerr("In way tags but way is NULL!\n");
1330                                 return;
1331                         }
1332                         g_hash_table_insert(osm_way_tags, g_strdup(k), g_strdup(v));
1333                         osm_new_way_data(cway);
1334                 }
1335                 break;
1336                 case IS_RELATION:
1337
1338                 break;
1339                 }
1340         break;
1341         case IN_BOUND_TAG:
1342                 /* Ignore for now */
1343                 g_printf("Ignoring bound tag\n");
1344         break;
1345         case IN_RELATION_TAG:
1346                 tag_parent=IS_RELATION;
1347
1348         break;
1349         case IN_MEMBER_TAG:
1350
1351         break;
1352         default:
1353                 tag_parent=IS_NONE;
1354                 g_printf("Unknown tag: %s\n", name);
1355         break;
1356 }
1357 }
1358
1359 static void
1360 _osm_tag_end(void *userData, const char *name)
1361 {
1362 tag_state_t t;
1363 gchar *v;
1364 guint i;
1365 t=check_tag(name);
1366 switch (t) {
1367         case IN_NODE_TAG:
1368
1369                 if (node_cnt % 262140==0) {
1370                         g_printf("Nodes: %d of %d, POIs: %d, Outside box: %d\n", node_cnt-node_skip_cnt, node_cnt, noded_cnt, node_skip_cnt);
1371                 }
1372
1373                 if (!osm_node_tags)
1374                         return;
1375
1376                 osm_new_node_data(cnode);
1377
1378                 for (i=0; nodeinfo[i].k; i++) {
1379                         v=g_hash_table_lookup(osm_node_tags, nodeinfo[i].k);
1380                         if (!v)
1381                                 continue;
1382                         if (strcasecmp (v, nodeinfo[i].v)==0) {
1383                                 cnode->type=nodeinfo[i].type;
1384                                 break;
1385                         }
1386                 }
1387
1388                 /* Check if node is inside bounding box, if not skip it.
1389                  * But keep it if it's something we might need for other nodes:
1390                  * - Places (for is_in)
1391                  * - ...
1392                  */
1393                 if ((osm_node_check_box(cnode->lat, cnode->lon)==FALSE) &&
1394                                 (cnode->type<NODE_PLACE_START)) {
1395                         osm_free_node_data(cnode);
1396                         osm_free_node(cnode);
1397                         g_hash_table_destroy(osm_node_tags);
1398                         node_skip_cnt++;
1399                         return;
1400                 }
1401
1402                 g_hash_table_insert(osm_nodes, GINT_TO_POINTER(cnode->id), cnode);
1403
1404                 if (cnode->type!=NODE_PLAIN) {
1405                         cnode->data->name=NULL;
1406                         v=g_hash_table_lookup(osm_node_tags, "name");
1407                         if (v)
1408                                 cnode->data->name=g_strstrip(g_utf8_normalize(v, -1, G_NORMALIZE_ALL_COMPOSE));
1409                         v=g_hash_table_lookup(osm_node_tags, "note");
1410                         if (v)
1411                                 cnode->data->desc=g_strstrip(g_strdup(v));
1412                         v=g_hash_table_lookup(osm_node_tags, "postal_code");
1413                         if (v)
1414                                 cnode->data->postal_code=g_strstrip(g_strdup(v));
1415
1416                         /* Links */
1417                         v=g_hash_table_lookup(osm_node_tags, "url");
1418                         if (v) {
1419                                 cnode->data->url=g_strstrip(g_strdup(v));
1420                         } else {
1421                                 v=g_hash_table_lookup(osm_node_tags, "wikipedia");
1422                                 if (v && strncmp(v,"http:", 5)==0)
1423                                         cnode->data->url=g_strstrip(g_strdup(v));
1424                         }
1425                 }
1426
1427                 cnode->data->isin_c=0;
1428                 cnode->data->isin_p=0;
1429                 v=g_hash_table_lookup(osm_node_tags, "is_in");
1430                 if (v) {
1431                         gchar **isin;
1432                         isin=g_strsplit(v, ",", 10);
1433                         g_hash_table_insert(osm_node_isin, GINT_TO_POINTER(cnode->id), isin);
1434                 }
1435
1436                 if (cnode->type==NODE_PLAIN) {
1437                         osm_free_node_data(cnode);
1438                 } else {
1439                         osm_poi=g_slist_prepend(osm_poi, cnode);
1440                         if (cnode->data->name) {
1441                                 switch (cnode->type) {
1442                                 case NODE_PLACE_COUNTRY:
1443                                         g_hash_table_insert(osm_place_country, cnode->data->name, cnode);
1444                                 break;
1445                                 case NODE_PLACE_CITY:
1446                                 case NODE_PLACE_TOWN:
1447                                         g_hash_table_insert(osm_place_city, cnode->data->name, cnode);
1448                                 break;
1449                                 case NODE_PLACE_SUBURB:
1450                                         g_hash_table_insert(osm_place_suburb, cnode->data->name, cnode);
1451                                 break;
1452                                 case NODE_PLACE_VILLAGE:
1453                                 case NODE_PLACE_HAMLET:
1454                                 case NODE_PLACE_LOCALITY:
1455                                         g_hash_table_insert(osm_place_village, cnode->data->name, cnode);
1456                                 break;
1457                                 case NODE_PLACE_ISLAND:
1458                                         /* Ignore for now */
1459                                 break;
1460                                 default:;
1461                                 }
1462                         }
1463                 }
1464                 g_hash_table_destroy(osm_node_tags);
1465                 cnode=NULL;
1466         break;
1467         case IN_WAY_TAG:
1468                 if (way_cnt % 1024==0) {
1469                         g_printf("\rWays: %d\n", way_cnt);
1470                 }
1471
1472                 cway->nodes=g_slist_reverse(cway->nodes);
1473
1474                 for (i=0; wayinfo[i].k; i++) {
1475                         v=g_hash_table_lookup(osm_way_tags, wayinfo[i].k);
1476                         if (!v)
1477                                 continue;
1478                         if (strcasecmp (v, wayinfo[i].v)==0) {
1479                                 if (wayinfo[i].link==TRUE)
1480                                         cway->flags|=W_LINK;
1481                                 if (wayinfo[i].area==TRUE)
1482                                         cway->flags|=W_AREA;
1483                                 if (wayinfo[i].oneway==TRUE)
1484                                         cway->flags|=W_ONEWAY;
1485                                 cway->type=wayinfo[i].type;
1486                                 if (cway->data->speed==0)
1487                                         cway->data->speed=wayinfo[i].defspeed;
1488                                 break;
1489                         }
1490                 }
1491
1492                 v=g_hash_table_lookup(osm_way_tags, "name");
1493                 if (v) {
1494                         cway->data->name=g_utf8_normalize(v, -1, G_NORMALIZE_ALL_COMPOSE);
1495                         /* Try to find other language names */
1496                         cway->data->names=g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free);
1497                         g_hash_table_foreach(osm_way_tags, find_nls_names, cway->data->names);
1498                         if (g_hash_table_size(cway->data->names)==0) {
1499                                 g_hash_table_destroy(cway->data->names);
1500                                 cway->data->names=NULL;
1501                         }
1502                 }
1503
1504                 v=g_hash_table_lookup(osm_way_tags, "ref");
1505                 if (v)
1506                         cway->data->ref=g_strdup(v);
1507                 v=g_hash_table_lookup(osm_way_tags, "int_ref");
1508                 if (v)
1509                         cway->data->int_ref=g_strdup(v);
1510
1511                 v=g_hash_table_lookup(osm_way_tags, "postal_code");
1512                 if (v)
1513                         cway->data->postal_code=g_strdup(v);
1514
1515                 /* XXX: somehow handle the silly -1 'reversed' oneway */
1516                 v=g_hash_table_lookup(osm_way_tags, "oneway");
1517                 if (v)
1518                         cway->flags|=W_ONEWAY;
1519
1520                 v=g_hash_table_lookup(osm_way_tags, "noexit");
1521                 if (v)
1522                         cway->flags|=W_NOEXIT;
1523
1524                 v=g_hash_table_lookup(osm_way_tags, "speedlimit");
1525                 if (v)
1526                         cway->data->speed=atoi(v);
1527                 v=g_hash_table_lookup(osm_way_tags, "maxspeed");
1528                 if (v)
1529                         cway->data->speed=atoi(v);
1530
1531 #if 0
1532                 v=g_hash_table_lookup(osm_way_tags, "layer");
1533                 if (v)
1534                         cway->data->layer=atoi(v);
1535 #endif
1536
1537                 v=g_hash_table_lookup(osm_way_tags, "junction");
1538                 if (v && strcasecmp(v,"roundabout")==0) {
1539                         cway->flags|=W_ROUNDABOUT;
1540                         cway->flags|=W_ONEWAY;
1541                 } else if (v && strcasecmp(v,"mini_roundabout")==0) {
1542                         cway->flags|=W_ROUNDABOUT;
1543                         cway->flags|=W_ONEWAY;
1544                 }
1545
1546                 /* XXX: Should check keys */
1547                 v=g_hash_table_lookup(osm_way_tags, "access");
1548                 if (v && (strcasecmp(v, "private")==0)) {
1549                         cway->flags|=W_NOACCESS;
1550                 }
1551
1552                 print_way(cway);
1553
1554                 v=g_hash_table_lookup(osm_way_tags, "is_in");
1555                 if (v) {
1556                         gchar **isin;
1557                         isin=g_strsplit(v, ",", 10);
1558                         g_hash_table_insert(osm_way_isin, GINT_TO_POINTER(cway->id), isin);
1559                 }
1560
1561                 if (cway->data && cway->data->name==NULL && cway->data->ref==NULL &&
1562                         cway->data->int_ref==NULL && cway->data->layer==0 && cway->data->speed==0)
1563                         osm_free_way_data(cway);
1564
1565                 if (cway->id!=0)
1566                         osm_way_add_to_list(cway);
1567
1568                 cway=NULL;
1569                 g_hash_table_destroy(osm_way_tags);
1570         break;
1571         case IN_BOUND_TAG:
1572                 /* */
1573         break;
1574         case IN_OSM_TAG:
1575                 g_printf("\nPlanet loaded.\n");
1576         break;
1577         default:;
1578 }
1579 }
1580
1581 /************************************************************************/
1582
1583 static void
1584 storage_init(void)
1585 {
1586 osm_nodes=g_hash_table_new(g_direct_hash, g_direct_equal);
1587
1588 osm_place_country=g_hash_table_new(g_str_hash, g_str_equal);
1589 osm_place_city=g_hash_table_new(g_str_hash, g_str_equal);
1590 osm_place_suburb=g_hash_table_new(g_str_hash, g_str_equal);
1591 osm_place_village=g_hash_table_new(g_str_hash, g_str_equal);
1592 osm_place_region=g_hash_table_new(g_str_hash, g_str_equal);
1593 osm_node_isin=g_hash_table_new(g_direct_hash, g_direct_equal);
1594 osm_way_isin=g_hash_table_new(g_direct_hash, g_direct_equal);
1595 }
1596
1597 static void
1598 storage_free(void)
1599 {
1600 g_hash_table_destroy(osm_nodes);
1601
1602 g_hash_table_destroy(osm_place_country);
1603 g_hash_table_destroy(osm_place_city);
1604 g_hash_table_destroy(osm_place_suburb);
1605 g_hash_table_destroy(osm_place_village);
1606 g_hash_table_destroy(osm_place_region);
1607 g_hash_table_destroy(osm_node_isin);
1608 }
1609
1610 /************************************************************************/
1611
1612 static gint
1613 print_fail(const gchar *msg, gint ret)
1614 {
1615 g_printerr("ERROR: %s\n", msg);
1616 return ret;
1617 }
1618
1619 /************************************************************************/
1620
1621 static void
1622 print_memory_usage(void)
1623 {
1624 g_print("Memory usage per item:\n");
1625 g_printf("Node  size: %d\n", (gint)sizeof(node));
1626 g_printf("NodeD size: %d\n", (gint)sizeof(node_data));
1627 g_printf("Way   size: %d\n", (gint)sizeof(way));
1628 g_printf("WayD  size: %d\n", (gint)sizeof(way_data));
1629 }
1630
1631 /************************************************************************
1632  * Public inteface
1633  ************************************************************************/
1634
1635 void
1636 osm_planet_parser_init(void)
1637 {
1638 xp=XML_ParserCreate(NULL);
1639 XML_SetElementHandler(xp, _osm_tag_start, _osm_tag_end);
1640 storage_init();
1641 }
1642
1643 void
1644 osm_planet_parser_deinit(void)
1645 {
1646 XML_ParserFree(xp);
1647 storage_free();
1648 }
1649
1650 gboolean
1651 osm_planet_parse_buffer(gchar *buffer, size_t r)
1652 {
1653 if (XML_Parse(xp, buffer, r, r>0 ? 0:1) == XML_STATUS_ERROR) {
1654         g_printerr("Parse error at line %d:\n%s\n",
1655         XML_GetCurrentLineNumber(xp),
1656         XML_ErrorString(XML_GetErrorCode(xp)));
1657         return FALSE;
1658 }
1659 return TRUE;
1660 }
1661
1662 gboolean
1663 osm_planet_parse_file(gchar *pfile)
1664 {
1665 FILE *f;
1666 BZFILE *b;
1667 int bzerror;
1668 int r;
1669 gchar buffer[FILE_BUFFER];
1670 gboolean res=TRUE;
1671
1672 f=fopen(pfile, "r");
1673 if (!f) {
1674         perror("fopen failed\n");
1675         return FALSE;
1676 }
1677
1678 b=BZ2_bzReadOpen(&bzerror, f, 0, 0, NULL, 0);
1679 if (bzerror != BZ_OK) {
1680         g_printf("BZ2_bzReadOpen failed\n");
1681         BZ2_bzReadClose(&bzerror, b);
1682         return FALSE;
1683 }
1684
1685 do {
1686         r=BZ2_bzRead(&bzerror, b, buffer, FILE_BUFFER);
1687         if ((bzerror!=BZ_STREAM_END) && (bzerror!=BZ_OK)) {
1688                 res=FALSE;
1689                 break;
1690         }
1691         if (!osm_planet_parse_buffer(buffer, r)) {
1692                 res=FALSE;
1693                 break;
1694         }
1695 } while (bzerror==BZ_OK);
1696
1697 BZ2_bzReadClose(&bzerror, b);
1698 fclose(f);
1699 return res;
1700 }
1701
1702 void
1703 osm_import_set_bbox(gboolean use_bb, gdouble latmin, gdouble lonmin, gdouble latmax, gdouble lonmax)
1704 {
1705 use_bbox=use_bb;
1706 bbox.lat_min=latmin;
1707 bbox.lon_min=lonmin;
1708 bbox.lat_max=latmax;
1709 bbox.lon_max=lonmax;
1710 g_printf("Skipping data outside of box: %f,%f - %f,%f\n",
1711         bbox.lat_min, bbox.lon_min,     bbox.lat_max, bbox.lon_max);
1712 }
1713
1714 gboolean
1715 osm_import(const gchar *planet, const gchar *database)
1716 {
1717 if (db_connect(&db, database)!=TRUE) {
1718         g_printerr("Database open failed: %s", database);
1719         return FALSE;
1720 }
1721
1722 db_create_tables(db);
1723 db_create_indexes(db);
1724 db_prepare(db);
1725
1726 osm_planet_parser_init();
1727
1728 if (osm_planet_parse_file(planet)==FALSE) {
1729         g_printerr("Failed to parse file: %s\n", planet);
1730         return FALSE;
1731 }
1732
1733 g_printf("Total nodes %d, POIs: %d and Ways %d.\n",     node_cnt, noded_cnt, way_cnt);
1734 g_printf("Cities/Towns: %d\n", g_hash_table_size(osm_place_city));
1735 g_printf("Villages/Hamlets: %d\n", g_hash_table_size(osm_place_village));
1736 g_printf("Suburbs: %d\n", g_hash_table_size(osm_place_suburb));
1737 g_printf("Nodes: %d\n", g_hash_table_size(osm_nodes));
1738
1739 osm_planet_save_all_nodes();
1740 osm_planet_save_all_ways();
1741 osm_planet_parser_deinit();
1742 db_close(&db);
1743 return TRUE;
1744 }
1745