From 6319d0885feaafda3ed792c2f8325b1fe6934d7b Mon Sep 17 00:00:00 2001 From: Kaj-Michael Lang Date: Mon, 18 Feb 2008 23:04:48 +0200 Subject: [PATCH] Split out the import code from osm.c to osm-db-import.c and make osm.c just a simple cli interface to the import code. --- src/Makefile.am | 6 +- src/osm-db-import.c | 1746 ++++++++++++++++++++++++++++++++++++++++++ src/osm-db-import.h | 62 ++ src/osm-db.c | 2 - src/osm.c | 1762 +------------------------------------------ 5 files changed, 1821 insertions(+), 1757 deletions(-) create mode 100644 src/osm-db-import.c create mode 100644 src/osm-db-import.h diff --git a/src/Makefile.am b/src/Makefile.am index a88b1a2..879e0e0 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -78,7 +78,7 @@ mapper_SOURCES = utils.c \ # audio-note.c osm2db_CFLAGS = $(EXPAT_CFLAGS) $(SQLITE_CFLAGS) $(GLIBGTK_CFLAGS) -osm2db_LDADD = $(EXPAT_LIBS) $(SQLITE_LIBS) $(GLIBGTK_LIBS) $(BZ2_LIBS) -lm libosmdb.la +osm2db_LDADD = $(EXPAT_LIBS) $(SQLITE_LIBS) $(GLIBGTK_LIBS) -lm libosmdb.la osm2db_SOURCES = osm.c lib_LTLIBRARIES = libgtkgps.la libosmdb.la libmappergps.la @@ -103,9 +103,9 @@ libgtkgps_la_CFLAGS = $(GLIBGTK_CFLAGS) $(defines) libgtkgps_la_LIBADD = $(GLIBGTK_LIBS) libgtkgps_la_LDFLAGS = -lm -no-undefined -libosmdb_la_SOURCES = db.c osm-db.c latlon.c +libosmdb_la_SOURCES = db.c osm-db.c osm-db-import.c latlon.c libosmdb_la_CFLAGS = $(GLIBGTK_CFLAGS) $(SQLITE_CFLAGS) $(defines) -libosmdb_la_LIBADD = $(GLIBGTK_LIBS) $(SQLITE_LIBS) +libosmdb_la_LIBADD = $(GLIBGTK_LIBS) $(SQLITE_LIBS) $(BZ2_LIBS) libosmdb_la_LDFLAGS = -lm -no-undefined EXTRA_DIST = gps-bluetooth-maemo-marshal.list gps-bluetooth-bluez-marshal.list diff --git a/src/osm-db-import.c b/src/osm-db-import.c new file mode 100644 index 0000000..1bb3bf0 --- /dev/null +++ b/src/osm-db-import.c @@ -0,0 +1,1746 @@ +/* + * This file is part of mapper + * + * Copyright (C) 2007 Kaj-Michael Lang + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/* + * Routines to read OSM planet XML file and store it in a sqlite3 database. + * Reads in all nodes (if used, skips nodes outside bounding box) + * Special POI nodes are stored in POI table. + * Place POI nodes are stored in place table. + * + * Ways are read in and their data (name, type, etc) are stored + * in way, way_name and way_ref tables. + * + * Nodes used by they ways are stored in way_n2n table. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "osm.h" +#include "latlon.h" +#include "db.h" +#include "osm-db-import.h" + +#if 0 +#define VERBOSE +#endif +/* #define VERBOSE_KEYS */ + + +/* Use g_convert to transliterate names.. my iconv seems to be fucked so this didn't work... */ +/* #define TRANSLIT_NAMES */ + +#define FILE_BUFFER 65535 + +static guint node_cnt=0; +static guint node_skip_cnt=0; +static guint noded_cnt=0; +static guint way_cnt=0; +static guint way_names=0; +static guint way_refs=0; + +static guint dbnode_cnt=0; +static guint dbnoded_cnt=0; +static guint dbway_cnt=0; + +static gboolean is_update=FALSE; +static XML_Parser xp; + + +/* XML tag IDs */ +typedef enum { + START, + IN_OSM_TAG, + IN_NODE_TAG, + IN_WNODE_TAG, + IN_WAY_TAG, + IN_KEY_TAG, + IN_BOUND_TAG, + IN_RELATION_TAG, + IN_MEMBER_TAG, + END, + ERROR +} tag_state_t; + +/* Parent tag type */ +typedef enum { + IS_NONE, + IS_NODE, + IS_WAY, + IS_RELATION +} tag_parent_t; + +/* Node types table */ +/* XXX: Add support for parent category */ +struct _nodeinfo { + gchar *k, *v; + node_type_t type; +} nodeinfo[] = { + { "amenity", "fuel", NODE_AMENITY_FUEL }, + { "amenity", "parking", NODE_AMENITY_PARKING }, + + { "amenity", "pub", NODE_AMENITY_PUB }, + { "amenity", "nightclub", NODE_AMENITY_NIGHTCLUB }, + { "amenity", "biergarten", NODE_AMENITY_PUB }, + { "amenity", "cafe", NODE_AMENITY_CAFE }, + { "amenity", "fast_food", NODE_AMENITY_FOOD }, + { "amenity", "restaurant", NODE_AMENITY_FOOD }, + + { "amenity", "telephone", NODE_AMENITY_TELEPHONE }, + { "amenity", "toilets", NODE_AMENITY_WC }, + + { "amenity", "hospital", NODE_AMENITY_HOSPITAL }, + { "amenity", "doctors", NODE_AMENITY_HOSPITAL }, + { "amenity", "pharmacy", NODE_AMENITY_PHARMACY }, + + { "amenity", "post_office", NODE_AMENITY_POST }, + { "amenity", "post_box", NODE_AMENITY_POST_BOX }, + + { "amenity", "cinema", NODE_AMENITY_CINEMA }, + { "amenity", "theatre", NODE_AMENITY_THEATRE }, + + { "amenity", "atm", NODE_AMENITY_ATM }, + { "amenity", "bank", NODE_AMENITY_BANK }, + + { "amenity", "police", NODE_AMENITY_POLICE }, + { "amenity", "speed_trap", NODE_AMENITY_SPEEDCAM }, + { "amenity", "speed_camera", NODE_AMENITY_SPEEDCAM }, + { "amenity", "speed camera", NODE_AMENITY_SPEEDCAM }, + + { "amenity", "place_of_worship",NODE_AMENITY_POW }, + + { "amenity", "school", NODE_AMENITY_SCHOOL }, + { "amenity", "college", NODE_AMENITY_COLLEGE }, + { "amenity", "university", NODE_AMENITY_COLLEGE }, + + { "amenity", "library", NODE_AMENITY_LIBRARY }, + { "amenity", "townhall", NODE_AMENITY_TOWNHALL }, + + { "amenity", "supermarket", NODE_AMENITY_SHOP }, + { "amenity", "shopping_centre", NODE_AMENITY_SHOP }, + { "amenity", "shop", NODE_AMENITY_SHOP }, + { "amenity", "shops", NODE_AMENITY_SHOP }, + { "amenity", "shopping", NODE_AMENITY_SHOP }, + { "amenity", "shopping_mall",NODE_AMENITY_SHOP }, + { "amenity", "cycle_shop", NODE_AMENITY_SHOP }, + { "amenity", "bike_shop", NODE_AMENITY_SHOP }, + { "amenity", "coffee_shop", NODE_AMENITY_SHOP }, + { "amenity", "indoor_shopping_centre", NODE_AMENITY_SHOP }, + { "amenity", "farm_shop", NODE_AMENITY_SHOP }, + { "amenity", "tea_shop", NODE_AMENITY_SHOP }, + + /* Shops */ + { "shop", "supermarket", NODE_AMENITY_SHOP }, + { "shop", "bakery", NODE_AMENITY_SHOP }, + { "shop", "alcohol", NODE_AMENITY_SHOP }, + { "shop", "butcher", NODE_AMENITY_SHOP }, + { "shop", "flowers", NODE_AMENITY_SHOP }, + { "shop", "clothing", NODE_AMENITY_SHOP }, + { "shop", "souvenir", NODE_AMENITY_SHOP }, + { "shop", "bicycles", NODE_AMENITY_SHOP }, + { "shop", "grocers", NODE_AMENITY_SHOP }, + { "shop", "newsagents", NODE_AMENITY_SHOP }, + { "shop", "convenience", NODE_AMENITY_SHOP }, + { "shop", "bakers", NODE_AMENITY_SHOP }, + { "shop", "garden_centre",NODE_AMENITY_SHOP }, + { "shop", "photography", NODE_AMENITY_SHOP }, + { "shop", "general_store",NODE_AMENITY_SHOP }, + { "shop", "food", NODE_AMENITY_SHOP }, + { "shop", "drinks", NODE_AMENITY_SHOP }, + { "shop", "pharmacy", NODE_AMENITY_PHARMACY }, + + /* Sport */ + { "sport" , "swimming", NODE_SPORT_SWIMMING }, + { "sport" , "golf", NODE_SPORT_GOLF }, + { "sport" , "tennis", NODE_SPORT_TENNIS }, + { "sport" , "football", NODE_SPORT_FOOTBALL }, + { "sport" , "soccer", NODE_SPORT_SOCCER }, + { "sport" , "baskteball", NODE_SPORT_BASKETBALL }, + { "sport" , "rugby", NODE_SPORT_RUGBY }, + { "sport" , "skating", NODE_SPORT_SKATING }, + { "sport" , "hockey", NODE_SPORT_HOCKEY }, + { "sport" , "skateboard", NODE_SPORT_SKATEBOARD }, + { "sport" , "bowling", NODE_SPORT_BOWLING }, + { "sport" , "10pin", NODE_SPORT_BOWLING }, + { "sport" , "motor", NODE_SPORT_MOTOR }, + { "sport" , "shooting_range",NODE_SPORT_SHOOTING }, + { "sport" , "paintball", NODE_SPORT_PAINTBALL }, + { "sport" , "horse_racing",NODE_SPORT_HORSES }, + { "sport" , "horse", NODE_SPORT_HORSES }, + { "sport" , "horses", NODE_SPORT_HORSES }, + { "sport" , "dog_racing", NODE_SPORT_DOG }, + { "sport" , "pelota", NODE_SPORT_PELOTA }, + { "sport" , "racquet", NODE_SPORT_RACQUET }, + { "sport" , "equestrian", NODE_SPORT_HORSES }, + { "sport" , "baseball", NODE_SPORT_BASEBALL }, + { "sport" , "cricket", NODE_SPORT_CRICKET }, + { "sport" , "croquet", NODE_SPORT_CROQUET }, + { "sport" , "cycling", NODE_SPORT_CYCLING }, + { "sport" , "bowls", NODE_SPORT_BOWLS }, + { "sport" , "athletics", NODE_SPORT_ATHLETICS }, + { "sport" , "gymnastics", NODE_SPORT_GYMNASTICS }, + { "sport" , "multi", NODE_SPORT_OTHER }, + { "leisure", "sport_centre",NODE_SPORT_CENTER }, + + /* Tourism */ + { "tourism", "information", NODE_TOURISM_INFO }, + { "tourism", "camp_site", NODE_TOURISM_CAMP_SITE }, + { "tourism", "caravan_site",NODE_TOURISM_CARAVAN_SITE }, + { "tourism", "picnic_site", NODE_TOURISM_PICNIC_SITE }, + { "tourism", "theme_park", NODE_TOURISM_THEME_PARK }, + { "tourism", "hotel", NODE_TOURISM_HOTEL }, + { "tourism", "motel", NODE_TOURISM_MOTEL }, + { "tourism", "hostel", NODE_TOURISM_HOSTEL }, + { "tourism", "attraction", NODE_TOURISM_ATTRACTION }, + { "tourism", "zoo", NODE_TOURISM_ATTRACTION }, + + { "historic", "ruins", NODE_TOURISM_ATTRACTION }, + { "historic", "monument", NODE_TOURISM_ATTRACTION }, + { "historic", "memorial", NODE_TOURISM_ATTRACTION }, + { "historic", "museum", NODE_HISTORIC_MUSEUM }, + { "historic", "castle", NODE_HISTORIC_CASTLE }, + + { "railway", "station", NODE_RAILWAY_STATION }, + { "railway", "halt", NODE_RAILWAY_HALT }, + + { "aeroway", "terminal", NODE_AIRPORT_TERMINAL }, + + /* Places */ + { "place", "city", NODE_PLACE_CITY }, + { "place", "town", NODE_PLACE_TOWN }, + { "place", "village", NODE_PLACE_VILLAGE }, + { "place", "hamlet", NODE_PLACE_HAMLET }, + { "place", "locality", NODE_PLACE_LOCALITY }, + { "place", "suburb", NODE_PLACE_SUBURB }, + { "place", "island", NODE_PLACE_ISLAND }, + + { "highway", "traffic_signals", NODE_TRAFFIC_SIGNALS }, + { "highway", "motorway_junction", NODE_JUNCTION }, + { "highway", "services", NODE_AMENITY_PARKING }, + { "highway", "toll_booth", NODE_TOLLBOOTH }, + { "highway", "gate", NODE_GATE }, + + { NULL, NULL, NODE_PLAIN } +}; + +/* Array to get id number and defaults for ways of different types */ +struct _wayinfo { + gchar *k, *v; + guint defspeed; + way_type_t type; + gboolean oneway, link, area, car, foot; +} wayinfo[] = { + { "highway", "motorway",120,WAY_MOTORWAY, TRUE, FALSE, FALSE, TRUE, FALSE }, + { "highway", "motorway_link",120,WAY_MOTORWAY, TRUE, TRUE, FALSE, TRUE, FALSE }, + { "highway", "trunk",100,WAY_TRUNK, FALSE, FALSE, FALSE, TRUE, FALSE }, + { "highway", "trunk_link",100,WAY_TRUNK, FALSE, TRUE, FALSE, TRUE, FALSE }, + { "highway", "primary",80,WAY_PRIMARY, FALSE, FALSE, FALSE, TRUE, TRUE }, + { "highway", "primary_link",60,WAY_PRIMARY, FALSE, TRUE, FALSE, TRUE, TRUE }, + { "highway", "secondary",80,WAY_SECONDARY, FALSE, FALSE, FALSE, TRUE, TRUE }, + { "highway", "secondary_link",60,WAY_SECONDARY, FALSE, TRUE, FALSE, TRUE, TRUE }, + { "highway", "tertiary",60,WAY_TERTIARY, FALSE, FALSE, FALSE, TRUE, TRUE }, + { "highway", "unclassified",50,WAY_UNCLASSIFIED, FALSE, FALSE, FALSE, TRUE, TRUE }, + { "highway", "byway",40,WAY_UNCLASSIFIED, FALSE, FALSE, FALSE, TRUE, TRUE }, + { "highway", "residential",40,WAY_RESIDENTIAL, FALSE, FALSE, FALSE, TRUE, TRUE }, + { "highway", "service",20,WAY_SERVICE, FALSE, FALSE, FALSE, TRUE, TRUE }, + { "highway", "track",20,WAY_TRACK, FALSE, FALSE, FALSE, TRUE, TRUE }, + { "highway", "unsurfaced",60,WAY_TRACK, FALSE, FALSE, FALSE, TRUE, TRUE }, + { "highway", "minor",60,WAY_TRACK, FALSE, FALSE, FALSE, TRUE, TRUE }, + { "highway", "pedestrian",20,WAY_FOOTWAY, FALSE, FALSE, FALSE, FALSE, TRUE }, + { "highway", "footway",1,WAY_FOOTWAY, FALSE, FALSE, FALSE, FALSE, TRUE }, + { "highway", "steps",0,WAY_FOOTWAY, FALSE, FALSE, FALSE, FALSE, TRUE}, + { "highway", "bridleway",10,WAY_FOOTWAY, FALSE, FALSE, FALSE, FALSE, TRUE }, + { "highway", "cycleway",10,WAY_CYCLEWAY, FALSE, FALSE, FALSE, FALSE, TRUE }, + { "railway", "rail",0,WAY_RAIL, FALSE, FALSE, FALSE, FALSE, FALSE }, + { "aeroway", "runway",0,WAY_RUNWAY, FALSE, FALSE, FALSE, FALSE, FALSE }, + { "aeroway", "taxiway",0,WAY_TAXIWAY, FALSE, FALSE, FALSE, FALSE, FALSE }, + { "natural", "water",0,WAY_WATER, FALSE, FALSE, TRUE, FALSE, FALSE }, + { "waterway", "river",0,WAY_WATER, FALSE, FALSE, FALSE, FALSE, FALSE }, + { "waterway", "canal",0,WAY_WATER, FALSE, FALSE, FALSE, FALSE, FALSE }, + { "waterway", "stream",0,WAY_WATER, FALSE, FALSE, FALSE, FALSE, FALSE }, + { "building", "*",0,WAY_UNWAYED, FALSE, FALSE, TRUE, FALSE, FALSE }, + { NULL, NULL, 0, WAY_UNWAYED, FALSE, FALSE, FALSE, FALSE, FALSE } +}; + +static sqlite3 *db; +tag_parent_t tag_parent=IS_NONE; + +static GHashTable *osm_nodes; +static GHashTable *osm_node_tags; +static GHashTable *osm_way_tags; +static GSList *osm_ways; +static GSList *osm_poi; + +static GHashTable *osm_place_country; +static GHashTable *osm_place_region; +static GHashTable *osm_place_city; +static GHashTable *osm_place_suburb; +static GHashTable *osm_place_village; +static GHashTable *osm_node_isin; +static GHashTable *osm_way_isin; + +static node *cnode=NULL; +static way *cway=NULL; + +struct sql_stmt { + sqlite3_stmt *insert_poi; + sqlite3_stmt *delete_osm_poi; + + sqlite3_stmt *insert_node; + sqlite3_stmt *delete_nodes; + sqlite3_stmt *select_node; + sqlite3_stmt *update_node; + + sqlite3_stmt *insert_way_data; + sqlite3_stmt *insert_way_ref; + sqlite3_stmt *insert_way_pc; + sqlite3_stmt *insert_way_name; + sqlite3_stmt *insert_way_names_nls; + sqlite3_stmt *insert_way_n2n; + sqlite3_stmt *delete_way; + sqlite3_stmt *delete_way_n2n; + sqlite3_stmt *delete_way_name; + sqlite3_stmt *delete_way_names_nls; + sqlite3_stmt *delete_way_ref; + sqlite3_stmt *delete_way_pc; + + sqlite3_stmt *insert_place; + sqlite3_stmt *delete_place; +}; +static struct sql_stmt sql; + + +static struct map_bbox bbox; +static gboolean use_bbox; + +void osm_free_way_data(way *w); +void print_way(way *w); + +void db_prepare(sqlite3 *db); +gboolean db_insert_node(node *n); +guint32 osm_find_way_place(way *w, node_type_t nt); + +/****************************************************/ +/* Functions */ +/****************************************************/ + +static void +db_finalize(void) +{ +sqlite3_finalize(sql.insert_poi); +sqlite3_finalize(sql.delete_osm_poi); + +sqlite3_finalize(sql.insert_node); +sqlite3_finalize(sql.select_node); +sqlite3_finalize(sql.delete_nodes); +sqlite3_finalize(sql.update_node); + +sqlite3_finalize(sql.insert_place); +sqlite3_finalize(sql.delete_place); + +sqlite3_finalize(sql.delete_way); +sqlite3_finalize(sql.insert_way_data); + +sqlite3_finalize(sql.delete_way_name); +sqlite3_finalize(sql.insert_way_name); + +sqlite3_finalize(sql.delete_way_n2n); +sqlite3_finalize(sql.insert_way_n2n); + +sqlite3_finalize(sql.delete_way_pc); +sqlite3_finalize(sql.insert_way_pc); + +sqlite3_finalize(sql.delete_way_names_nls); +sqlite3_finalize(sql.insert_way_names_nls); +} + +void +db_prepare(sqlite3 *db) +{ +/* Way nodes */ +sqlite3_prepare_v2(db, "insert or replace into nodes (nid,ilat,ilon,rlat,rlon,l,f) values (?,?,?,?,?,0,?)", -1, &sql.insert_node, NULL); +sqlite3_prepare_v2(db, "select ilat,ilon,l from nodes where nid=?", -1, &sql.select_node, NULL); +sqlite3_prepare_v2(db, "delete from nodes", -1, &sql.delete_nodes, NULL); +sqlite3_prepare_v2(db, "update nodes set l=l+1 where nid=?", -1, &sql.update_node, NULL); + +/* Places */ +sqlite3_prepare_v2(db, "insert or replace into places (nid,type,name,isin_c,isin_p) values (?, ?, ?, ?, ?)", -1, &sql.insert_place, NULL); +sqlite3_prepare_v2(db, "delete from places", -1, &sql.delete_place, NULL); + +/* POI nodes */ +if (sqlite3_prepare_v2(db, "insert or replace into poi (osm_id, lat, lon, label, cat_id, public, source, priority, isin_c, isin_p, desc, url, postal_code) " + " values (?, ?, ?, ?, ?, 1, 1, ?, ?, ?, ?, ?, ?)", -1, &sql.insert_poi, NULL)!=SQLITE_OK) + g_printf("SQL: %s\n", sqlite3_errmsg(db)); + +sqlite3_prepare_v2(db, "delete from poi where osm_id>0 and source=1", -1, &sql.delete_osm_poi, NULL); + +/* Ways */ +sqlite3_prepare_v2(db, "insert or replace into way (wid,nodes,type,flags,speed,isin_c,isin_p,lat,lon) values (?, ?, ?, ?, ?, ?, ?, ?, ?)", -1, &sql.insert_way_data, NULL); +sqlite3_prepare_v2(db, "delete from way", -1, &sql.delete_way, NULL); + +/* Way nodes */ +sqlite3_prepare_v2(db, "insert into way_n2n (wid,f,t) values (?,?,?)", -1, &sql.insert_way_n2n, NULL); +sqlite3_prepare_v2(db, "delete from way_n2n where wid=?", -1, &sql.delete_way_n2n, NULL); + +/* Way names */ +sqlite3_prepare_v2(db, "insert or replace into way_names (wid,name,norm) values (?, ?, ?)", -1, &sql.insert_way_name, NULL); +sqlite3_prepare_v2(db, "delete from way_names", -1, &sql.delete_way_name, NULL); + +/* Way postal codes */ +sqlite3_prepare_v2(db, "insert or replace into way_pc (wid,pc) values (?, ?)", -1, &sql.insert_way_pc, NULL); +sqlite3_prepare_v2(db, "delete from way_pc", -1, &sql.delete_way_pc, NULL); + +/* Other language names for ways */ +sqlite3_prepare_v2(db, "insert into way_names_nls (wid,lang,name, norm) values (?, ?, ?, ?)", -1, &sql.insert_way_names_nls, NULL); +sqlite3_prepare_v2(db, "delete from way_names_nls where wid=?", -1, &sql.delete_way_names_nls, NULL); + +/* Way ref and int_ref */ +sqlite3_prepare_v2(db, "insert or replace into way_ref (rid,ref,int_ref) values (?, ?, ?)", -1, &sql.insert_way_ref, NULL); +sqlite3_prepare_v2(db, "delete from way_ref", -1, &sql.delete_way_ref, NULL); +} + +/********************************************************************/ + +void +print_way(way *w) +{ +#ifdef VERBOSE +g_assert(w); +g_printf("Way #%d(N:%d T:%d S:%d IS: %d/%d): %s [%s:%s:%s]\n", + w->id, + g_slist_length(w->nodes), + w->type, + w->data ? w->data->speed : 0, + w->data ? w->data->isin_c : -1, + w->data ? w->data->isin_p : -1, + w->data ? w->data->name ? w->data->name : "" : "", + w->flags & W_ONEWAY ? "-" : "=", + w->flags & W_ROUNDABOUT ? "O" : "-", + w->flags & W_LINK ? "|" : " "); +#endif +} + +void +print_node(node *n) +{ +#ifdef VERBOSE +g_assert(n); +g_printf("Node #%d: T:%d IS: %d/%d [%s]\n", + n->id, + n->type, + n->data ? n->data->isin_c : -1, + n->data ? n->data->isin_p : -1, + n->data ? n->data->name : ""); +#endif +} + +/********************************************************************/ + +gboolean +db_insert_node(node *n) +{ +gint32 lat, lon; + +g_assert(n); + +lat=lat2mp_int(n->lat); +lon=lon2mp_int(n->lon); + +sqlite3_bind_int(sql.insert_node, 1, n->id); + +/* Projected and integerized lat/lot */ +sqlite3_bind_int(sql.insert_node, 2, lat); +sqlite3_bind_int(sql.insert_node, 3, lon); +/* Original */ +sqlite3_bind_double(sql.insert_node, 4, n->lat); +sqlite3_bind_double(sql.insert_node, 5, n->lon); +sqlite3_bind_int(sql.insert_node, 6, n->type); + +db_exec(db, sql.insert_node); + +return TRUE; +} + +static gboolean +db_insert_place(node *n) +{ +g_assert(n); +if (!n->data) + return FALSE; +if (!n->data->name) + return FALSE; +sqlite3_bind_int(sql.insert_place, 1, n->id); +sqlite3_bind_int(sql.insert_place, 2, n->type); +sqlite3_bind_text(sql.insert_place, 3, n->data->name, -1, SQLITE_TRANSIENT); +sqlite3_bind_int(sql.insert_place, 4, n->data->isin_p); +sqlite3_bind_int(sql.insert_place, 5, n->data->isin_c); + +return db_exec(db,sql.insert_place); +} + +static gboolean +db_insert_poi(node *n) +{ +g_assert(n); +sqlite3_bind_int(sql.insert_poi, 1, n->id); +sqlite3_bind_double(sql.insert_poi, 2, n->lat); +sqlite3_bind_double(sql.insert_poi, 3, n->lon); +if (n->data->name) + sqlite3_bind_text(sql.insert_poi, 4, n->data->name, -1, SQLITE_TRANSIENT); +else + sqlite3_bind_text(sql.insert_poi, 4, "", -1, SQLITE_TRANSIENT); +sqlite3_bind_int(sql.insert_poi, 5, n->type); +sqlite3_bind_int(sql.insert_poi, 6, n->type/100); +sqlite3_bind_int(sql.insert_poi, 7, n->data->isin_c); +sqlite3_bind_int(sql.insert_poi, 8, n->data->isin_p); + +if (n->data->desc) + sqlite3_bind_text(sql.insert_poi, 9, n->data->desc, -1, SQLITE_TRANSIENT); +if (n->data->url) + sqlite3_bind_text(sql.insert_poi, 10, n->data->url, -1, SQLITE_TRANSIENT); +if (n->data->postal_code) + sqlite3_bind_text(sql.insert_poi, 11, n->data->postal_code, -1, SQLITE_TRANSIENT); + +return db_exec(db,sql.insert_poi); +} + +/** + * Update node usage count + */ +static gboolean +db_update_node_links(node *n) +{ +g_assert(n); +sqlite3_bind_int(sql.update_node, 1, n->id); + +return db_exec(db,sql.update_node); +} + +/** + * Insert way,node1,node2 triplet + */ +static gboolean +db_insert_way_n2n(way *w, node *nf, node *nt) +{ +if (!w) { + g_printf("NULL WAY\n"); + return FALSE; +} + +if (!nf) { + g_printf("NULL NODE 1\n"); + return FALSE; +} + +if (!nt) { + g_printf("NULL NODE 2\n"); + return FALSE; +} + +sqlite3_bind_int(sql.insert_way_n2n, 1, w->id); +sqlite3_bind_int(sql.insert_way_n2n, 2, nf->id); +sqlite3_bind_int(sql.insert_way_n2n, 3, nt->id); + +#ifdef VERBOSE_N2N +g_printf("%d [%d - %d]\n", w->id, nf->id, nt->id); +#endif + +db_exec(db,sql.insert_way_n2n); +db_update_node_links(nf); +db_update_node_links(nt); +return TRUE; +} + +/** + * Insert way ref and int_ref + */ +static void +db_insert_way_ref(way *w) +{ +if (!w->data) + return; + +if (!w->data->ref && !w->data->int_ref) + return; + +way_refs++; + +sqlite3_bind_int(sql.insert_way_ref, 1, w->id); +if (w->data->ref) + sqlite3_bind_text(sql.insert_way_ref, 2, w->data->ref, -1, SQLITE_TRANSIENT); +if (w->data->int_ref) + sqlite3_bind_text(sql.insert_way_ref, 3, w->data->int_ref, -1, SQLITE_TRANSIENT); + +db_exec(db,sql.insert_way_ref); +} + +/** + * Insert way name + */ +static void +db_insert_way_name(way *w) +{ +gchar *norm; + +if (!w->data) + return; +if (!w->data->name) + return; + +way_names++; + +sqlite3_bind_int(sql.insert_way_name, 1, w->id); +sqlite3_bind_text(sql.insert_way_name, 2, w->data->name, -1, SQLITE_TRANSIENT); + +#ifdef TRANSLIT_NAMES +norm=g_convert(w->data->name, -1, "ASCII//TRANSLIT//IGNORE", "utf8", NULL, NULL, NULL); +if (norm && strcmp(w->data->name, norm)!=0) { + sqlite3_bind_text(sql.insert_way_name, 3, norm, -1, SQLITE_TRANSIENT); +} +if (norm) + g_free(norm); +#endif + +db_exec(db,sql.insert_way_name); +} + +static void +db_delete_way_names_nls(way *w) +{ +sqlite3_bind_int(sql.delete_way_names_nls, 1, w->id); +db_exec(db,sql.delete_way_names_nls); +} + +static void +db_insert_way_pc(way *w) +{ +if (!w->data) + return; +if (!w->data->postal_code) + return; + +sqlite3_bind_int(sql.insert_way_pc, 1, w->id); +sqlite3_bind_text(sql.insert_way_pc, 2, w->data->postal_code, -1, SQLITE_TRANSIENT); + +db_exec(db,sql.insert_way_pc); +} + +static void +db_delete_way_pc(way *w) +{ +sqlite3_bind_int(sql.delete_way_pc, 1, w->id); +db_exec(db,sql.delete_way_pc); +} + +static void +db_insert_way_names_nls_cb(gpointer key, gpointer value, gpointer user_data) +{ +gchar *norm; + +way *w=(way *)user_data; + +sqlite3_bind_int(sql.insert_way_names_nls, 1, w->id); +sqlite3_bind_text(sql.insert_way_names_nls, 2, (gchar *)key, -1, SQLITE_TRANSIENT); +sqlite3_bind_text(sql.insert_way_names_nls, 3, (gchar *)value, -1, SQLITE_TRANSIENT); +#ifdef TRANSLIT_NAMES +norm=g_convert((gchar *value), -1, "ASCII//TRANSLIT//IGNORE", "utf8", NULL, NULL, NULL); +if (norm && strcmp((gchar *)value, norm)!=0) { + sqlite3_bind_text(sql.insert_way_names_nls, 4, norm, -1, SQLITE_TRANSIENT); +} +if (norm) + g_free(norm); +#endif +db_exec(db,sql.insert_way_names_nls); +} + +static void +db_insert_way_names_nls(way *w) +{ +if (!w->data) + return; +if (!w->data->names) + return; + +g_hash_table_foreach(w->data->names, db_insert_way_names_nls_cb, w); +} + +/** + * Insert all data for the given way + * - name + * - ref + * - nodes + * + */ +static gboolean +db_insert_way(way *w) +{ +GSList *iter; +guint ncnt; +node *wmn; + +if (!w) + return FALSE; + +/* Skip things we don't use (yet) */ +if (w->type==WAY_UNWAYED || w->type>WAY_ROAD_END) + return TRUE; + +/* Insert nodes */ +for (iter=w->nodes; iter!=NULL; iter=iter->next) { + if (!iter->next) + break; + db_insert_way_n2n(w, iter->data, iter->next->data); +} + +if (w->id==0) + return FALSE; + +if (w->data) { + w->data->isin_p=osm_find_way_place(w, NODE_PLACE_CITY); + w->data->isin_c=osm_find_way_place(w, NODE_PLACE_COUNTRY); +} + +print_way(w); + +/* Get middle node, use it as way location */ +ncnt=g_slist_length(w->nodes); +if (ncnt>1) + wmn=g_slist_nth_data(w->nodes, ncnt/2); +else + wmn=0; + +sqlite3_bind_int(sql.insert_way_data, 1, w->id); +sqlite3_bind_int(sql.insert_way_data, 2, w->ncnt); +sqlite3_bind_int(sql.insert_way_data, 3, w->type); +sqlite3_bind_int(sql.insert_way_data, 4, w->flags); +if (w->data) { + sqlite3_bind_int(sql.insert_way_data, 5, w->data->speed); + sqlite3_bind_int(sql.insert_way_data, 6, w->data->isin_c); + sqlite3_bind_int(sql.insert_way_data, 7, w->data->isin_p); +} +if (wmn) { + sqlite3_bind_double(sql.insert_way_data, 8, wmn->lat); + sqlite3_bind_double(sql.insert_way_data, 9, wmn->lon); +} else { + g_printerr("Failed to get way location node!\n"); +} + +db_exec(db,sql.insert_way_data); + +db_insert_way_ref(w); +db_insert_way_name(w); +db_insert_way_names_nls(w); +db_insert_way_pc(w); + +osm_free_way_data(w); +return TRUE; +} + +/********************************************************************/ + +static gchar * +get_attr_key_value(const gchar **p, gchar *key) +{ +gchar **d; + +d=p; +while (*d!=NULL) { + if (strncmp(*d, key, strlen(key))==0) { + d++; + return *d; + } + d++; + d++; +} +return NULL; +} + +static tag_state_t +check_tag(const gchar *tag) +{ +if (strcmp(tag,"node")==0) return IN_NODE_TAG; +else if (strcmp(tag,"nd")==0) return IN_WNODE_TAG; +else if (strcmp(tag,"way")==0) return IN_WAY_TAG; +else if (strcmp(tag,"tag")==0) return IN_KEY_TAG; +else if (strcmp(tag,"osm")==0) return IN_OSM_TAG; +else if (strcmp(tag,"bound")==0) return IN_BOUND_TAG; +else if (strcmp(tag,"relation")==0) return IN_RELATION_TAG; +else if (strcmp(tag,"member")==0) return IN_MEMBER_TAG; +else return ERROR; +} + +static void +find_nls_names(gpointer key, gpointer value, gpointer user_data) +{ +gchar *k, *v; +gchar *tmp; +GHashTable *nls; + +k=(gchar *)key; +v=(gchar *)value; +nls=(GHashTable *)user_data; + +/* Check if it is a name key, return if not. */ +if (g_str_has_prefix(k, "name:")==FALSE) + return; + +tmp=g_strrstr(k, ":"); +if (!tmp) + return; +tmp++; /* skip : */ +if (*tmp==0) + return; +g_hash_table_insert(nls, g_strdup(tmp), g_strdup(v)); +#ifdef VERBOSE +g_printf("NLS(%s): [%s]\n", tmp, v); +#endif +} + +/********************************************************************/ + +static void +node_print (node *n) +{ +g_assert(n); +if (n->data) { + g_printf("N: %d [%f:%f][%s](%d)\n", + n->id, n->lat, n->lon, + n->data->name ? n->data->name : "-", + n->type); +} else { + g_printf("N: %d [%f:%f]\n", + n->id, n->lat, n->lon); +} +} + +#ifdef DEBUG +static void +dump_array(const gchar **p) +{ +char **d; + +d=p; +while (*d!=NULL) { + g_printf("[%s]", *d); + d++; +} +g_print("\n"); +} +#endif + +static inline gboolean +osm_node_check_box(gdouble nlat, gdouble nlon) +{ +if (use_bbox==FALSE) + return TRUE; +return (nlat > bbox.lat_min && nlat < bbox.lat_max && nlon > bbox.lon_min && nlon < bbox.lon_max) ? TRUE : FALSE; +} + +static void +osm_new_node_data(node *n) +{ +if (n==NULL) + return; +if (n->data!=NULL) + return; +n->data=g_slice_new(node_data); +n->data->name=NULL; +n->data->url=NULL; +n->data->desc=NULL; +n->data->postal_code=NULL; +n->type=NODE_PLAIN; +noded_cnt++; +} + +static void +osm_free_node_data(node *n) +{ +g_assert(n); +g_assert(n->data); +if (n->data->name) + g_free(n->data->name); +if (n->data->url) + g_free(n->data->url); +if (n->data->desc) + g_free(n->data->desc); +if (n->data->postal_code) + g_free(n->data->postal_code); +g_slice_free(node_data, n->data); +n->data=NULL; +noded_cnt--; +} + +static node * +osm_new_node(gint id, gdouble lat, gdouble lon) +{ +node *n=NULL; + +n=g_slice_new(node); +g_assert(n); +n->id=id; +n->lat=lat; +n->lon=lon; +n->data=(node_data *)NULL; +return n; +} + +static void +osm_free_node(node *n) +{ +g_assert(n); +g_slice_free(node, n); +} + +static node * +osm_find_node(guint32 nid) +{ +node *n; + +g_assert(osm_nodes); +n=g_hash_table_lookup(osm_nodes, GINT_TO_POINTER(nid)); +#if 0 +if (!n) + g_printerr("ERROR: Node %d not found!\n", nid); +#endif +return n; +} + +static void +osm_new_way_data(way *w) +{ +if (w==NULL) + return; +if (w->data!=NULL) + return; + +w->data=g_slice_new(way_data); +w->data->name=NULL; +w->data->names=NULL; +w->data->ref=NULL; +w->data->int_ref=NULL; +w->data->postal_code=NULL; +w->data->layer=0; +w->data->speed=0; +} + +void +osm_free_way_data(way *w) +{ +g_assert(w); +if (!w->data) + return; +if (w->data->name) + g_free(w->data->name); +if (w->data->ref) + g_free(w->data->ref); +if (w->data->int_ref) + g_free(w->data->int_ref); +g_slice_free(way_data, w->data); +w->data=NULL; +} + +static way * +osm_new_way(gint id) +{ +way *w; + +w=g_slice_new(way); +g_assert(w); +w->id=id; +w->nodes=NULL; +w->type=WAY_UNWAYED; +w->data=NULL; +w->ncnt=0; +w->flags=0; + +/* Add to list of ways */ +return w; +} + +static void +osm_way_add_to_list(way *w) +{ +g_assert(w); +osm_ways=g_slist_prepend(osm_ways, w); +} + +static void +osm_way_new_node(way *w, gint nid) +{ +node *n; + +g_assert(w); +n=osm_find_node(nid); +w->nodes=g_slist_prepend(w->nodes, n); +w->ncnt++; +} + +/** + * Search the place hash table for the location of the node. + * + */ +static guint32 +osm_find_node_place(node *n) +{ +node *t; +gchar **isin; +gchar **place; + +if (!n->data) + return 0; + +isin=g_hash_table_lookup(osm_node_isin, GINT_TO_POINTER(n->id)); + +if (!isin) + return 0; + +place=isin; +while (*place!=NULL) { + gchar *ps; + ps=g_strstrip(*place); +#ifdef VERBOSE + g_printf("Checking (%d) [%s] in [%s]\n",n->type, n->data->name, ps); +#endif + switch (n->type) { + case NODE_PLACE_CITY: + case NODE_PLACE_TOWN: + case NODE_PLACE_VILLAGE: + case NODE_PLACE_HAMLET: + t=g_hash_table_lookup(osm_place_region, ps); + if (t) + return t->id; + t=g_hash_table_lookup(osm_place_country, ps); + if (t) + return t->id; + break; + case NODE_PLACE_SUBURB: + case NODE_PLACE_LOCALITY: + t=g_hash_table_lookup(osm_place_city, ps); + if (t) + return t->id; + break; + case NODE_PLACE_ISLAND: + return 0; + break; + default: + t=g_hash_table_lookup(osm_place_city, ps); + if (t) + return t->id; + break; + } + place++; +} + +return 0; +} + +guint32 +osm_find_way_place(way *w, node_type_t nt) +{ +gchar **isin; +gchar **place; + +isin=g_hash_table_lookup(osm_way_isin, GINT_TO_POINTER(w->id)); +if (!isin) + return 0; + +place=isin; +while (*place!=NULL) { + node *t; + gchar *ps; + + ps=g_strstrip(*place); + +#ifdef VERBOSE + g_printf("Checking (%d) in [%s]\n",w->id, ps); +#endif +switch (nt) { + case NODE_PLACE_CITY: + case NODE_PLACE_TOWN: + case NODE_PLACE_VILLAGE: + case NODE_PLACE_HAMLET: + case NODE_PLACE_LOCALITY: + t=g_hash_table_lookup(osm_place_city, ps); + if (t) + return t->id; + break; + case NODE_PLACE_COUNTRY: + t=g_hash_table_lookup(osm_place_country, ps); + if (t) + return t->id; + break; + default: + g_assert_not_reached(); + break; + } + place++; +} + +return 0; +} + +/***********************************************************************/ + +static void +osm_node_save_node(gint key, gpointer value, gpointer user_data) +{ +node *n=(node *)value; + +dbnode_cnt++; +db_insert_node(n); +if (dbnode_cnt % 26214==0) + g_printf("\rNodes: %f%%\n",((float)dbnode_cnt/(float)node_cnt)*100); +} + +/** + * Check node type and insert as POI or Place + * Discard extra data after insert. + */ +static gboolean +osm_node_save_poi(node *n, gpointer user_data) +{ +if (!n) { + g_printerr("ERROR: null poi\n"); + return FALSE; +} + +if (!n->data) { + g_printerr("POI node with no data ?\n"); + return FALSE; +} + +n->data->isin_p=osm_find_node_place(n); +n->data->isin_c=0; + +if (n->type>NODE_POI_START && n->typetype>NODE_PLACE_START && n->type0) { + g_printf("\rWays: %f%%\n",(((float)dbway_cnt/(float)way_cnt)*100)); + print_way(value); +} +} + +static void +osm_planet_clear_ways(void) +{ +g_print("Clearing old data\n"); +sqlite3_step(sql.delete_way); +sqlite3_step(sql.delete_way_name); +sqlite3_step(sql.delete_way_ref); +sqlite3_step(sql.delete_way_n2n); +} + +static gboolean +osm_planet_save_ways(void) +{ +g_print("Inserting new ways\n"); +db_transaction_begin(db); +g_slist_foreach(osm_ways, osm_way_save, NULL); +return db_transaction_commit(db); +} + +/*********************************************************************/ + +static void +osm_planet_save_all_nodes(void) +{ +g_printf("Saving planet nodes to database:\n"); + +osm_planet_poi_clear_nodes(); +osm_planet_poi_save_nodes(); + +if (!is_update) { + osm_planet_clear_nodes(); + osm_planet_clear_ways(); +} +osm_planet_save_nodes(); +} + +static void +osm_planet_save_all_ways(void) +{ +g_printf("Saving planet way to database:\n"); + +osm_planet_save_ways(); +} + +/***********************************************************************/ + +static void +_osm_tag_start(void *userData, const char *name, const char **atts) +{ +tag_state_t t; +gchar *k, *v; +guint32 id, ndref; +gdouble nlat, nlon; + +t=check_tag(name); +switch (t) { + case IN_OSM_TAG: + g_printf("Starting...\n"); + break; + case IN_NODE_TAG: + tag_parent=IS_NODE; + node_cnt++; + + id=atoi(get_attr_key_value(atts, "id")); + nlat=atof(get_attr_key_value(atts, "lat")); + nlon=atof(get_attr_key_value(atts, "lon")); + + cnode=osm_new_node(id, nlat, nlon); + osm_node_tags=g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free); + break; + case IN_WAY_TAG: + tag_parent=IS_WAY; + way_cnt++; + id=atoi(get_attr_key_value(atts, "id")); + cway=osm_new_way(id); + osm_way_tags=g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free); + break; + case IN_WNODE_TAG: + ndref=atoi(get_attr_key_value(atts, "ref")); + if (use_bbox==TRUE) { + if (osm_find_node(ndref)==NULL) { + cway->id=0; + return; + } + } + osm_way_new_node(cway, ndref); + break; + case IN_KEY_TAG: + k=get_attr_key_value(atts, "k"); + if (strcmp(k,"created_by")==0) + return; + if (strcmp(k,"source")==0) + return; + + v=get_attr_key_value(atts, "v"); +#ifdef VERBOSE_KEYS + g_printf("TAG: K=[%s] V=[%s]\n", k, v); +#endif + + switch (tag_parent) { + case IS_NONE: + g_printf("Tag key/value pair but unknown owner\n"); + break; + case IS_NODE: + { + if (!osm_node_tags) + return; + + /* Insert key/value pairs into hash table */ + if (cnode==NULL) { + g_printerr("In node tags but node is NULL!\n"); + return; + } + g_hash_table_insert(osm_node_tags, g_strdup(k), g_strdup(v)); + } + break; + case IS_WAY: + { + if (cway==NULL) { + g_printerr("In way tags but way is NULL!\n"); + return; + } + g_hash_table_insert(osm_way_tags, g_strdup(k), g_strdup(v)); + osm_new_way_data(cway); + } + break; + case IS_RELATION: + + break; + } + break; + case IN_BOUND_TAG: + /* Ignore for now */ + g_printf("Ignoring bound tag\n"); + break; + case IN_RELATION_TAG: + tag_parent=IS_RELATION; + + break; + case IN_MEMBER_TAG: + + break; + default: + tag_parent=IS_NONE; + g_printf("Unknown tag: %s\n", name); + break; +} +} + +static void +_osm_tag_end(void *userData, const char *name) +{ +tag_state_t t; +gchar *v; +guint i; +t=check_tag(name); +switch (t) { + case IN_NODE_TAG: + + if (node_cnt % 262140==0) { + g_printf("Nodes: %d of %d, POIs: %d, Outside box: %d\n", node_cnt-node_skip_cnt, node_cnt, noded_cnt, node_skip_cnt); + } + + if (!osm_node_tags) + return; + + osm_new_node_data(cnode); + + for (i=0; nodeinfo[i].k; i++) { + v=g_hash_table_lookup(osm_node_tags, nodeinfo[i].k); + if (!v) + continue; + if (strcasecmp (v, nodeinfo[i].v)==0) { + cnode->type=nodeinfo[i].type; + break; + } + } + + /* Check if node is inside bounding box, if not skip it. + * But keep it if it's something we might need for other nodes: + * - Places (for is_in) + * - ... + */ + if ((osm_node_check_box(cnode->lat, cnode->lon)==FALSE) && + (cnode->typeid), cnode); + + if (cnode->type!=NODE_PLAIN) { + cnode->data->name=NULL; + v=g_hash_table_lookup(osm_node_tags, "name"); + if (v) + cnode->data->name=g_strstrip(g_utf8_normalize(v, -1, G_NORMALIZE_ALL_COMPOSE)); + v=g_hash_table_lookup(osm_node_tags, "note"); + if (v) + cnode->data->desc=g_strstrip(g_strdup(v)); + v=g_hash_table_lookup(osm_node_tags, "postal_code"); + if (v) + cnode->data->postal_code=g_strstrip(g_strdup(v)); + + /* Links */ + v=g_hash_table_lookup(osm_node_tags, "url"); + if (v) { + cnode->data->url=g_strstrip(g_strdup(v)); + } else { + v=g_hash_table_lookup(osm_node_tags, "wikipedia"); + if (v && strncmp(v,"http:", 5)==0) + cnode->data->url=g_strstrip(g_strdup(v)); + } + } + + cnode->data->isin_c=0; + cnode->data->isin_p=0; + v=g_hash_table_lookup(osm_node_tags, "is_in"); + if (v) { + gchar **isin; + isin=g_strsplit(v, ",", 10); + g_hash_table_insert(osm_node_isin, GINT_TO_POINTER(cnode->id), isin); + } + + if (cnode->type==NODE_PLAIN) { + osm_free_node_data(cnode); + } else { + osm_poi=g_slist_prepend(osm_poi, cnode); + if (cnode->data->name) { + switch (cnode->type) { + case NODE_PLACE_COUNTRY: + g_hash_table_insert(osm_place_country, cnode->data->name, cnode); + break; + case NODE_PLACE_CITY: + case NODE_PLACE_TOWN: + g_hash_table_insert(osm_place_city, cnode->data->name, cnode); + break; + case NODE_PLACE_SUBURB: + g_hash_table_insert(osm_place_suburb, cnode->data->name, cnode); + break; + case NODE_PLACE_VILLAGE: + case NODE_PLACE_HAMLET: + case NODE_PLACE_LOCALITY: + g_hash_table_insert(osm_place_village, cnode->data->name, cnode); + break; + case NODE_PLACE_ISLAND: + /* Ignore for now */ + break; + default:; + } + } + } + g_hash_table_destroy(osm_node_tags); + cnode=NULL; + break; + case IN_WAY_TAG: + if (way_cnt % 1024==0) { + g_printf("\rWays: %d\n", way_cnt); + } + + cway->nodes=g_slist_reverse(cway->nodes); + + for (i=0; wayinfo[i].k; i++) { + v=g_hash_table_lookup(osm_way_tags, wayinfo[i].k); + if (!v) + continue; + if (strcasecmp (v, wayinfo[i].v)==0) { + if (wayinfo[i].link==TRUE) + cway->flags|=W_LINK; + if (wayinfo[i].area==TRUE) + cway->flags|=W_AREA; + if (wayinfo[i].oneway==TRUE) + cway->flags|=W_ONEWAY; + cway->type=wayinfo[i].type; + if (cway->data->speed==0) + cway->data->speed=wayinfo[i].defspeed; + break; + } + } + + v=g_hash_table_lookup(osm_way_tags, "name"); + if (v) { + cway->data->name=g_utf8_normalize(v, -1, G_NORMALIZE_ALL_COMPOSE); + /* Try to find other language names */ + cway->data->names=g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free); + g_hash_table_foreach(osm_way_tags, find_nls_names, cway->data->names); + if (g_hash_table_size(cway->data->names)==0) { + g_hash_table_destroy(cway->data->names); + cway->data->names=NULL; + } + } + + v=g_hash_table_lookup(osm_way_tags, "ref"); + if (v) + cway->data->ref=g_strdup(v); + v=g_hash_table_lookup(osm_way_tags, "int_ref"); + if (v) + cway->data->int_ref=g_strdup(v); + + v=g_hash_table_lookup(osm_way_tags, "postal_code"); + if (v) + cway->data->postal_code=g_strdup(v); + + /* XXX: somehow handle the silly -1 'reversed' oneway */ + v=g_hash_table_lookup(osm_way_tags, "oneway"); + if (v) + cway->flags|=W_ONEWAY; + + v=g_hash_table_lookup(osm_way_tags, "noexit"); + if (v) + cway->flags|=W_NOEXIT; + + v=g_hash_table_lookup(osm_way_tags, "speedlimit"); + if (v) + cway->data->speed=atoi(v); + v=g_hash_table_lookup(osm_way_tags, "maxspeed"); + if (v) + cway->data->speed=atoi(v); + +#if 0 + v=g_hash_table_lookup(osm_way_tags, "layer"); + if (v) + cway->data->layer=atoi(v); +#endif + + v=g_hash_table_lookup(osm_way_tags, "junction"); + if (v && strcasecmp(v,"roundabout")==0) { + cway->flags|=W_ROUNDABOUT; + cway->flags|=W_ONEWAY; + } else if (v && strcasecmp(v,"mini_roundabout")==0) { + cway->flags|=W_ROUNDABOUT; + cway->flags|=W_ONEWAY; + } + + /* XXX: Should check keys */ + v=g_hash_table_lookup(osm_way_tags, "access"); + if (v && (strcasecmp(v, "private")==0)) { + cway->flags|=W_NOACCESS; + } + + print_way(cway); + + v=g_hash_table_lookup(osm_way_tags, "is_in"); + if (v) { + gchar **isin; + isin=g_strsplit(v, ",", 10); + g_hash_table_insert(osm_way_isin, GINT_TO_POINTER(cway->id), isin); + } + + if (cway->data && cway->data->name==NULL && cway->data->ref==NULL && + cway->data->int_ref==NULL && cway->data->layer==0 && cway->data->speed==0) + osm_free_way_data(cway); + + if (cway->id!=0) + osm_way_add_to_list(cway); + + cway=NULL; + g_hash_table_destroy(osm_way_tags); + break; + case IN_BOUND_TAG: + /* */ + break; + case IN_OSM_TAG: + g_printf("\nPlanet loaded.\n"); + break; + default:; +} +} + +/************************************************************************/ + +static void +storage_init(void) +{ +osm_nodes=g_hash_table_new(g_direct_hash, g_direct_equal); + +osm_place_country=g_hash_table_new(g_str_hash, g_str_equal); +osm_place_city=g_hash_table_new(g_str_hash, g_str_equal); +osm_place_suburb=g_hash_table_new(g_str_hash, g_str_equal); +osm_place_village=g_hash_table_new(g_str_hash, g_str_equal); +osm_place_region=g_hash_table_new(g_str_hash, g_str_equal); +osm_node_isin=g_hash_table_new(g_direct_hash, g_direct_equal); +osm_way_isin=g_hash_table_new(g_direct_hash, g_direct_equal); +} + +static void +storage_free(void) +{ +g_hash_table_destroy(osm_nodes); + +g_hash_table_destroy(osm_place_country); +g_hash_table_destroy(osm_place_city); +g_hash_table_destroy(osm_place_suburb); +g_hash_table_destroy(osm_place_village); +g_hash_table_destroy(osm_place_region); +g_hash_table_destroy(osm_node_isin); +} + + +/************************************************************************/ + +static gint +print_fail(const gchar *msg, gint ret) +{ +g_printerr("ERROR: %s\n", msg); +return ret; +} + +/************************************************************************/ + +static void +print_memory_usage(void) +{ +g_print("Memory usage per item:\n"); +g_printf("Node size: %d\n", (gint)sizeof(node)); +g_printf("NodeD size: %d\n", (gint)sizeof(node_data)); +g_printf("Way size: %d\n", (gint)sizeof(way)); +g_printf("WayD size: %d\n", (gint)sizeof(way_data)); +} + +/************************************************************************ + * Public inteface + ************************************************************************/ + +void +osm_planet_parser_init(void) +{ +xp=XML_ParserCreate(NULL); +XML_SetElementHandler(xp, _osm_tag_start, _osm_tag_end); +storage_init(); +} + +void +osm_planet_parser_deinit(void) +{ +XML_ParserFree(xp); +storage_free(); +} + +gboolean +osm_planet_parse_buffer(gchar *buffer, size_t r) +{ +if (XML_Parse(xp, buffer, r, r>0 ? 0:1) == XML_STATUS_ERROR) { + g_printerr("Parse error at line %d:\n%s\n", + XML_GetCurrentLineNumber(xp), + XML_ErrorString(XML_GetErrorCode(xp))); + return FALSE; +} +return TRUE; +} + +gboolean +osm_planet_parse_file(gchar *pfile) +{ +FILE *f; +BZFILE *b; +int bzerror; +int r; +gchar buffer[FILE_BUFFER]; +gboolean res=TRUE; + +f=fopen(pfile, "r"); +if (!f) { + perror("fopen failed\n"); + return FALSE; +} + +b=BZ2_bzReadOpen(&bzerror, f, 0, 0, NULL, 0); +if (bzerror != BZ_OK) { + g_printf("BZ2_bzReadOpen failed\n"); + BZ2_bzReadClose(&bzerror, b); + return FALSE; +} + +do { + r=BZ2_bzRead(&bzerror, b, buffer, FILE_BUFFER); + if ((bzerror!=BZ_STREAM_END) && (bzerror!=BZ_OK)) { + res=FALSE; + break; + } + if (!osm_planet_parse_buffer(buffer, r)) { + res=FALSE; + break; + } +} while (bzerror==BZ_OK); + +BZ2_bzReadClose(&bzerror, b); +fclose(f); +return res; +} + +void +osm_import_set_bbox(gboolean use_bb, gdouble latmin, gdouble lonmin, gdouble latmax, gdouble lonmax) +{ +use_bbox=use_bb; +bbox.lat_min=latmin; +bbox.lon_min=lonmin; +bbox.lat_max=latmax; +bbox.lon_max=lonmax; +g_printf("Skipping data outside of box: %f,%f - %f,%f\n", + bbox.lat_min, bbox.lon_min, bbox.lat_max, bbox.lon_max); +} + +gboolean +osm_import(const gchar *planet, const gchar *database) +{ +if (db_connect(&db, database)!=TRUE) { + g_printerr("Database open failed: %s", database); + return FALSE; +} + +db_create_tables(db); +db_create_indexes(db); +db_prepare(db); + +osm_planet_parser_init(); + +if (osm_planet_parse_file(planet)==FALSE) { + g_printerr("Failed to parse file: %s\n", planet); + return FALSE; +} + +g_printf("Total nodes %d, POIs: %d and Ways %d.\n", node_cnt, noded_cnt, way_cnt); +g_printf("Cities/Towns: %d\n", g_hash_table_size(osm_place_city)); +g_printf("Villages/Hamlets: %d\n", g_hash_table_size(osm_place_village)); +g_printf("Suburbs: %d\n", g_hash_table_size(osm_place_suburb)); +g_printf("Nodes: %d\n", g_hash_table_size(osm_nodes)); + +osm_planet_save_all_nodes(); +osm_planet_save_all_ways(); +osm_planet_parser_deinit(); +db_close(&db); +return TRUE; +} + diff --git a/src/osm-db-import.h b/src/osm-db-import.h new file mode 100644 index 0000000..42c0059 --- /dev/null +++ b/src/osm-db-import.h @@ -0,0 +1,62 @@ +#ifndef _OSM_DB_IMPORT_H +#define _OSM_DB_IMPORT_H + +#include + +/* POI or Place node extra data */ +typedef struct _node_data node_data; +struct _node_data { + gchar *name; + gchar *desc; + gchar *url; + gchar *postal_code; + guint32 isin_c; + guint32 isin_p; +}; + +/* Node type */ +typedef struct _node node; +struct _node { + guint32 id; + gdouble lat; + gdouble lon; + node_type_t type; + node_data *data; +}; + +/* Way data structure */ +typedef struct _way_data way_data; +struct _way_data { + gchar *name; + GHashTable *names; + gchar *ref; + gchar *int_ref; + gchar *postal_code; + guint32 isin_c; /* Country */ + guint32 isin_p; /* Primary (city, village) place */ + guint speed; + gint8 layer; +}; + +/* Way structure */ +typedef struct _way way; +struct _way { + guint32 id; + way_type_t type; + guint16 ncnt; + guint8 flags; + way_data *data; + GSList *nodes; +}; + +struct map_bbox { + gdouble lat_min; + gdouble lon_min; + gdouble lat_max; + gdouble lon_max; +}; + +void osm_import_set_bbox(gboolean use_bb, gdouble latmin, gdouble lonmin, gdouble latmax, gdouble lonmax); +gboolean osm_import(const gchar *planet, const gchar *database); + +#endif diff --git a/src/osm-db.c b/src/osm-db.c index 98e41a1..9b9be2e 100644 --- a/src/osm-db.c +++ b/src/osm-db.c @@ -1100,5 +1100,3 @@ sqlite3_clear_bindings(sql.select_way_name_search); return TRUE; } - - diff --git a/src/osm.c b/src/osm.c index 3c945f4..9b0511c 100644 --- a/src/osm.c +++ b/src/osm.c @@ -19,1719 +19,23 @@ */ /* - * Utility to read OSM planet XML file and store it in a sqlite3 database. - * Reads in all nodes (if used, skips nodes outside bounding box) - * Special POI nodes are stored in POI table. - * Place POI nodes are stored in place table. - * - * Ways are read in and their data (name, type, etc) are stored - * in way, way_name and way_ref tables. - * - * Nodes used by they ways are stored in way_n2n table. - * - * ... - * + * A simple CLI interface to the mapper planet2sqlite import code. */ #include #include +#include #include -#include -#include -#include -#include -#include #include #include -#include -#include -#include #include "osm.h" #include "latlon.h" #include "db.h" - - -#if 0 -#define VERBOSE -#endif -/* #define VERBOSE_KEYS */ - - -/* Use g_convert to transliterate names.. my iconv seems to be fucked so this didn't work... */ -/* #define TRANSLIT_NAMES */ - -#define FILE_BUFFER 65535 +#include "osm-db-import.h" #define OSM_DB_FILE "osm-planet.db" -static guint node_cnt=0; -static guint node_skip_cnt=0; -static guint noded_cnt=0; -static guint way_cnt=0; -static guint way_names=0; -static guint way_refs=0; - -static guint dbnode_cnt=0; -static guint dbnoded_cnt=0; -static guint dbway_cnt=0; - -static gboolean is_update=FALSE; -static XML_Parser xp; - -/* POI or Place node extra data */ -typedef struct _node_data node_data; -struct _node_data { - gchar *name; - gchar *desc; - gchar *url; - gchar *postal_code; - guint32 isin_c; - guint32 isin_p; -}; - -/* Node type */ -typedef struct _node node; -struct _node { - guint32 id; - gdouble lat; - gdouble lon; - node_type_t type; - node_data *data; -}; - -/* Way data structure */ -typedef struct _way_data way_data; -struct _way_data { - gchar *name; - GHashTable *names; - gchar *ref; - gchar *int_ref; - gchar *postal_code; - guint32 isin_c; /* Country */ - guint32 isin_p; /* Primary (city, village) place */ - guint speed; - gint8 layer; -}; - -/* Way structure */ -typedef struct _way way; -struct _way { - guint32 id; - way_type_t type; - guint16 ncnt; - guint8 flags; - way_data *data; - GSList *nodes; -}; - -/* XML tag IDs */ -typedef enum { - START, - IN_OSM_TAG, - IN_NODE_TAG, - IN_WNODE_TAG, - IN_WAY_TAG, - IN_KEY_TAG, - IN_BOUND_TAG, - IN_RELATION_TAG, - IN_MEMBER_TAG, - END, - ERROR -} tag_state_t; - -/* Parent tag type */ -typedef enum { - IS_NONE, - IS_NODE, - IS_WAY, - IS_RELATION -} tag_parent_t; - -/* Node types table */ -/* XXX: Add support for parent category */ -struct _nodeinfo { - gchar *k, *v; - node_type_t type; -} nodeinfo[] = { - { "amenity", "fuel", NODE_AMENITY_FUEL }, - { "amenity", "parking", NODE_AMENITY_PARKING }, - - { "amenity", "pub", NODE_AMENITY_PUB }, - { "amenity", "nightclub", NODE_AMENITY_NIGHTCLUB }, - { "amenity", "biergarten", NODE_AMENITY_PUB }, - { "amenity", "cafe", NODE_AMENITY_CAFE }, - { "amenity", "fast_food", NODE_AMENITY_FOOD }, - { "amenity", "restaurant", NODE_AMENITY_FOOD }, - - { "amenity", "telephone", NODE_AMENITY_TELEPHONE }, - { "amenity", "toilets", NODE_AMENITY_WC }, - - { "amenity", "hospital", NODE_AMENITY_HOSPITAL }, - { "amenity", "doctors", NODE_AMENITY_HOSPITAL }, - { "amenity", "pharmacy", NODE_AMENITY_PHARMACY }, - - { "amenity", "post_office", NODE_AMENITY_POST }, - { "amenity", "post_box", NODE_AMENITY_POST_BOX }, - - { "amenity", "cinema", NODE_AMENITY_CINEMA }, - { "amenity", "theatre", NODE_AMENITY_THEATRE }, - - { "amenity", "atm", NODE_AMENITY_ATM }, - { "amenity", "bank", NODE_AMENITY_BANK }, - - { "amenity", "police", NODE_AMENITY_POLICE }, - { "amenity", "speed_trap", NODE_AMENITY_SPEEDCAM }, - { "amenity", "speed_camera", NODE_AMENITY_SPEEDCAM }, - { "amenity", "speed camera", NODE_AMENITY_SPEEDCAM }, - - { "amenity", "place_of_worship",NODE_AMENITY_POW }, - - { "amenity", "school", NODE_AMENITY_SCHOOL }, - { "amenity", "college", NODE_AMENITY_COLLEGE }, - { "amenity", "university", NODE_AMENITY_COLLEGE }, - - { "amenity", "library", NODE_AMENITY_LIBRARY }, - { "amenity", "townhall", NODE_AMENITY_TOWNHALL }, - - { "amenity", "supermarket", NODE_AMENITY_SHOP }, - { "amenity", "shopping_centre", NODE_AMENITY_SHOP }, - { "amenity", "shop", NODE_AMENITY_SHOP }, - { "amenity", "shops", NODE_AMENITY_SHOP }, - { "amenity", "shopping", NODE_AMENITY_SHOP }, - { "amenity", "shopping_mall",NODE_AMENITY_SHOP }, - { "amenity", "cycle_shop", NODE_AMENITY_SHOP }, - { "amenity", "bike_shop", NODE_AMENITY_SHOP }, - { "amenity", "coffee_shop", NODE_AMENITY_SHOP }, - { "amenity", "indoor_shopping_centre", NODE_AMENITY_SHOP }, - { "amenity", "farm_shop", NODE_AMENITY_SHOP }, - { "amenity", "tea_shop", NODE_AMENITY_SHOP }, - - /* Shops */ - { "shop", "supermarket", NODE_AMENITY_SHOP }, - { "shop", "bakery", NODE_AMENITY_SHOP }, - { "shop", "alcohol", NODE_AMENITY_SHOP }, - { "shop", "butcher", NODE_AMENITY_SHOP }, - { "shop", "flowers", NODE_AMENITY_SHOP }, - { "shop", "clothing", NODE_AMENITY_SHOP }, - { "shop", "souvenir", NODE_AMENITY_SHOP }, - { "shop", "bicycles", NODE_AMENITY_SHOP }, - { "shop", "grocers", NODE_AMENITY_SHOP }, - { "shop", "newsagents", NODE_AMENITY_SHOP }, - { "shop", "convenience", NODE_AMENITY_SHOP }, - { "shop", "bakers", NODE_AMENITY_SHOP }, - { "shop", "garden_centre",NODE_AMENITY_SHOP }, - { "shop", "photography", NODE_AMENITY_SHOP }, - { "shop", "general_store",NODE_AMENITY_SHOP }, - { "shop", "food", NODE_AMENITY_SHOP }, - { "shop", "drinks", NODE_AMENITY_SHOP }, - { "shop", "pharmacy", NODE_AMENITY_PHARMACY }, - - /* Sport */ - { "sport" , "swimming", NODE_SPORT_SWIMMING }, - { "sport" , "golf", NODE_SPORT_GOLF }, - { "sport" , "tennis", NODE_SPORT_TENNIS }, - { "sport" , "football", NODE_SPORT_FOOTBALL }, - { "sport" , "soccer", NODE_SPORT_SOCCER }, - { "sport" , "baskteball", NODE_SPORT_BASKETBALL }, - { "sport" , "rugby", NODE_SPORT_RUGBY }, - { "sport" , "skating", NODE_SPORT_SKATING }, - { "sport" , "hockey", NODE_SPORT_HOCKEY }, - { "sport" , "skateboard", NODE_SPORT_SKATEBOARD }, - { "sport" , "bowling", NODE_SPORT_BOWLING }, - { "sport" , "10pin", NODE_SPORT_BOWLING }, - { "sport" , "motor", NODE_SPORT_MOTOR }, - { "sport" , "shooting_range",NODE_SPORT_SHOOTING }, - { "sport" , "paintball", NODE_SPORT_PAINTBALL }, - { "sport" , "horse_racing",NODE_SPORT_HORSES }, - { "sport" , "horse", NODE_SPORT_HORSES }, - { "sport" , "horses", NODE_SPORT_HORSES }, - { "sport" , "dog_racing", NODE_SPORT_DOG }, - { "sport" , "pelota", NODE_SPORT_PELOTA }, - { "sport" , "racquet", NODE_SPORT_RACQUET }, - { "sport" , "equestrian", NODE_SPORT_HORSES }, - { "sport" , "baseball", NODE_SPORT_BASEBALL }, - { "sport" , "cricket", NODE_SPORT_CRICKET }, - { "sport" , "croquet", NODE_SPORT_CROQUET }, - { "sport" , "cycling", NODE_SPORT_CYCLING }, - { "sport" , "bowls", NODE_SPORT_BOWLS }, - { "sport" , "athletics", NODE_SPORT_ATHLETICS }, - { "sport" , "gymnastics", NODE_SPORT_GYMNASTICS }, - { "sport" , "multi", NODE_SPORT_OTHER }, - { "leisure", "sport_centre",NODE_SPORT_CENTER }, - - /* Tourism */ - { "tourism", "information", NODE_TOURISM_INFO }, - { "tourism", "camp_site", NODE_TOURISM_CAMP_SITE }, - { "tourism", "caravan_site",NODE_TOURISM_CARAVAN_SITE }, - { "tourism", "picnic_site", NODE_TOURISM_PICNIC_SITE }, - { "tourism", "theme_park", NODE_TOURISM_THEME_PARK }, - { "tourism", "hotel", NODE_TOURISM_HOTEL }, - { "tourism", "motel", NODE_TOURISM_MOTEL }, - { "tourism", "hostel", NODE_TOURISM_HOSTEL }, - { "tourism", "attraction", NODE_TOURISM_ATTRACTION }, - { "tourism", "zoo", NODE_TOURISM_ATTRACTION }, - - { "historic", "ruins", NODE_TOURISM_ATTRACTION }, - { "historic", "monument", NODE_TOURISM_ATTRACTION }, - { "historic", "memorial", NODE_TOURISM_ATTRACTION }, - { "historic", "museum", NODE_HISTORIC_MUSEUM }, - { "historic", "castle", NODE_HISTORIC_CASTLE }, - - { "railway", "station", NODE_RAILWAY_STATION }, - { "railway", "halt", NODE_RAILWAY_HALT }, - - { "aeroway", "terminal", NODE_AIRPORT_TERMINAL }, - - /* Places */ - { "place", "city", NODE_PLACE_CITY }, - { "place", "town", NODE_PLACE_TOWN }, - { "place", "village", NODE_PLACE_VILLAGE }, - { "place", "hamlet", NODE_PLACE_HAMLET }, - { "place", "locality", NODE_PLACE_LOCALITY }, - { "place", "suburb", NODE_PLACE_SUBURB }, - { "place", "island", NODE_PLACE_ISLAND }, - - { "highway", "traffic_signals", NODE_TRAFFIC_SIGNALS }, - { "highway", "motorway_junction", NODE_JUNCTION }, - { "highway", "services", NODE_AMENITY_PARKING }, - { "highway", "toll_booth", NODE_TOLLBOOTH }, - { "highway", "gate", NODE_GATE }, - - { NULL, NULL, NODE_PLAIN } -}; - -/* Array to get id number and defaults for ways of different types */ -struct _wayinfo { - gchar *k, *v; - guint defspeed; - way_type_t type; - gboolean oneway, link, area, car, foot; -} wayinfo[] = { - { "highway", "motorway",120,WAY_MOTORWAY, TRUE, FALSE, FALSE, TRUE, FALSE }, - { "highway", "motorway_link",120,WAY_MOTORWAY, TRUE, TRUE, FALSE, TRUE, FALSE }, - { "highway", "trunk",100,WAY_TRUNK, FALSE, FALSE, FALSE, TRUE, FALSE }, - { "highway", "trunk_link",100,WAY_TRUNK, FALSE, TRUE, FALSE, TRUE, FALSE }, - { "highway", "primary",80,WAY_PRIMARY, FALSE, FALSE, FALSE, TRUE, TRUE }, - { "highway", "primary_link",60,WAY_PRIMARY, FALSE, TRUE, FALSE, TRUE, TRUE }, - { "highway", "secondary",80,WAY_SECONDARY, FALSE, FALSE, FALSE, TRUE, TRUE }, - { "highway", "secondary_link",60,WAY_SECONDARY, FALSE, TRUE, FALSE, TRUE, TRUE }, - { "highway", "tertiary",60,WAY_TERTIARY, FALSE, FALSE, FALSE, TRUE, TRUE }, - { "highway", "unclassified",50,WAY_UNCLASSIFIED, FALSE, FALSE, FALSE, TRUE, TRUE }, - { "highway", "byway",40,WAY_UNCLASSIFIED, FALSE, FALSE, FALSE, TRUE, TRUE }, - { "highway", "residential",40,WAY_RESIDENTIAL, FALSE, FALSE, FALSE, TRUE, TRUE }, - { "highway", "service",20,WAY_SERVICE, FALSE, FALSE, FALSE, TRUE, TRUE }, - { "highway", "track",20,WAY_TRACK, FALSE, FALSE, FALSE, TRUE, TRUE }, - { "highway", "unsurfaced",60,WAY_TRACK, FALSE, FALSE, FALSE, TRUE, TRUE }, - { "highway", "minor",60,WAY_TRACK, FALSE, FALSE, FALSE, TRUE, TRUE }, - { "highway", "pedestrian",20,WAY_FOOTWAY, FALSE, FALSE, FALSE, FALSE, TRUE }, - { "highway", "footway",1,WAY_FOOTWAY, FALSE, FALSE, FALSE, FALSE, TRUE }, - { "highway", "steps",0,WAY_FOOTWAY, FALSE, FALSE, FALSE, FALSE, TRUE}, - { "highway", "bridleway",10,WAY_FOOTWAY, FALSE, FALSE, FALSE, FALSE, TRUE }, - { "highway", "cycleway",10,WAY_CYCLEWAY, FALSE, FALSE, FALSE, FALSE, TRUE }, - { "railway", "rail",0,WAY_RAIL, FALSE, FALSE, FALSE, FALSE, FALSE }, - { "aeroway", "runway",0,WAY_RUNWAY, FALSE, FALSE, FALSE, FALSE, FALSE }, - { "aeroway", "taxiway",0,WAY_TAXIWAY, FALSE, FALSE, FALSE, FALSE, FALSE }, - { "natural", "water",0,WAY_WATER, FALSE, FALSE, TRUE, FALSE, FALSE }, - { "waterway", "river",0,WAY_WATER, FALSE, FALSE, FALSE, FALSE, FALSE }, - { "waterway", "canal",0,WAY_WATER, FALSE, FALSE, FALSE, FALSE, FALSE }, - { "waterway", "stream",0,WAY_WATER, FALSE, FALSE, FALSE, FALSE, FALSE }, - { "building", "*",0,WAY_UNWAYED, FALSE, FALSE, TRUE, FALSE, FALSE }, - { NULL, NULL, 0, WAY_UNWAYED, FALSE, FALSE, FALSE, FALSE, FALSE } -}; - -static sqlite3 *db; -tag_parent_t tag_parent=IS_NONE; - -static GHashTable *osm_nodes; -static GHashTable *osm_node_tags; -static GHashTable *osm_way_tags; -static GSList *osm_ways; -static GSList *osm_poi; - -static GHashTable *osm_place_country; -static GHashTable *osm_place_region; -static GHashTable *osm_place_city; -static GHashTable *osm_place_suburb; -static GHashTable *osm_place_village; -static GHashTable *osm_node_isin; -static GHashTable *osm_way_isin; - -static node *cnode=NULL; -static way *cway=NULL; - -struct sql_stmt { - sqlite3_stmt *insert_poi; - sqlite3_stmt *delete_osm_poi; - - sqlite3_stmt *insert_node; - sqlite3_stmt *delete_nodes; - sqlite3_stmt *select_node; - sqlite3_stmt *update_node; - - sqlite3_stmt *insert_way_data; - sqlite3_stmt *insert_way_ref; - sqlite3_stmt *insert_way_pc; - sqlite3_stmt *insert_way_name; - sqlite3_stmt *insert_way_names_nls; - sqlite3_stmt *insert_way_n2n; - sqlite3_stmt *delete_way; - sqlite3_stmt *delete_way_n2n; - sqlite3_stmt *delete_way_name; - sqlite3_stmt *delete_way_names_nls; - sqlite3_stmt *delete_way_ref; - sqlite3_stmt *delete_way_pc; - - sqlite3_stmt *insert_place; - sqlite3_stmt *delete_place; -}; -static struct sql_stmt sql; - -struct map_bbox { - gdouble lat_min; - gdouble lon_min; - gdouble lat_max; - gdouble lon_max; -}; -static struct map_bbox bbox; -static gboolean use_bbox; - -void osm_free_way_data(way *w); -void print_way(way *w); - -void db_prepare(sqlite3 *db); -gboolean db_insert_node(node *n); -guint32 osm_find_way_place(way *w, node_type_t nt); - -/****************************************************/ -/* Functions */ -/****************************************************/ - -static void -db_finalize(void) -{ -sqlite3_finalize(sql.insert_poi); -sqlite3_finalize(sql.delete_osm_poi); - -sqlite3_finalize(sql.insert_node); -sqlite3_finalize(sql.select_node); -sqlite3_finalize(sql.delete_nodes); -sqlite3_finalize(sql.update_node); - -sqlite3_finalize(sql.insert_place); -sqlite3_finalize(sql.delete_place); - -sqlite3_finalize(sql.delete_way); -sqlite3_finalize(sql.insert_way_data); - -sqlite3_finalize(sql.delete_way_name); -sqlite3_finalize(sql.insert_way_name); - -sqlite3_finalize(sql.delete_way_n2n); -sqlite3_finalize(sql.insert_way_n2n); - -sqlite3_finalize(sql.delete_way_pc); -sqlite3_finalize(sql.insert_way_pc); - -sqlite3_finalize(sql.delete_way_names_nls); -sqlite3_finalize(sql.insert_way_names_nls); -} - - - -void -db_prepare(sqlite3 *db) -{ -/* Way nodes */ -sqlite3_prepare_v2(db, "insert or replace into nodes (nid,ilat,ilon,rlat,rlon,l,f) values (?,?,?,?,?,0,?)", -1, &sql.insert_node, NULL); -sqlite3_prepare_v2(db, "select ilat,ilon,l from nodes where nid=?", -1, &sql.select_node, NULL); -sqlite3_prepare_v2(db, "delete from nodes", -1, &sql.delete_nodes, NULL); -sqlite3_prepare_v2(db, "update nodes set l=l+1 where nid=?", -1, &sql.update_node, NULL); - -/* Places */ -sqlite3_prepare_v2(db, "insert or replace into places (nid,type,name,isin_c,isin_p) values (?, ?, ?, ?, ?)", -1, &sql.insert_place, NULL); -sqlite3_prepare_v2(db, "delete from places", -1, &sql.delete_place, NULL); - -/* POI nodes */ -if (sqlite3_prepare_v2(db, "insert or replace into poi (osm_id, lat, lon, label, cat_id, public, source, priority, isin_c, isin_p, desc, url, postal_code) " - " values (?, ?, ?, ?, ?, 1, 1, ?, ?, ?, ?, ?, ?)", -1, &sql.insert_poi, NULL)!=SQLITE_OK) - g_printf("SQL: %s\n", sqlite3_errmsg(db)); - -sqlite3_prepare_v2(db, "delete from poi where osm_id>0 and source=1", -1, &sql.delete_osm_poi, NULL); - -/* Ways */ -sqlite3_prepare_v2(db, "insert or replace into way (wid,nodes,type,flags,speed,isin_c,isin_p,lat,lon) values (?, ?, ?, ?, ?, ?, ?, ?, ?)", -1, &sql.insert_way_data, NULL); -sqlite3_prepare_v2(db, "delete from way", -1, &sql.delete_way, NULL); - -/* Way nodes */ -sqlite3_prepare_v2(db, "insert into way_n2n (wid,f,t) values (?,?,?)", -1, &sql.insert_way_n2n, NULL); -sqlite3_prepare_v2(db, "delete from way_n2n where wid=?", -1, &sql.delete_way_n2n, NULL); - -/* Way names */ -sqlite3_prepare_v2(db, "insert or replace into way_names (wid,name,norm) values (?, ?, ?)", -1, &sql.insert_way_name, NULL); -sqlite3_prepare_v2(db, "delete from way_names", -1, &sql.delete_way_name, NULL); - -/* Way postal codes */ -sqlite3_prepare_v2(db, "insert or replace into way_pc (wid,pc) values (?, ?)", -1, &sql.insert_way_pc, NULL); -sqlite3_prepare_v2(db, "delete from way_pc", -1, &sql.delete_way_pc, NULL); - -/* Other language names for ways */ -sqlite3_prepare_v2(db, "insert into way_names_nls (wid,lang,name, norm) values (?, ?, ?, ?)", -1, &sql.insert_way_names_nls, NULL); -sqlite3_prepare_v2(db, "delete from way_names_nls where wid=?", -1, &sql.delete_way_names_nls, NULL); - -/* Way ref and int_ref */ -sqlite3_prepare_v2(db, "insert or replace into way_ref (rid,ref,int_ref) values (?, ?, ?)", -1, &sql.insert_way_ref, NULL); -sqlite3_prepare_v2(db, "delete from way_ref", -1, &sql.delete_way_ref, NULL); -} - -/********************************************************************/ - -void -print_way(way *w) -{ -#ifdef VERBOSE -g_assert(w); -g_printf("Way #%d(N:%d T:%d S:%d IS: %d/%d): %s [%s:%s:%s]\n", - w->id, - g_slist_length(w->nodes), - w->type, - w->data ? w->data->speed : 0, - w->data ? w->data->isin_c : -1, - w->data ? w->data->isin_p : -1, - w->data ? w->data->name ? w->data->name : "" : "", - w->flags & W_ONEWAY ? "-" : "=", - w->flags & W_ROUNDABOUT ? "O" : "-", - w->flags & W_LINK ? "|" : " "); -#endif -} - -void -print_node(node *n) -{ -#ifdef VERBOSE -g_assert(n); -g_printf("Node #%d: T:%d IS: %d/%d [%s]\n", - n->id, - n->type, - n->data ? n->data->isin_c : -1, - n->data ? n->data->isin_p : -1, - n->data ? n->data->name : ""); -#endif -} - -/********************************************************************/ - -gboolean -db_insert_node(node *n) -{ -gint32 lat, lon; - -g_assert(n); - -lat=lat2mp_int(n->lat); -lon=lon2mp_int(n->lon); - -sqlite3_bind_int(sql.insert_node, 1, n->id); - -/* Projected and integerized lat/lot */ -sqlite3_bind_int(sql.insert_node, 2, lat); -sqlite3_bind_int(sql.insert_node, 3, lon); -/* Original */ -sqlite3_bind_double(sql.insert_node, 4, n->lat); -sqlite3_bind_double(sql.insert_node, 5, n->lon); -sqlite3_bind_int(sql.insert_node, 6, n->type); - -db_exec(db, sql.insert_node); - -return TRUE; -} - -static gboolean -db_insert_place(node *n) -{ -g_assert(n); -if (!n->data) - return FALSE; -if (!n->data->name) - return FALSE; -sqlite3_bind_int(sql.insert_place, 1, n->id); -sqlite3_bind_int(sql.insert_place, 2, n->type); -sqlite3_bind_text(sql.insert_place, 3, n->data->name, -1, SQLITE_TRANSIENT); -sqlite3_bind_int(sql.insert_place, 4, n->data->isin_p); -sqlite3_bind_int(sql.insert_place, 5, n->data->isin_c); - -return db_exec(db,sql.insert_place); -} - -static gboolean -db_insert_poi(node *n) -{ -g_assert(n); -sqlite3_bind_int(sql.insert_poi, 1, n->id); -sqlite3_bind_double(sql.insert_poi, 2, n->lat); -sqlite3_bind_double(sql.insert_poi, 3, n->lon); -if (n->data->name) - sqlite3_bind_text(sql.insert_poi, 4, n->data->name, -1, SQLITE_TRANSIENT); -else - sqlite3_bind_text(sql.insert_poi, 4, "", -1, SQLITE_TRANSIENT); -sqlite3_bind_int(sql.insert_poi, 5, n->type); -sqlite3_bind_int(sql.insert_poi, 6, n->type/100); -sqlite3_bind_int(sql.insert_poi, 7, n->data->isin_c); -sqlite3_bind_int(sql.insert_poi, 8, n->data->isin_p); - -if (n->data->desc) - sqlite3_bind_text(sql.insert_poi, 9, n->data->desc, -1, SQLITE_TRANSIENT); -if (n->data->url) - sqlite3_bind_text(sql.insert_poi, 10, n->data->url, -1, SQLITE_TRANSIENT); -if (n->data->postal_code) - sqlite3_bind_text(sql.insert_poi, 11, n->data->postal_code, -1, SQLITE_TRANSIENT); - -return db_exec(db,sql.insert_poi); -} - -/** - * Update node usage count - */ -static gboolean -db_update_node_links(node *n) -{ -g_assert(n); -sqlite3_bind_int(sql.update_node, 1, n->id); - -return db_exec(db,sql.update_node); -} - -/** - * Insert way,node1,node2 triplet - */ -static gboolean -db_insert_way_n2n(way *w, node *nf, node *nt) -{ -if (!w) { - g_printf("NULL WAY\n"); - return FALSE; -} - -if (!nf) { - g_printf("NULL NODE 1\n"); - return FALSE; -} - -if (!nt) { - g_printf("NULL NODE 2\n"); - return FALSE; -} - -sqlite3_bind_int(sql.insert_way_n2n, 1, w->id); -sqlite3_bind_int(sql.insert_way_n2n, 2, nf->id); -sqlite3_bind_int(sql.insert_way_n2n, 3, nt->id); - -#ifdef VERBOSE_N2N -g_printf("%d [%d - %d]\n", w->id, nf->id, nt->id); -#endif - -db_exec(db,sql.insert_way_n2n); -db_update_node_links(nf); -db_update_node_links(nt); -return TRUE; -} - -/** - * Insert way ref and int_ref - */ -static void -db_insert_way_ref(way *w) -{ -if (!w->data) - return; - -if (!w->data->ref && !w->data->int_ref) - return; - -way_refs++; - -sqlite3_bind_int(sql.insert_way_ref, 1, w->id); -if (w->data->ref) - sqlite3_bind_text(sql.insert_way_ref, 2, w->data->ref, -1, SQLITE_TRANSIENT); -if (w->data->int_ref) - sqlite3_bind_text(sql.insert_way_ref, 3, w->data->int_ref, -1, SQLITE_TRANSIENT); - -db_exec(db,sql.insert_way_ref); -} - -/** - * Insert way name - */ -static void -db_insert_way_name(way *w) -{ -gchar *norm; - -if (!w->data) - return; -if (!w->data->name) - return; - -way_names++; - -sqlite3_bind_int(sql.insert_way_name, 1, w->id); -sqlite3_bind_text(sql.insert_way_name, 2, w->data->name, -1, SQLITE_TRANSIENT); - -#ifdef TRANSLIT_NAMES -norm=g_convert(w->data->name, -1, "ASCII//TRANSLIT//IGNORE", "utf8", NULL, NULL, NULL); -if (norm && strcmp(w->data->name, norm)!=0) { - sqlite3_bind_text(sql.insert_way_name, 3, norm, -1, SQLITE_TRANSIENT); -} -if (norm) - g_free(norm); -#endif - -db_exec(db,sql.insert_way_name); -} - -static void -db_delete_way_names_nls(way *w) -{ -sqlite3_bind_int(sql.delete_way_names_nls, 1, w->id); -db_exec(db,sql.delete_way_names_nls); -} - -static void -db_insert_way_pc(way *w) -{ -if (!w->data) - return; -if (!w->data->postal_code) - return; - -sqlite3_bind_int(sql.insert_way_pc, 1, w->id); -sqlite3_bind_text(sql.insert_way_pc, 2, w->data->postal_code, -1, SQLITE_TRANSIENT); - -db_exec(db,sql.insert_way_pc); -} - -static void -db_delete_way_pc(way *w) -{ -sqlite3_bind_int(sql.delete_way_pc, 1, w->id); -db_exec(db,sql.delete_way_pc); -} - -static void -db_insert_way_names_nls_cb(gpointer key, gpointer value, gpointer user_data) -{ -gchar *norm; - -way *w=(way *)user_data; - -sqlite3_bind_int(sql.insert_way_names_nls, 1, w->id); -sqlite3_bind_text(sql.insert_way_names_nls, 2, (gchar *)key, -1, SQLITE_TRANSIENT); -sqlite3_bind_text(sql.insert_way_names_nls, 3, (gchar *)value, -1, SQLITE_TRANSIENT); -#ifdef TRANSLIT_NAMES -norm=g_convert((gchar *value), -1, "ASCII//TRANSLIT//IGNORE", "utf8", NULL, NULL, NULL); -if (norm && strcmp((gchar *)value, norm)!=0) { - sqlite3_bind_text(sql.insert_way_names_nls, 4, norm, -1, SQLITE_TRANSIENT); -} -if (norm) - g_free(norm); -#endif -db_exec(db,sql.insert_way_names_nls); -} - -static void -db_insert_way_names_nls(way *w) -{ -if (!w->data) - return; -if (!w->data->names) - return; - -g_hash_table_foreach(w->data->names, db_insert_way_names_nls_cb, w); -} - -/** - * Insert all data for the given way - * - name - * - ref - * - nodes - * - */ -static gboolean -db_insert_way(way *w) -{ -GSList *iter; -guint ncnt; -node *wmn; - -if (!w) - return FALSE; - -/* Skip things we don't use (yet) */ -if (w->type==WAY_UNWAYED || w->type>WAY_ROAD_END) - return TRUE; - -/* Insert nodes */ -for (iter=w->nodes; iter!=NULL; iter=iter->next) { - if (!iter->next) - break; - db_insert_way_n2n(w, iter->data, iter->next->data); -} - -if (w->id==0) - return FALSE; - -if (w->data) { - w->data->isin_p=osm_find_way_place(w, NODE_PLACE_CITY); - w->data->isin_c=osm_find_way_place(w, NODE_PLACE_COUNTRY); -} - -print_way(w); - -/* Get middle node, use it as way location */ -ncnt=g_slist_length(w->nodes); -if (ncnt>1) - wmn=g_slist_nth_data(w->nodes, ncnt/2); -else - wmn=0; - -sqlite3_bind_int(sql.insert_way_data, 1, w->id); -sqlite3_bind_int(sql.insert_way_data, 2, w->ncnt); -sqlite3_bind_int(sql.insert_way_data, 3, w->type); -sqlite3_bind_int(sql.insert_way_data, 4, w->flags); -if (w->data) { - sqlite3_bind_int(sql.insert_way_data, 5, w->data->speed); - sqlite3_bind_int(sql.insert_way_data, 6, w->data->isin_c); - sqlite3_bind_int(sql.insert_way_data, 7, w->data->isin_p); -} -if (wmn) { - sqlite3_bind_double(sql.insert_way_data, 8, wmn->lat); - sqlite3_bind_double(sql.insert_way_data, 9, wmn->lon); -} else { - g_printerr("Failed to get way location node!\n"); -} - -db_exec(db,sql.insert_way_data); - -db_insert_way_ref(w); -db_insert_way_name(w); -db_insert_way_names_nls(w); -db_insert_way_pc(w); - -osm_free_way_data(w); -return TRUE; -} - -/********************************************************************/ - -static gchar * -get_attr_key_value(const gchar **p, gchar *key) -{ -gchar **d; - -d=p; -while (*d!=NULL) { - if (strncmp(*d, key, strlen(key))==0) { - d++; - return *d; - } - d++; - d++; -} -return NULL; -} - -static tag_state_t -check_tag(const gchar *tag) -{ -if (strcmp(tag,"node")==0) return IN_NODE_TAG; -else if (strcmp(tag,"nd")==0) return IN_WNODE_TAG; -else if (strcmp(tag,"way")==0) return IN_WAY_TAG; -else if (strcmp(tag,"tag")==0) return IN_KEY_TAG; -else if (strcmp(tag,"osm")==0) return IN_OSM_TAG; -else if (strcmp(tag,"bound")==0) return IN_BOUND_TAG; -else if (strcmp(tag,"relation")==0) return IN_RELATION_TAG; -else if (strcmp(tag,"member")==0) return IN_MEMBER_TAG; -else return ERROR; -} - -static void -find_nls_names(gpointer key, gpointer value, gpointer user_data) -{ -gchar *k, *v; -gchar *tmp; -GHashTable *nls; - -k=(gchar *)key; -v=(gchar *)value; -nls=(GHashTable *)user_data; - -/* Check if it is a name key, return if not. */ -if (g_str_has_prefix(k, "name:")==FALSE) - return; - -tmp=g_strrstr(k, ":"); -if (!tmp) - return; -tmp++; /* skip : */ -if (*tmp==0) - return; -g_hash_table_insert(nls, g_strdup(tmp), g_strdup(v)); -#ifdef VERBOSE -g_printf("NLS(%s): [%s]\n", tmp, v); -#endif -} - -/********************************************************************/ - -static void -node_print (node *n) -{ -g_assert(n); -if (n->data) { - g_printf("N: %d [%f:%f][%s](%d)\n", - n->id, n->lat, n->lon, - n->data->name ? n->data->name : "-", - n->type); -} else { - g_printf("N: %d [%f:%f]\n", - n->id, n->lat, n->lon); -} -} - -#ifdef DEBUG -static void -dump_array(const gchar **p) -{ -char **d; - -d=p; -while (*d!=NULL) { - g_printf("[%s]", *d); - d++; -} -g_print("\n"); -} -#endif - -static inline gboolean -osm_node_check_box(gdouble nlat, gdouble nlon) -{ -if (use_bbox==FALSE) - return TRUE; -return (nlat > bbox.lat_min && nlat < bbox.lat_max && nlon > bbox.lon_min && nlon < bbox.lon_max) ? TRUE : FALSE; -} - -static void -osm_new_node_data(node *n) -{ -if (n==NULL) - return; -if (n->data!=NULL) - return; -n->data=g_slice_new(node_data); -n->data->name=NULL; -n->data->url=NULL; -n->data->desc=NULL; -n->data->postal_code=NULL; -n->type=NODE_PLAIN; -noded_cnt++; -} - -static void -osm_free_node_data(node *n) -{ -g_assert(n); -g_assert(n->data); -if (n->data->name) - g_free(n->data->name); -if (n->data->url) - g_free(n->data->url); -if (n->data->desc) - g_free(n->data->desc); -if (n->data->postal_code) - g_free(n->data->postal_code); -g_slice_free(node_data, n->data); -n->data=NULL; -noded_cnt--; -} - -static node * -osm_new_node(gint id, gdouble lat, gdouble lon) -{ -node *n=NULL; - -n=g_slice_new(node); -g_assert(n); -n->id=id; -n->lat=lat; -n->lon=lon; -n->data=(node_data *)NULL; -return n; -} - -static void -osm_free_node(node *n) -{ -g_assert(n); -g_slice_free(node, n); -} - -static node * -osm_find_node(guint32 nid) -{ -node *n; - -g_assert(osm_nodes); -n=g_hash_table_lookup(osm_nodes, GINT_TO_POINTER(nid)); -#if 0 -if (!n) - g_printerr("ERROR: Node %d not found!\n", nid); -#endif -return n; -} - -static void -osm_new_way_data(way *w) -{ -if (w==NULL) - return; -if (w->data!=NULL) - return; - -w->data=g_slice_new(way_data); -w->data->name=NULL; -w->data->names=NULL; -w->data->ref=NULL; -w->data->int_ref=NULL; -w->data->postal_code=NULL; -w->data->layer=0; -w->data->speed=0; -} - -void -osm_free_way_data(way *w) -{ -g_assert(w); -if (!w->data) - return; -if (w->data->name) - g_free(w->data->name); -if (w->data->ref) - g_free(w->data->ref); -if (w->data->int_ref) - g_free(w->data->int_ref); -g_slice_free(way_data, w->data); -w->data=NULL; -} - -static way * -osm_new_way(gint id) -{ -way *w; - -w=g_slice_new(way); -g_assert(w); -w->id=id; -w->nodes=NULL; -w->type=WAY_UNWAYED; -w->data=NULL; -w->ncnt=0; -w->flags=0; - -/* Add to list of ways */ -return w; -} - -static void -osm_way_add_to_list(way *w) -{ -g_assert(w); -osm_ways=g_slist_prepend(osm_ways, w); -} - -static void -osm_way_new_node(way *w, gint nid) -{ -node *n; - -g_assert(w); -n=osm_find_node(nid); -w->nodes=g_slist_prepend(w->nodes, n); -w->ncnt++; -} - -/** - * Search the place hash table for the location of the node. - * - */ -static guint32 -osm_find_node_place(node *n) -{ -node *t; -gchar **isin; -gchar **place; - -if (!n->data) - return 0; - -isin=g_hash_table_lookup(osm_node_isin, GINT_TO_POINTER(n->id)); - -if (!isin) - return 0; - -place=isin; -while (*place!=NULL) { - gchar *ps; - ps=g_strstrip(*place); -#ifdef VERBOSE - g_printf("Checking (%d) [%s] in [%s]\n",n->type, n->data->name, ps); -#endif - switch (n->type) { - case NODE_PLACE_CITY: - case NODE_PLACE_TOWN: - case NODE_PLACE_VILLAGE: - case NODE_PLACE_HAMLET: - t=g_hash_table_lookup(osm_place_region, ps); - if (t) - return t->id; - t=g_hash_table_lookup(osm_place_country, ps); - if (t) - return t->id; - break; - case NODE_PLACE_SUBURB: - case NODE_PLACE_LOCALITY: - t=g_hash_table_lookup(osm_place_city, ps); - if (t) - return t->id; - break; - case NODE_PLACE_ISLAND: - return 0; - break; - default: - t=g_hash_table_lookup(osm_place_city, ps); - if (t) - return t->id; - break; - } - place++; -} - -return 0; -} - -guint32 -osm_find_way_place(way *w, node_type_t nt) -{ -gchar **isin; -gchar **place; - -isin=g_hash_table_lookup(osm_way_isin, GINT_TO_POINTER(w->id)); -if (!isin) - return 0; - -place=isin; -while (*place!=NULL) { - node *t; - gchar *ps; - - ps=g_strstrip(*place); - -#ifdef VERBOSE - g_printf("Checking (%d) in [%s]\n",w->id, ps); -#endif -switch (nt) { - case NODE_PLACE_CITY: - case NODE_PLACE_TOWN: - case NODE_PLACE_VILLAGE: - case NODE_PLACE_HAMLET: - case NODE_PLACE_LOCALITY: - t=g_hash_table_lookup(osm_place_city, ps); - if (t) - return t->id; - break; - case NODE_PLACE_COUNTRY: - t=g_hash_table_lookup(osm_place_country, ps); - if (t) - return t->id; - break; - default: - g_assert_not_reached(); - break; - } - place++; -} - -return 0; -} - -/***********************************************************************/ - -static void -osm_node_save_node(gint key, gpointer value, gpointer user_data) -{ -node *n=(node *)value; - -dbnode_cnt++; -db_insert_node(n); -if (dbnode_cnt % 26214==0) - g_printf("\rNodes: %f%%\n",((float)dbnode_cnt/(float)node_cnt)*100); -} - -/** - * Check node type and insert as POI or Place - * Discard extra data after insert. - */ -static gboolean -osm_node_save_poi(node *n, gpointer user_data) -{ -if (!n) { - g_printerr("ERROR: null poi\n"); - return FALSE; -} - -if (!n->data) { - g_printerr("POI node with no data ?\n"); - return FALSE; -} - -n->data->isin_p=osm_find_node_place(n); -n->data->isin_c=0; - -if (n->type>NODE_POI_START && n->typetype>NODE_PLACE_START && n->type0) { - g_printf("\rWays: %f%%\n",(((float)dbway_cnt/(float)way_cnt)*100)); - print_way(value); -} -} - -static void -osm_planet_clear_ways(void) -{ -g_print("Clearing old data\n"); -sqlite3_step(sql.delete_way); -sqlite3_step(sql.delete_way_name); -sqlite3_step(sql.delete_way_ref); -sqlite3_step(sql.delete_way_n2n); -} - -static gboolean -osm_planet_save_ways(void) -{ -g_print("Inserting new ways\n"); -db_transaction_begin(db); -g_slist_foreach(osm_ways, osm_way_save, NULL); -return db_transaction_commit(db); -} - -/*********************************************************************/ - -static void -osm_planet_save_all_nodes(void) -{ -g_printf("Saving planet nodes to database:\n"); - -osm_planet_poi_clear_nodes(); -osm_planet_poi_save_nodes(); - -if (!is_update) { - osm_planet_clear_nodes(); - osm_planet_clear_ways(); -} -osm_planet_save_nodes(); -} - -static void -osm_planet_save_all_ways(void) -{ -g_printf("Saving planet way to database:\n"); - -osm_planet_save_ways(); -} - -/***********************************************************************/ - -static void -_osm_tag_start(void *userData, const char *name, const char **atts) -{ -tag_state_t t; -gchar *k, *v; -guint32 id, ndref; -gdouble nlat, nlon; - -t=check_tag(name); -switch (t) { - case IN_OSM_TAG: - g_printf("Starting...\n"); - break; - case IN_NODE_TAG: - tag_parent=IS_NODE; - node_cnt++; - - id=atoi(get_attr_key_value(atts, "id")); - nlat=atof(get_attr_key_value(atts, "lat")); - nlon=atof(get_attr_key_value(atts, "lon")); - - cnode=osm_new_node(id, nlat, nlon); - osm_node_tags=g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free); - break; - case IN_WAY_TAG: - tag_parent=IS_WAY; - way_cnt++; - id=atoi(get_attr_key_value(atts, "id")); - cway=osm_new_way(id); - osm_way_tags=g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free); - break; - case IN_WNODE_TAG: - ndref=atoi(get_attr_key_value(atts, "ref")); - if (use_bbox==TRUE) { - if (osm_find_node(ndref)==NULL) { - cway->id=0; - return; - } - } - osm_way_new_node(cway, ndref); - break; - case IN_KEY_TAG: - k=get_attr_key_value(atts, "k"); - if (strcmp(k,"created_by")==0) - return; - if (strcmp(k,"source")==0) - return; - - v=get_attr_key_value(atts, "v"); -#ifdef VERBOSE_KEYS - g_printf("TAG: K=[%s] V=[%s]\n", k, v); -#endif - - switch (tag_parent) { - case IS_NONE: - g_printf("Tag key/value pair but unknown owner\n"); - break; - case IS_NODE: - { - if (!osm_node_tags) - return; - - /* Insert key/value pairs into hash table */ - if (cnode==NULL) { - g_printerr("In node tags but node is NULL!\n"); - return; - } - g_hash_table_insert(osm_node_tags, g_strdup(k), g_strdup(v)); - } - break; - case IS_WAY: - { - if (cway==NULL) { - g_printerr("In way tags but way is NULL!\n"); - return; - } - g_hash_table_insert(osm_way_tags, g_strdup(k), g_strdup(v)); - osm_new_way_data(cway); - } - break; - case IS_RELATION: - - break; - } - break; - case IN_BOUND_TAG: - /* Ignore for now */ - g_printf("Ignoring bound tag\n"); - break; - case IN_RELATION_TAG: - tag_parent=IS_RELATION; - - break; - case IN_MEMBER_TAG: - - break; - default: - tag_parent=IS_NONE; - g_printf("Unknown tag: %s\n", name); - break; -} -} - -static void -_osm_tag_end(void *userData, const char *name) -{ -tag_state_t t; -gchar *v; -guint i; -t=check_tag(name); -switch (t) { - case IN_NODE_TAG: - - if (node_cnt % 262140==0) { - g_printf("Nodes: %d of %d, POIs: %d, Outside box: %d\n", node_cnt-node_skip_cnt, node_cnt, noded_cnt, node_skip_cnt); - } - - if (!osm_node_tags) - return; - - osm_new_node_data(cnode); - - for (i=0; nodeinfo[i].k; i++) { - v=g_hash_table_lookup(osm_node_tags, nodeinfo[i].k); - if (!v) - continue; - if (strcasecmp (v, nodeinfo[i].v)==0) { - cnode->type=nodeinfo[i].type; - break; - } - } - - /* Check if node is inside bounding box, if not skip it. - * But keep it if it's something we might need for other nodes: - * - Places (for is_in) - * - ... - */ - if ((osm_node_check_box(cnode->lat, cnode->lon)==FALSE) && - (cnode->typeid), cnode); - - if (cnode->type!=NODE_PLAIN) { - cnode->data->name=NULL; - v=g_hash_table_lookup(osm_node_tags, "name"); - if (v) - cnode->data->name=g_strstrip(g_utf8_normalize(v, -1, G_NORMALIZE_ALL_COMPOSE)); - v=g_hash_table_lookup(osm_node_tags, "note"); - if (v) - cnode->data->desc=g_strstrip(g_strdup(v)); - v=g_hash_table_lookup(osm_node_tags, "postal_code"); - if (v) - cnode->data->postal_code=g_strstrip(g_strdup(v)); - - /* Links */ - v=g_hash_table_lookup(osm_node_tags, "url"); - if (v) { - cnode->data->url=g_strstrip(g_strdup(v)); - } else { - v=g_hash_table_lookup(osm_node_tags, "wikipedia"); - if (v && strncmp(v,"http:", 5)==0) - cnode->data->url=g_strstrip(g_strdup(v)); - } - } - - cnode->data->isin_c=0; - cnode->data->isin_p=0; - v=g_hash_table_lookup(osm_node_tags, "is_in"); - if (v) { - gchar **isin; - isin=g_strsplit(v, ",", 10); - g_hash_table_insert(osm_node_isin, GINT_TO_POINTER(cnode->id), isin); - } - - if (cnode->type==NODE_PLAIN) { - osm_free_node_data(cnode); - } else { - osm_poi=g_slist_prepend(osm_poi, cnode); - if (cnode->data->name) { - switch (cnode->type) { - case NODE_PLACE_COUNTRY: - g_hash_table_insert(osm_place_country, cnode->data->name, cnode); - break; - case NODE_PLACE_CITY: - case NODE_PLACE_TOWN: - g_hash_table_insert(osm_place_city, cnode->data->name, cnode); - break; - case NODE_PLACE_SUBURB: - g_hash_table_insert(osm_place_suburb, cnode->data->name, cnode); - break; - case NODE_PLACE_VILLAGE: - case NODE_PLACE_HAMLET: - case NODE_PLACE_LOCALITY: - g_hash_table_insert(osm_place_village, cnode->data->name, cnode); - break; - case NODE_PLACE_ISLAND: - /* Ignore for now */ - break; - default:; - } - } - } - g_hash_table_destroy(osm_node_tags); - cnode=NULL; - break; - case IN_WAY_TAG: - if (way_cnt % 1024==0) { - g_printf("\rWays: %d\n", way_cnt); - } - - cway->nodes=g_slist_reverse(cway->nodes); - - for (i=0; wayinfo[i].k; i++) { - v=g_hash_table_lookup(osm_way_tags, wayinfo[i].k); - if (!v) - continue; - if (strcasecmp (v, wayinfo[i].v)==0) { - if (wayinfo[i].link==TRUE) - cway->flags|=W_LINK; - if (wayinfo[i].area==TRUE) - cway->flags|=W_AREA; - if (wayinfo[i].oneway==TRUE) - cway->flags|=W_ONEWAY; - cway->type=wayinfo[i].type; - if (cway->data->speed==0) - cway->data->speed=wayinfo[i].defspeed; - break; - } - } - - v=g_hash_table_lookup(osm_way_tags, "name"); - if (v) { - cway->data->name=g_utf8_normalize(v, -1, G_NORMALIZE_ALL_COMPOSE); - /* Try to find other language names */ - cway->data->names=g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free); - g_hash_table_foreach(osm_way_tags, find_nls_names, cway->data->names); - if (g_hash_table_size(cway->data->names)==0) { - g_hash_table_destroy(cway->data->names); - cway->data->names=NULL; - } - } - - v=g_hash_table_lookup(osm_way_tags, "ref"); - if (v) - cway->data->ref=g_strdup(v); - v=g_hash_table_lookup(osm_way_tags, "int_ref"); - if (v) - cway->data->int_ref=g_strdup(v); - - v=g_hash_table_lookup(osm_way_tags, "postal_code"); - if (v) - cway->data->postal_code=g_strdup(v); - - /* XXX: somehow handle the silly -1 'reversed' oneway */ - v=g_hash_table_lookup(osm_way_tags, "oneway"); - if (v) - cway->flags|=W_ONEWAY; - - v=g_hash_table_lookup(osm_way_tags, "noexit"); - if (v) - cway->flags|=W_NOEXIT; - - v=g_hash_table_lookup(osm_way_tags, "speedlimit"); - if (v) - cway->data->speed=atoi(v); - v=g_hash_table_lookup(osm_way_tags, "maxspeed"); - if (v) - cway->data->speed=atoi(v); - -#if 0 - v=g_hash_table_lookup(osm_way_tags, "layer"); - if (v) - cway->data->layer=atoi(v); -#endif - - v=g_hash_table_lookup(osm_way_tags, "junction"); - if (v && strcasecmp(v,"roundabout")==0) { - cway->flags|=W_ROUNDABOUT; - cway->flags|=W_ONEWAY; - } else if (v && strcasecmp(v,"mini_roundabout")==0) { - cway->flags|=W_ROUNDABOUT; - cway->flags|=W_ONEWAY; - } - - /* XXX: Should check keys */ - v=g_hash_table_lookup(osm_way_tags, "access"); - if (v && (strcasecmp(v, "private")==0)) { - cway->flags|=W_NOACCESS; - } - - print_way(cway); - - v=g_hash_table_lookup(osm_way_tags, "is_in"); - if (v) { - gchar **isin; - isin=g_strsplit(v, ",", 10); - g_hash_table_insert(osm_way_isin, GINT_TO_POINTER(cway->id), isin); - } - - if (cway->data && cway->data->name==NULL && cway->data->ref==NULL && - cway->data->int_ref==NULL && cway->data->layer==0 && cway->data->speed==0) - osm_free_way_data(cway); - - if (cway->id!=0) - osm_way_add_to_list(cway); - - cway=NULL; - g_hash_table_destroy(osm_way_tags); - break; - case IN_BOUND_TAG: - /* */ - break; - case IN_OSM_TAG: - g_printf("\nPlanet loaded.\n"); - break; - default:; -} -} - -/************************************************************************/ - -static void -storage_init(void) -{ -osm_nodes=g_hash_table_new(g_direct_hash, g_direct_equal); - -osm_place_country=g_hash_table_new(g_str_hash, g_str_equal); -osm_place_city=g_hash_table_new(g_str_hash, g_str_equal); -osm_place_suburb=g_hash_table_new(g_str_hash, g_str_equal); -osm_place_village=g_hash_table_new(g_str_hash, g_str_equal); -osm_place_region=g_hash_table_new(g_str_hash, g_str_equal); -osm_node_isin=g_hash_table_new(g_direct_hash, g_direct_equal); -osm_way_isin=g_hash_table_new(g_direct_hash, g_direct_equal); -} - -static void -storage_free(void) -{ -g_hash_table_destroy(osm_nodes); - -g_hash_table_destroy(osm_place_country); -g_hash_table_destroy(osm_place_city); -g_hash_table_destroy(osm_place_suburb); -g_hash_table_destroy(osm_place_village); -g_hash_table_destroy(osm_place_region); -g_hash_table_destroy(osm_node_isin); -} - -void -osm_planet_parser_init(void) -{ -xp=XML_ParserCreate(NULL); -XML_SetElementHandler(xp, _osm_tag_start, _osm_tag_end); -storage_init(); -} - -void -osm_planet_parser_deinit(void) -{ -XML_ParserFree(xp); -storage_free(); -} - -gboolean -osm_planet_parse_buffer(gchar *buffer, size_t r) -{ -if (XML_Parse(xp, buffer, r, r>0 ? 0:1) == XML_STATUS_ERROR) { - g_printerr("Parse error at line %d:\n%s\n", - XML_GetCurrentLineNumber(xp), - XML_ErrorString(XML_GetErrorCode(xp))); - return FALSE; -} -return TRUE; -} - -gboolean -osm_planet_parse_file(gchar *pfile) -{ -FILE *f; -BZFILE *b; -int bzerror; -int r; -gchar buffer[FILE_BUFFER]; -gboolean res=TRUE; - -f=fopen(pfile, "r"); -if (!f) { - perror("fopen failed\n"); - return FALSE; -} - -b=BZ2_bzReadOpen(&bzerror, f, 0, 0, NULL, 0); -if (bzerror != BZ_OK) { - g_printf("BZ2_bzReadOpen failed\n"); - BZ2_bzReadClose(&bzerror, b); - return FALSE; -} - -do { - r=BZ2_bzRead(&bzerror, b, buffer, FILE_BUFFER); - if ((bzerror!=BZ_STREAM_END) && (bzerror!=BZ_OK)) { - res=FALSE; - break; - } - if (!osm_planet_parse_buffer(buffer, r)) { - res=FALSE; - break; - } -} while (bzerror==BZ_OK); - -BZ2_bzReadClose(&bzerror, b); -fclose(f); -return res; -} - -/************************************************************************/ - static gint print_fail(const gchar *msg, gint ret) { @@ -1739,72 +43,26 @@ g_printerr("ERROR: %s\n", msg); return ret; } -/************************************************************************/ - -static void -print_memory_usage(void) -{ -g_print("Memory usage per item:\n"); -g_printf("Node size: %d\n", (gint)sizeof(node)); -g_printf("NodeD size: %d\n", (gint)sizeof(node_data)); -g_printf("Way size: %d\n", (gint)sizeof(way)); -g_printf("WayD size: %d\n", (gint)sizeof(way_data)); -} - -/************************************************************************/ - int main (int argc, char **argv) { gchar *in_file; gchar *out_file; -if (argc<2) { +if (argc<2) return print_fail("Give bzip2 compressed planet XML file as argument", 1); -} out_file=OSM_DB_FILE; in_file=argv[1]; g_printf("Using file: %s\n", in_file); if (argc==6) { - use_bbox=TRUE; - bbox.lat_min=atof(argv[2]); - bbox.lon_min=atof(argv[3]); - bbox.lat_max=atof(argv[4]); - bbox.lon_max=atof(argv[5]); - g_printf("Skipping data outside of box: %f,%f - %f,%f\n", - bbox.lat_min, bbox.lon_min, - bbox.lat_max, bbox.lon_max); - g_print("Note: missing nodes can't be reported.\n"); -} else use_bbox=FALSE; - -if (db_connect(&db, out_file)!=TRUE) - return print_fail("Database open failed", 2); - -db_create_tables(db); -db_create_indexes(db); -db_prepare(db); - -print_memory_usage(); - -osm_planet_parser_init(); -if (osm_planet_parse_file(argv[1])==FALSE) { - g_printf("Failed to parse file: %s\n", in_file); - return 1; + osm_import_set_bbox(TRUE, atof(argv[2]), atof(argv[3]), atof(argv[4]), atof(argv[5])); +} else { + osm_import_set_bbox(FALSE, 0, 0, 0, 0); } -g_printf("Total nodes %d, POIs: %d and Ways %d.\n", node_cnt, noded_cnt, way_cnt); -g_printf("Cities/Towns: %d\n", g_hash_table_size(osm_place_city)); -g_printf("Villages/Hamlets: %d\n", g_hash_table_size(osm_place_village)); -g_printf("Suburbs: %d\n", g_hash_table_size(osm_place_suburb)); -g_printf("Nodes: %d\n", g_hash_table_size(osm_nodes)); - -osm_planet_save_all_nodes(); -osm_planet_save_all_ways(); -osm_planet_parser_deinit(); -db_close(&db); - -g_printf("Named ways: %d, Numbered ways: %d\n", way_names, way_refs); - +if (osm_import(in_file, out_file)==FALSE) { + return print_fail("Import failed", 4); +} return 0; } -- 2.39.5