From: Kaj-Michael Lang Date: Sat, 13 Oct 2007 09:21:27 +0000 (+0300) Subject: Port to 0.5 API. Plus some bug fixes: X-Git-Url: https://err.no/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7d3a36224012fd9b0b233b423a25e2a85fcfaaa4;p=mapper Port to 0.5 API. Plus some bug fixes: - Don't skip POIs with no name --- diff --git a/src/osm.c b/src/osm.c index ff404da..206b1c8 100644 --- a/src/osm.c +++ b/src/osm.c @@ -18,18 +18,20 @@ #include "latlon.h" /* #define VERBOSE */ +/* #define VERBOSE_KEYS */ #define FILE_BUFFER 65535 #define OSM_DB_FILE "osm-planet.db" -static gint node_cnt=0; -static gint node_skip_cnt=0; -static gint noded_cnt=0; -static gint way_cnt=0; -static gint seg_cnt=0; +static guint node_cnt=0; +static guint node_skip_cnt=0; +static guint noded_cnt=0; +static guint way_cnt=0; +static guint way_names=0; +static guint way_refs=0; -static gint dbnode_cnt=0; -static gint dbnoded_cnt=0; -static gint dbway_cnt=0; +static guint dbnode_cnt=0; +static guint dbnoded_cnt=0; +static guint dbway_cnt=0; static gboolean is_update=FALSE; static XML_Parser xp; @@ -50,25 +52,17 @@ struct _node { node_data *data; }; -/* Segment */ -typedef struct _segment segment; -struct _segment { - guint32 id; - guint32 from; - guint32 to; -}; - typedef struct _way_data way_data; struct _way_data { gchar *name; gchar *ref; gchar *int_ref; guint32 isin; - gint speed; + guint speed; gint8 layer; }; -/* Segment/Way structure */ +/* Way structure */ typedef struct _way way; struct _way { guint32 id; @@ -83,10 +77,10 @@ typedef enum { START, IN_OSM_TAG, IN_NODE_TAG, - IN_SEGMENT_TAG, - IN_SEG_TAG, + IN_WNODE_TAG, IN_WAY_TAG, IN_KEY_TAG, + IN_BOUND_TAG, END, ERROR } tag_state_t; @@ -94,8 +88,7 @@ typedef enum { typedef enum { IS_NONE, IS_NODE, - IS_WAY, - IS_SEGMENT + IS_WAY } tag_parent_t; struct _nodeinfo { @@ -103,50 +96,49 @@ struct _nodeinfo { node_type_t type; } nodeinfo[] = { { "amenity", "fuel", NODE_AMENITY_FUEL }, - { "amenity", "pub", NODE_AMENITY_PUB }, - { "amenity", "biergarten", NODE_AMENITY_PUB }, + { "amenity", "pub", NODE_AMENITY_PUB }, + { "amenity", "biergarten", NODE_AMENITY_PUB }, { "amenity", "cafe", NODE_AMENITY_CAFE }, - { "amenity", "telephone", NODE_AMENITY_TELEPHONE }, + { "amenity", "telephone", NODE_AMENITY_TELEPHONE }, { "amenity", "toilets", NODE_AMENITY_WC }, { "amenity", "fast_food", NODE_AMENITY_FOOD }, { "amenity", "restaurant", NODE_AMENITY_FOOD }, { "amenity", "parking", NODE_AMENITY_PARKING }, { "amenity", "hospital", NODE_AMENITY_HOSPITAL }, - { "amenity", "doctors", NODE_AMENITY_HOSPITAL }, + { "amenity", "doctors", NODE_AMENITY_HOSPITAL }, { "amenity", "pharmacy", NODE_AMENITY_PHARMACY }, { "amenity", "post_office", NODE_AMENITY_POST }, { "amenity", "post_box", NODE_AMENITY_POST_BOX }, { "amenity", "cinema", NODE_AMENITY_CINEMA }, { "amenity", "theatre", NODE_AMENITY_THEATRE }, - { "amenity", "atm", NODE_AMENITY_ATM }, + { "amenity", "atm", NODE_AMENITY_ATM }, { "amenity", "bank", NODE_AMENITY_BANK }, { "amenity", "police", NODE_AMENITY_POLICE }, - { "amenity", "speed_trap", NODE_AMENITY_SPEEDCAM }, + { "amenity", "speed_trap", NODE_AMENITY_SPEEDCAM }, { "amenity", "supermarket", NODE_AMENITY_SHOP }, { "amenity", "shop", NODE_AMENITY_SHOP }, { "amenity", "place_of_worship",NODE_AMENITY_POW }, - { "amenity", "school",NODE_AMENITY_SCHOOL }, - { "amenity", "college",NODE_AMENITY_COLLEGE }, - { "amenity", "university",NODE_AMENITY_COLLEGE }, + { "amenity", "school", NODE_AMENITY_SCHOOL }, + { "amenity", "college", NODE_AMENITY_COLLEGE }, + { "amenity", "university", NODE_AMENITY_COLLEGE }, - { "tourism", "information",NODE_AMENITY_TOURISM_INFO }, - { "tourism", "camp_site",NODE_AMENITY_CAMP_SITE }, + { "tourism", "information", NODE_AMENITY_TOURISM_INFO }, + { "tourism", "camp_site", NODE_AMENITY_CAMP_SITE }, { "tourism", "caravan_site",NODE_AMENITY_CARAVAN_SITE }, - { "tourism", "picnic_site",NODE_AMENITY_PICNIC_SITE }, - { "tourism", "theme_park",NODE_AMENITY_THEME_PARK }, - { "tourism", "hotel",NODE_AMENITY_HOTEL }, - { "tourism", "motel",NODE_AMENITY_MOTEL }, - { "tourism", "hostel",NODE_AMENITY_HOSTEL }, - { "tourism", "attraction",NODE_AMENITY_ATTRACTION }, + { "tourism", "picnic_site", NODE_AMENITY_PICNIC_SITE }, + { "tourism", "theme_park", NODE_AMENITY_THEME_PARK }, + { "tourism", "hotel", NODE_AMENITY_HOTEL }, + { "tourism", "motel", NODE_AMENITY_MOTEL }, + { "tourism", "hostel", NODE_AMENITY_HOSTEL }, + { "tourism", "attraction", NODE_AMENITY_ATTRACTION }, { "railway", "station", NODE_AMENITY_RAILWAY_STATION }, { "railway", "halt", NODE_AMENITY_RAILWAY_HALT }, - { "aeroway", "terminal", NODE_AMENITY_AIRPORT }, - + { "aeroway", "terminal", NODE_AMENITY_AIRPORT }, - { "place", "city", NODE_PLACE_CITY }, - { "place", "town", NODE_PLACE_TOWN }, + { "place", "city", NODE_PLACE_CITY }, + { "place", "town", NODE_PLACE_TOWN }, { "place", "village", NODE_PLACE_VILLAGE }, { "place", "hamlet", NODE_PLACE_HAMLET }, { "place", "suburb", NODE_PLACE_SUBURB }, @@ -155,7 +147,7 @@ struct _nodeinfo { { "highway", "traffic_signals", NODE_TRAFFIC_SIGNALS }, { "highway", "services", NODE_AMENITY_PARKING }, { "highway", "toll_booth", NODE_TOLLBOOTH }, - { "highway", "gate", NODE_GATE }, + { "highway", "gate", NODE_GATE }, { NULL, NULL, NODE_PLAIN } }; @@ -163,36 +155,38 @@ struct _nodeinfo { /* Array to get id number and defaults for ways of different types */ struct _wayinfo { gchar *k, *v; + guint defspeed; way_type_t type; gboolean oneway, link, area, car, foot; } wayinfo[] = { - { "highway", "motorway",WAY_MOTORWAY, TRUE, FALSE, FALSE, TRUE, FALSE }, - { "highway", "motorway_link",WAY_MOTORWAY, TRUE, TRUE, FALSE, TRUE, FALSE }, - { "highway", "trunk",WAY_TRUNK, FALSE, FALSE, FALSE, TRUE, FALSE }, - { "highway", "trunk_link",WAY_TRUNK, FALSE, TRUE, FALSE, TRUE, FALSE }, - { "highway", "primary",WAY_PRIMARY, FALSE, FALSE, FALSE, TRUE, TRUE }, - { "highway", "primary_link",WAY_PRIMARY, FALSE, TRUE, FALSE, TRUE, TRUE }, - { "highway", "secondary",WAY_SECONDARY, FALSE, FALSE, FALSE, TRUE, TRUE }, - { "highway", "secondary_link",WAY_SECONDARY, FALSE, TRUE, FALSE, TRUE, TRUE }, - { "highway", "tertiary",WAY_TERTIARY, FALSE, FALSE, FALSE, TRUE, TRUE }, - { "highway", "unclasified",WAY_UNCLASSIFIED, FALSE, FALSE, FALSE, TRUE, TRUE }, - { "highway", "residential",WAY_RESIDENTIAL, FALSE, FALSE, FALSE, TRUE, TRUE }, - { "highway", "service",WAY_SERVICE, FALSE, FALSE, FALSE, TRUE, TRUE }, - { "highway", "track",WAY_TRACK, FALSE, FALSE, FALSE, TRUE, TRUE }, - { "highway", "unsurfaced",WAY_TRACK, FALSE, FALSE, FALSE, TRUE, TRUE }, - { "highway", "pedestrian",WAY_FOOTWAY, FALSE, FALSE, FALSE, FALSE, TRUE }, - { "highway", "footway",WAY_FOOTWAY, FALSE, FALSE, FALSE, FALSE, TRUE }, - { "highway", "steps",WAY_FOOTWAY, FALSE, FALSE, FALSE, FALSE, TRUE}, - { "highway", "bridleway",WAY_FOOTWAY, FALSE, FALSE, FALSE, FALSE, TRUE }, - { "highway", "cycleway",WAY_CYCLEWAY, FALSE, FALSE, FALSE, FALSE, TRUE }, - { "railway", "rail",WAY_RAIL, FALSE, FALSE, FALSE, FALSE, FALSE }, - { "aeroway", "runway",WAY_RUNWAY, FALSE, FALSE, FALSE, FALSE, FALSE }, - { "aeroway", "taxiway",WAY_TAXIWAY, FALSE, FALSE, FALSE, FALSE, FALSE }, - { "natural", "water",WAY_WATER, FALSE, FALSE, TRUE, FALSE, FALSE }, - { "waterway", "river",WAY_WATER, FALSE, FALSE, FALSE, FALSE, FALSE }, - { "waterway", "canal",WAY_WATER, FALSE, FALSE, FALSE, FALSE, FALSE }, - { "waterway", "stream",WAY_WATER, FALSE, FALSE, FALSE, FALSE, FALSE }, - { NULL, NULL, WAY_UNWAYED, FALSE, FALSE, FALSE, FALSE, FALSE } + { "highway", "motorway",120,WAY_MOTORWAY, TRUE, FALSE, FALSE, TRUE, FALSE }, + { "highway", "motorway_link",120,WAY_MOTORWAY, TRUE, TRUE, FALSE, TRUE, FALSE }, + { "highway", "trunk",100,WAY_TRUNK, FALSE, FALSE, FALSE, TRUE, FALSE }, + { "highway", "trunk_link",100,WAY_TRUNK, FALSE, TRUE, FALSE, TRUE, FALSE }, + { "highway", "primary",80,WAY_PRIMARY, FALSE, FALSE, FALSE, TRUE, TRUE }, + { "highway", "primary_link",60,WAY_PRIMARY, FALSE, TRUE, FALSE, TRUE, TRUE }, + { "highway", "secondary",80,WAY_SECONDARY, FALSE, FALSE, FALSE, TRUE, TRUE }, + { "highway", "secondary_link",60,WAY_SECONDARY, FALSE, TRUE, FALSE, TRUE, TRUE }, + { "highway", "tertiary",60,WAY_TERTIARY, FALSE, FALSE, FALSE, TRUE, TRUE }, + { "highway", "unclasified",50,WAY_UNCLASSIFIED, FALSE, FALSE, FALSE, TRUE, TRUE }, + { "highway", "residential",40,WAY_RESIDENTIAL, FALSE, FALSE, FALSE, TRUE, TRUE }, + { "highway", "service",20,WAY_SERVICE, FALSE, FALSE, FALSE, TRUE, TRUE }, + { "highway", "track",20,WAY_TRACK, FALSE, FALSE, FALSE, TRUE, TRUE }, + { "highway", "unsurfaced",60,WAY_TRACK, FALSE, FALSE, FALSE, TRUE, TRUE }, + { "highway", "pedestrian",20,WAY_FOOTWAY, FALSE, FALSE, FALSE, FALSE, TRUE }, + { "highway", "footway",1,WAY_FOOTWAY, FALSE, FALSE, FALSE, FALSE, TRUE }, + { "highway", "steps",0,WAY_FOOTWAY, FALSE, FALSE, FALSE, FALSE, TRUE}, + { "highway", "bridleway",10,WAY_FOOTWAY, FALSE, FALSE, FALSE, FALSE, TRUE }, + { "highway", "cycleway",10,WAY_CYCLEWAY, FALSE, FALSE, FALSE, FALSE, TRUE }, + { "railway", "rail",0,WAY_RAIL, FALSE, FALSE, FALSE, FALSE, FALSE }, + { "aeroway", "runway",0,WAY_RUNWAY, FALSE, FALSE, FALSE, FALSE, FALSE }, + { "aeroway", "taxiway",0,WAY_TAXIWAY, FALSE, FALSE, FALSE, FALSE, FALSE }, + { "natural", "water",0,WAY_WATER, FALSE, FALSE, TRUE, FALSE, FALSE }, + { "waterway", "river",0,WAY_WATER, FALSE, FALSE, FALSE, FALSE, FALSE }, + { "waterway", "canal",0,WAY_WATER, FALSE, FALSE, FALSE, FALSE, FALSE }, + { "waterway", "stream",0,WAY_WATER, FALSE, FALSE, FALSE, FALSE, FALSE }, + { "building", "",0,WAY_UNWAYED, FALSE, FALSE, TRUE, FALSE, FALSE }, + { NULL, NULL, 0, WAY_UNWAYED, FALSE, FALSE, FALSE, FALSE, FALSE } }; static sqlite3 *db; @@ -200,7 +194,6 @@ tag_parent_t tag_parent=IS_NONE; static GHashTable *osm_nodes; static GHashTable *osm_node_tags; -static GHashTable *osm_segments; static GSList *osm_ways; static GSList *osm_poi; @@ -227,11 +220,9 @@ struct sql_stmt { sqlite3_stmt *insert_way_data; sqlite3_stmt *insert_way_ref; sqlite3_stmt *insert_way_name; - sqlite3_stmt *insert_way_seg; - sqlite3_stmt *insert_way_seg2seg; + sqlite3_stmt *insert_way_n2n; sqlite3_stmt *delete_way; - sqlite3_stmt *delete_way_seg; - sqlite3_stmt *delete_way_seg2seg; + sqlite3_stmt *delete_way_n2n; sqlite3_stmt *delete_way_seg_wsid; sqlite3_stmt *delete_way_name; sqlite3_stmt *delete_way_ref; @@ -250,16 +241,13 @@ struct map_bbox { struct map_bbox bbox; gboolean use_bbox; -gint wsegcnt; - -inline node *osm_find_node(guint32 nid); void osm_free_way_data(way *w); void print_way(way *w); gboolean db_open(void); +gboolean db_close(void); + void db_prepare(void); gboolean db_insert_node(node *n); -void db_close(void); -void db_prepare(void); /****************************************************/ /* Functions */ @@ -272,10 +260,16 @@ if (SQLITE_OK != (sqlite3_open(OSM_DB_FILE, &db))) { sqlite3_close(db); return FALSE; } +/* Turn of syncing to speed up data import */ +sqlite3_exec(db, "PRAGMA synchronous = OFF;", NULL, NULL, NULL); + +/* Use more cache memory */ +sqlite3_exec(db, "PRAGMA cache_size = 8000;", NULL, NULL, NULL); + return TRUE; } -void +gboolean db_close(void) { sqlite3_finalize(sql.insert_poi); @@ -289,67 +283,73 @@ sqlite3_finalize(sql.update_node); sqlite3_finalize(sql.insert_place); sqlite3_finalize(sql.delete_place); -sqlite3_close(db); +return sqlite3_close(db)==SQLITE_OK ? TRUE : FALSE; } void db_prepare(void) { /* Way nodes */ -sqlite3_prepare_v2(db, "insert or replace into nodes (nid,lat,lon,l) values (?, ?, ?, 0)", - -1, &sql.insert_node, NULL); -sqlite3_prepare_v2(db, "select lat,lon,l from nodes where nid=?", - -1, &sql.select_node, NULL); +sqlite3_prepare_v2(db, "insert or replace into nodes (nid,lat,lon,l) values (?, ?, ?, 0)", -1, &sql.insert_node, NULL); +sqlite3_prepare_v2(db, "select lat,lon,l from nodes where nid=?", -1, &sql.select_node, NULL); sqlite3_prepare_v2(db, "delete from nodes", -1, &sql.delete_nodes, NULL); sqlite3_prepare_v2(db, "update nodes set l=l+1 where nid=?", -1, &sql.update_node, NULL); /* Places */ -sqlite3_prepare_v2(db, "insert or replace into places (nid,type,name,isin) values (?, ?, ?, ?)", - -1, &sql.insert_place, NULL); +sqlite3_prepare_v2(db, "insert or replace into places (nid,type,name,isin) values (?, ?, ?, ?)", -1, &sql.insert_place, NULL); sqlite3_prepare_v2(db, "delete from places", -1, &sql.delete_place, NULL); /* POI nodes */ -sqlite3_prepare_v2(db, "insert or replace into poi (osm_id, lat, lon, label, desc, cat_id, public, source, priority, isin)" - " values (?, ?, ?, ?, ?, ?, 1, 1, ?, ?)", -1, &sql.insert_poi, NULL); +sqlite3_prepare_v2(db, "insert or replace into poi (osm_id, lat, lon, label, cat_id, public, source, priority, isin, desc) values (?, ?, ?, ?, ?, 1, 1, ?, ?, ?)", + -1, &sql.insert_poi, NULL); sqlite3_prepare_v2(db, "delete from poi where osm_id>0", -1, &sql.delete_osm_poi, NULL); /* Ways */ -sqlite3_prepare_v2(db, "insert or replace into way (wid,nodes,type,flags,speed,isin) values (?, ?, ?, ?, ?, ?)", - -1, &sql.insert_way_data, NULL); -sqlite3_prepare_v2(db, "delete from way", -1, - &sql.delete_way, NULL); - -/* Way segments, try 1*/ -sqlite3_prepare_v2(db, "insert into way_seg (wsid,num,node) values (?, ?, ?)", - -1, &sql.insert_way_seg, NULL); -sqlite3_prepare_v2(db, "delete from way_seg", -1, - &sql.delete_way_seg, NULL); -sqlite3_prepare_v2(db, "delete from way_seg where wsid=?", -1, - &sql.delete_way_seg_wsid, NULL); - -/* Way segments, try 2*/ -sqlite3_prepare_v2(db, "insert into way_s2s (wsid,f,t) values (?, ?, ?)", - -1, &sql.insert_way_seg2seg, NULL); -sqlite3_prepare_v2(db, "delete from way_s2s where wsid=?", -1, - &sql.delete_way_seg2seg, NULL); +sqlite3_prepare_v2(db, "insert or replace into way (wid,nodes,type,flags,speed,isin) values (?, ?, ?, ?, ?, ?)", -1, &sql.insert_way_data, NULL); +sqlite3_prepare_v2(db, "delete from way", -1, &sql.delete_way, NULL); + +/* Way nodes */ +sqlite3_prepare_v2(db, "insert into way_s2s (wsid,f,t) values (?, ?, ?)", -1, &sql.insert_way_n2n, NULL); +sqlite3_prepare_v2(db, "delete from way_s2s where wsid=?", -1, &sql.delete_way_n2n, NULL); /* Way names */ -sqlite3_prepare_v2(db, "insert or replace into way_names (wid,name) values (?, ?)", - -1, &sql.insert_way_name, NULL); -sqlite3_prepare_v2(db, "delete from way_names", -1, - &sql.delete_way_name, NULL); +sqlite3_prepare_v2(db, "insert or replace into way_names (wid,name) values (?, ?)", -1, &sql.insert_way_name, NULL); +sqlite3_prepare_v2(db, "delete from way_names", -1, &sql.delete_way_name, NULL); /* Way ref and int_ref */ -sqlite3_prepare_v2(db, "insert or replace into way_ref (rid,ref,int_ref) values (?, ?, ?)", - -1, &sql.insert_way_ref, NULL); -sqlite3_prepare_v2(db, "delete from way_ref", -1, - &sql.delete_way_ref, NULL); +sqlite3_prepare_v2(db, "insert or replace into way_ref (rid,ref,int_ref) values (?, ?, ?)", -1, &sql.insert_way_ref, NULL); +sqlite3_prepare_v2(db, "delete from way_ref", -1, &sql.delete_way_ref, NULL); +} -/* Turn of syncing to speed up data import */ -sqlite3_exec(db, "PRAGMA synchronous = OFF;", NULL, NULL, NULL); +/********************************************************************/ -/* Use more cache memory */ -sqlite3_exec(db, "PRAGMA cache_size = 8000;", NULL, NULL, NULL); +void +print_way(way *w) +{ +#ifdef VERBOSE +g_assert(w); +g_printf("Way #%d(N:%d T:%d S:%d): %s [%s:%s:%s]\n", + w->id, + g_slist_length(w->nodes), + w->type, + w->data ? w->data->speed : 0, + w->data ? w->data->name ? w->data->name : "" : "", + w->flags & W_ONEWAY ? "-" : "=", + w->flags & W_ROUNDABOUT ? "O" : "-", + w->flags & W_LINK ? "|" : " "); +#endif +} + +void +print_node(node *n) +{ +#ifdef VERBOSE +g_assert(n); +g_printf("Node #%d: T:%d [%s]\n", + n->id, + n->data ? n->data->type : 0, + n->data ? n->data->name : ""); +#endif } /********************************************************************/ @@ -375,6 +375,8 @@ return TRUE; static gboolean db_insert_place(node *n) { +gint r; + if (!n->data) return FALSE; if (!n->data->name) @@ -383,38 +385,47 @@ sqlite3_bind_int(sql.insert_place, 1, n->id); sqlite3_bind_int(sql.insert_place, 2, n->data->type); sqlite3_bind_text(sql.insert_place, 3, n->data->name, -1, SQLITE_TRANSIENT); sqlite3_bind_int(sql.insert_place, 4, n->data->isin); -sqlite3_step(sql.insert_place); +r=sqlite3_step(sql.insert_place); sqlite3_reset(sql.insert_place); sqlite3_clear_bindings(sql.insert_place); -return TRUE; +return (r==SQLITE_OK) ? TRUE : FALSE; } static gboolean db_insert_poi(node *n) { +gint r; + sqlite3_bind_int(sql.insert_poi, 1, n->id); sqlite3_bind_double(sql.insert_poi, 2, n->lat); sqlite3_bind_double(sql.insert_poi, 3, n->lon); if (n->data->name) sqlite3_bind_text(sql.insert_poi, 4, n->data->name, -1, SQLITE_TRANSIENT); +else + sqlite3_bind_text(sql.insert_poi, 4, "", -1, SQLITE_TRANSIENT); +sqlite3_bind_int(sql.insert_poi, 5, n->data->type); +sqlite3_bind_int(sql.insert_poi, 6, n->data->type/100); +sqlite3_bind_int(sql.insert_poi, 7, n->data->isin); + #if 0 if (n->data->desc) - sqlite3_bind_text(sql.insert_poi, 5, n->data->desc, -1, SQLITE_TRANSIENT); + sqlite3_bind_text(sql.insert_poi, 8, n->data->desc, -1, SQLITE_TRANSIENT); +if (n->data->url) + sqlite3_bind_text(sql.insert_poi, 9, n->data->url, -1, SQLITE_TRANSIENT); #endif -sqlite3_bind_int(sql.insert_poi, 6, n->data->type); -sqlite3_bind_int(sql.insert_poi, 7, n->data->type/100); -sqlite3_bind_int(sql.insert_poi, 8, n->data->isin); - -sqlite3_step(sql.insert_poi); +r=sqlite3_step(sql.insert_poi); sqlite3_reset(sql.insert_poi); sqlite3_clear_bindings(sql.insert_poi); -return TRUE; +return (r==SQLITE_OK) ? TRUE : FALSE; } -static void +/** + * Update node usage count + */ +static gboolean db_update_node_links(node *n) { sqlite3_bind_int(sql.update_node, 1, n->id); @@ -422,40 +433,49 @@ sqlite3_bind_int(sql.update_node, 1, n->id); sqlite3_step(sql.update_node); sqlite3_reset(sql.update_node); sqlite3_clear_bindings(sql.update_node); +return TRUE; } -static void -db_insert_way_seg(node *n, way *w) +/** + * Insert way,node1,node2 triplet + */ +static gboolean +db_insert_way_n2n(way *w, node *nf, node *nt) { -sqlite3_bind_int(sql.insert_way_seg, 1, w->id); -sqlite3_bind_int(sql.insert_way_seg, 2, wsegcnt); -sqlite3_bind_int(sql.insert_way_seg, 3, n->id); +if (!w) { + g_printf("NULL WAY\n"); + return FALSE; +} -sqlite3_step(sql.insert_way_seg); -sqlite3_reset(sql.insert_way_seg); -sqlite3_clear_bindings(sql.insert_way_seg); -/* XXX: make this sooner ? */ -db_update_node_links(n); +if (!nf) { + g_printf("NULL NODE 1\n"); + return FALSE; +} -wsegcnt++; +if (!nt) { + g_printf("NULL NODE 2\n"); + return FALSE; } -static void -db_insert_way_seg2seg(way *w, node *nf, node *nt) -{ -sqlite3_bind_int(sql.insert_way_seg2seg, 1, w->id); -sqlite3_bind_int(sql.insert_way_seg2seg, 2, nf->id); -sqlite3_bind_int(sql.insert_way_seg2seg, 3, nt->id); +sqlite3_bind_int(sql.insert_way_n2n, 1, w->id); +sqlite3_bind_int(sql.insert_way_n2n, 2, nf->id); +sqlite3_bind_int(sql.insert_way_n2n, 3, nt->id); + +#ifdef VERBOSE_N2N +g_printf("%d [%d - %d]\n", w->id, nf->id, nt->id); +#endif -sqlite3_step(sql.insert_way_seg2seg); -sqlite3_reset(sql.insert_way_seg2seg); -sqlite3_clear_bindings(sql.insert_way_seg2seg); +sqlite3_step(sql.insert_way_n2n); +sqlite3_reset(sql.insert_way_n2n); +sqlite3_clear_bindings(sql.insert_way_n2n); db_update_node_links(nf); db_update_node_links(nt); - -wsegcnt++; +return TRUE; } +/** + * Insert way ref and int_ref + */ static void db_insert_way_ref(way *w) { @@ -465,6 +485,8 @@ if (!w->data) if (!w->data->ref && !w->data->int_ref) return; +way_refs++; + sqlite3_bind_int(sql.insert_way_ref, 1, w->id); if (w->data->ref) sqlite3_bind_text(sql.insert_way_ref, 2, w->data->ref, -1, SQLITE_TRANSIENT); @@ -476,13 +498,19 @@ sqlite3_reset(sql.insert_way_ref); sqlite3_clear_bindings(sql.insert_way_ref); } +/** + * Insert way name (nls names are not handled yet, ideas ?) + */ static void -db_insert_way_names(way *w) +db_insert_way_name(way *w) { if (!w->data) return; if (!w->data->name) return; + +way_names++; + sqlite3_bind_int(sql.insert_way_name, 1, w->id); sqlite3_bind_text(sql.insert_way_name, 2, w->data->name, -1, SQLITE_TRANSIENT); @@ -491,75 +519,34 @@ sqlite3_reset(sql.insert_way_name); sqlite3_clear_bindings(sql.insert_way_name); } -static void -db_delete_way_segments(way *w) -{ -sqlite3_bind_int(sql.delete_way_seg_wsid, 1, w->id); - -sqlite3_step(sql.delete_way_seg_wsid); -sqlite3_reset(sql.delete_way_seg_wsid); -sqlite3_clear_bindings(sql.delete_way_seg_wsid); -} - -static void -db_delete_way_segments2seg(way *w) -{ -sqlite3_bind_int(sql.delete_way_seg2seg, 1, w->id); - -sqlite3_step(sql.delete_way_seg2seg); -sqlite3_reset(sql.delete_way_seg2seg); -sqlite3_clear_bindings(sql.delete_way_seg2seg); -} - -static gboolean -db_insert_way_segments(segment *s, way *w) -{ -node *f; -node *t; - -if (!s) { - w->id=0; -#ifdef VERBOSE - g_printerr("ERROR: null segment!\n"); -#endif - return FALSE; -} - -f=osm_find_node(s->from); -t=osm_find_node(s->to); -if (!f) { - w->id=0; - return FALSE; -} -if (!t) { - w->id=0; - return FALSE; -} - -#if OLD_WAYSEG -db_insert_way_seg(f,w); -if (w->ncnt==wsegcnt) - db_insert_way_seg(t,w); -#else -db_insert_way_seg2seg(w,f,t); -#endif - -return TRUE; -} - +/** + * Insert all data for the given way + * - name + * - ref + * - nodes + * + */ static gboolean db_insert_way(way *w) { +GSList *iter; + if (!w) return FALSE; -/* Skip things we can't use */ +/* Skip things we don't use (yet) */ if (w->type==WAY_UNWAYED || w->type>WAY_ROAD_END) return TRUE; -wsegcnt=0; -g_slist_foreach(w->nodes, db_insert_way_segments, w); -/* Invalid/Skipped segments/nodes in way */ +/* Insert nodes */ +for (iter=w->nodes; iter!=NULL; iter=iter->next) { + if (!iter->next) + break; + db_insert_way_n2n(w, iter->data, iter->next->data); +} + +print_way(w); + if (w->id==0) return FALSE; @@ -577,7 +564,7 @@ sqlite3_reset(sql.insert_way_data); sqlite3_clear_bindings(sql.insert_way_data); db_insert_way_ref(w); -db_insert_way_names(w); +db_insert_way_name(w); osm_free_way_data(w); return TRUE; @@ -585,9 +572,10 @@ return TRUE; /********************************************************************/ -gchar *get_attr_key_value(const gchar **p, gchar *key) +static gchar * +get_attr_key_value(const gchar **p, gchar *key) { -char **d; +gchar **d; d=p; while (*d!=NULL) { @@ -604,11 +592,11 @@ return NULL; static tag_state_t check_tag(const gchar *tag) { if (strcmp(tag,"node")==0) return IN_NODE_TAG; -else if (strcmp(tag,"segment")==0) return IN_SEGMENT_TAG; -else if (strcmp(tag,"seg")==0) return IN_SEG_TAG; +else if (strcmp(tag,"nd")==0) return IN_WNODE_TAG; else if (strcmp(tag,"way")==0) return IN_WAY_TAG; else if (strcmp(tag,"tag")==0) return IN_KEY_TAG; else if (strcmp(tag,"osm")==0) return IN_OSM_TAG; +else if (strcmp(tag,"bound")==0) return IN_BOUND_TAG; else return ERROR; } @@ -644,7 +632,7 @@ g_print("\n"); } #endif -static inline gboolean +static gboolean osm_node_check_box(gdouble nlat, gdouble nlon) { if (use_bbox==FALSE) @@ -690,47 +678,7 @@ g_hash_table_insert(osm_nodes, GINT_TO_POINTER(id), n); return n; } -static void -print_segment(segment *s) -{ -g_assert(s); -g_printf("Segment %d %d-%d:\n", s->id, s->from, s->to); -} - -static void -_print_segment_helper(segment *s, void *data) -{ -print_segment(s); -} - -void -print_way(way *w) -{ -g_assert(w); -g_printf("Way #%d(%d/%d): %s [%s:%s:%s]\n", - w->id, - g_slist_length(w->nodes), - w->type, - w->data ? w->data->name ? w->data->name : "" : "", - w->flags & W_ONEWAY ? "-" : "=", - w->flags & W_ROUNDABOUT ? "O" : "-", - w->flags & W_LINK ? "|" : " "); -#ifdef VERBOSE -g_slist_foreach(w->nodes, _print_segment_helper, NULL); -#endif -} - -inline segment * -osm_get_segment_for_way(guint32 sid) -{ -segment *s; -s=g_hash_table_lookup(osm_segments, GINT_TO_POINTER(sid)); -if (!s && use_bbox==FALSE) - g_printerr("ERROR: Segment %d not found!\n", sid); -return s; -} - -inline node * +static node * osm_find_node(guint32 nid) { node *n; @@ -742,23 +690,12 @@ if (!n) return n; } -void -osm_new_segment(guint32 id, guint32 from, guint32 to) -{ -segment *s; - -s=g_slice_new(segment); -s->id=id; -s->from=from; -s->to=to; -g_hash_table_insert(osm_segments, GINT_TO_POINTER(id), s); -} - void osm_new_way_data(way *w) { if (w==NULL) return; if (w->data!=NULL) return; + w->data=g_slice_new(way_data); w->data->name=NULL; w->data->ref=NULL; @@ -795,17 +732,18 @@ w->data=NULL; w->ncnt=0; w->flags=0; +/* Add to list of ways */ osm_ways=g_slist_prepend(osm_ways, w); return w; } void -osm_way_new_seg(way *w, gint id) +osm_way_new_node(way *w, gint nid) { -segment *s; +node *n; -s=osm_get_segment_for_way(id); -w->nodes=g_slist_prepend(w->nodes, s); +n=osm_find_node(nid); +w->nodes=g_slist_prepend(w->nodes, n); w->ncnt++; } @@ -823,7 +761,7 @@ gchar **place; if (!n->data) return 0; -isin=g_hash_table_lookup(osm_node_isin, n->id); +isin=g_hash_table_lookup(osm_node_isin, GINT_TO_POINTER(n->id)); if (!isin) return 0; @@ -905,35 +843,50 @@ if (dbnode_cnt % 26214==0) g_printf("Nodes: %f%%\n",((float)dbnode_cnt/(float)node_cnt)*100); } -void +/** + * Check node type and insert as POI or Place + * Discard extra data after insert. + */ +gboolean osm_node_save_poi(node *n, gpointer user_data) { if (!n) { g_printerr("ERROR: null poi\n"); - return; + return FALSE; } -if (!n->data) - return; +if (!n->data) { + g_printerr("POI node with no data ?\n"); + return FALSE; +} n->data->isin=osm_find_node_place(n); if (n->data->type>NODE_POI_START && n->data->typedata->type>NODE_PLACE_START && n->data->typedata->type); + osm_free_node_data(n); + return FALSE; } /* Free the extra info, we don't need it anymore */ osm_free_node_data(n); +return TRUE; } void osm_planet_poi_clear_nodes(void) { g_print("Removing old OSM POIs...\n"); +sqlite3_exec(db, "begin;", NULL, NULL, NULL); sqlite3_step(sql.delete_osm_poi); sqlite3_step(sql.delete_place); +sqlite3_exec(db, "commit;", NULL, NULL, NULL); } void @@ -968,7 +921,7 @@ sqlite3_exec(db, "commit;", NULL, NULL, NULL); /*********************************************************************/ void -osm_node_save_way(way *value, gpointer user_data) +osm_way_save(way *value, gpointer user_data) { dbway_cnt++; db_insert_way(value); @@ -983,9 +936,9 @@ osm_planet_clear_ways(void) { g_print("Clearing old data:\n"); sqlite3_step(sql.delete_way); -sqlite3_step(sql.delete_way_seg); sqlite3_step(sql.delete_way_name); sqlite3_step(sql.delete_way_ref); +sqlite3_step(sql.delete_way_n2n); } void @@ -993,7 +946,7 @@ osm_planet_save_ways(void) { g_print("Inserting new ways:\n"); sqlite3_exec(db, "begin;", NULL, NULL, NULL); -g_slist_foreach(osm_ways, osm_node_save_way, NULL); +g_slist_foreach(osm_ways, osm_way_save, NULL); sqlite3_exec(db, "commit;", NULL, NULL, NULL); } @@ -1025,7 +978,7 @@ _osm_tag_start(void *userData, const char *name, const char **atts) { tag_state_t t; gchar *k, *v; -guint32 id, idf, idt; +guint32 id, ndref; gdouble nlat, nlon; t=check_tag(name); @@ -1049,31 +1002,19 @@ switch (t) { node_skip_cnt++; } break; - case IN_SEGMENT_TAG: - tag_parent=IS_SEGMENT; - seg_cnt++; - - id=atoi(get_attr_key_value(atts, "id")); - idf=atoi(get_attr_key_value(atts, "from")); - idt=atoi(get_attr_key_value(atts, "to")); - /* XXX: Check that nodes exist only if using bbox */ - if (use_bbox==TRUE) { - if (osm_find_node(idf)==NULL) - return; - if (osm_find_node(idt)==NULL) - return; - } - osm_new_segment(id, idf, idt); - break; case IN_WAY_TAG: tag_parent=IS_WAY; way_cnt++; id=atoi(get_attr_key_value(atts, "id")); cway=osm_new_way(id); break; - case IN_SEG_TAG: - id=atoi(get_attr_key_value(atts, "id")); - osm_way_new_seg(cway, id); + case IN_WNODE_TAG: + ndref=atoi(get_attr_key_value(atts, "ref")); + if (use_bbox==TRUE) { + if (osm_find_node(ndref)==NULL) + return; + } + osm_way_new_node(cway, ndref); break; case IN_KEY_TAG: switch (tag_parent) { @@ -1085,9 +1026,18 @@ switch (t) { if (!osm_node_tags) return; + /* Insert key/value pairs into hash table */ k=get_attr_key_value(atts, "k"); + if (strcmp(k,"created_by")==0) + return; + if (strcmp(k,"source")==0) + return; v=get_attr_key_value(atts, "v"); +#ifdef VERBOSE_KEYS + g_printf("NODE: K=[%s] V=[%s]\n", k, v); +#endif + if (cnode==NULL) { g_printf("In node tags but current node is NULL\n"); return; @@ -1100,10 +1050,19 @@ switch (t) { gint i; k=get_attr_key_value(atts, "k"); + if (strcmp(k,"created_by")==0) + return; + if (strcmp(k,"source")==0) + return; v=get_attr_key_value(atts, "v"); +#ifdef VERBOSE_KEYS + g_printf("WAY: K=[%s] V=[%s]\n", k, v); +#endif + osm_new_way_data(cway); + /* XXX: something faster than this should be used */ if (strcasecmp(k, "name")==0) cway->data->name=g_strdup(v); else if (strcasecmp(k, "ref")==0) @@ -1139,6 +1098,8 @@ switch (t) { if (wayinfo[i].oneway==TRUE) cway->flags|=W_ONEWAY; cway->type=wayinfo[i].type; + if (cway->data->speed==0) + cway->data->speed=wayinfo[i].defspeed; break; } } @@ -1150,11 +1111,12 @@ switch (t) { } break; - case IS_SEGMENT: - /* Just ignore tagged segments */ - break; } break; + case IN_BOUND_TAG: + /* Ignore for now */ + g_printf("Ignoring bound tag\n"); + break; default: tag_parent=IS_NONE; g_printf("Unknown tag: %s\n", name); @@ -1207,46 +1169,52 @@ switch (t) { if (cnode->data->type==NODE_PLAIN) { osm_free_node_data(cnode); - } else if (cnode->data->name) { + } else { osm_poi=g_slist_prepend(osm_poi, cnode); - switch (cnode->data->type) { - case NODE_PLACE_COUNTRY: - g_hash_table_insert(osm_place_country, cnode->data->name, cnode); - break; - case NODE_PLACE_CITY: - case NODE_PLACE_TOWN: - g_hash_table_insert(osm_place_city, cnode->data->name, cnode); - break; - case NODE_PLACE_SUBURB: - g_hash_table_insert(osm_place_suburb, cnode->data->name, cnode); - break; - case NODE_PLACE_VILLAGE: - case NODE_PLACE_HAMLET: - g_hash_table_insert(osm_place_village, cnode->data->name, cnode); - break; - case NODE_PLACE_ISLAND: - /* Ignore for now */ - break; - default:; + if (cnode->data->name) { + switch (cnode->data->type) { + case NODE_PLACE_COUNTRY: + g_hash_table_insert(osm_place_country, cnode->data->name, cnode); + break; + case NODE_PLACE_CITY: + case NODE_PLACE_TOWN: + g_hash_table_insert(osm_place_city, cnode->data->name, cnode); + break; + case NODE_PLACE_SUBURB: + g_hash_table_insert(osm_place_suburb, cnode->data->name, cnode); + break; + case NODE_PLACE_VILLAGE: + case NODE_PLACE_HAMLET: + g_hash_table_insert(osm_place_village, cnode->data->name, cnode); + break; + case NODE_PLACE_ISLAND: + /* Ignore for now */ + break; + default:; + } } } g_hash_table_destroy(osm_node_tags); cnode=NULL; break; case IN_WAY_TAG: + cway->nodes=g_slist_reverse(cway->nodes); + + print_way(cway); + + /* XXX */ + if (cway->data && cway->data->name==NULL && cway->data->ref==NULL && + cway->data->int_ref==NULL && cway->data->layer==0 && cway->data->speed==0) + osm_free_way_data(cway); + if (way_cnt % 32767==0) { g_printf("Ways: %d\n", way_cnt); } - cway->nodes=g_slist_reverse(cway->nodes); - if (cway->data && cway->data->name==NULL && cway->data->ref==NULL && - cway->data->int_ref==NULL && cway->data->layer==0) - osm_free_way_data(cway); + cway=NULL; break; - case IN_SEGMENT_TAG: - if (seg_cnt % 50000==0) { - g_printf("Segments: %d\n", seg_cnt); - } + case IN_BOUND_TAG: + /* */ break; case IN_OSM_TAG: g_printf("Planet loaded.\n"); @@ -1261,7 +1229,6 @@ static void storage_init(void) { osm_nodes=g_hash_table_new(g_direct_hash, g_direct_equal); -osm_segments=g_hash_table_new(g_direct_hash, g_direct_equal); osm_place_country=g_hash_table_new(g_str_hash, g_str_equal); osm_place_city=g_hash_table_new(g_str_hash, g_str_equal); @@ -1275,7 +1242,6 @@ static void storage_free(void) { g_hash_table_destroy(osm_nodes); -g_hash_table_destroy(osm_segments); g_hash_table_destroy(osm_place_country); g_hash_table_destroy(osm_place_city); @@ -1364,11 +1330,10 @@ static void print_memory_usage(void) { g_print("Memory usage per item:\n"); -g_printf("Node size: %d\n", sizeof(node)); -g_printf("NodeD size: %d\n", sizeof(node_data)); -g_printf("Seg size: %d\n", sizeof(segment)); -g_printf("Way size: %d\n", sizeof(way)); -g_printf("WayD size: %d\n", sizeof(way_data)); +g_printf("Node size: %d\n", (int)sizeof(node)); +g_printf("NodeD size: %d\n", (int)sizeof(node_data)); +g_printf("Way size: %d\n", (int)sizeof(way)); +g_printf("WayD size: %d\n", (int)sizeof(way_data)); } /************************************************************************/ @@ -1402,14 +1367,17 @@ print_memory_usage(); osm_planet_parser_init(); osm_planet_parse_file(argv[1]); -g_printf("Planet loaded.\nTotal nodes %d, POIs: %d, Segments %d and Ways %d.\n", - node_cnt, noded_cnt, seg_cnt, way_cnt); +g_printf("Total nodes %d, POIs: %d and Ways %d.\n", + node_cnt, noded_cnt, way_cnt); g_printf("Cities/Towns: %d\n", g_hash_table_size(osm_place_city)); g_printf("Villages/Hamlets: %d\n", g_hash_table_size(osm_place_village)); g_printf("Suburbs: %d\n", g_hash_table_size(osm_place_suburb)); osm_planet_save_to_db(); osm_planet_parser_deinit(); + +g_printf("Named ways: %d, Numbered ways: %d\n", way_names, way_refs); + db_close(); sync();