From 47881ecccc8aebaad292b41cf4936458b3519936 Mon Sep 17 00:00:00 2001 From: Kaj-Michael Lang Date: Mon, 16 Jul 2007 09:42:31 +0300 Subject: [PATCH] Optimize memory usage. Recognize more way and node types. --- src/osm.c | 383 ++++++++++++++++++++++++++++++++++++++---------------- src/osm.h | 24 +++- 2 files changed, 297 insertions(+), 110 deletions(-) diff --git a/src/osm.c b/src/osm.c index 4673848..cb88659 100644 --- a/src/osm.c +++ b/src/osm.c @@ -1,5 +1,9 @@ #define _GNU_SOURCE +#include +#include +#include +#include #include #include #include @@ -10,47 +14,51 @@ #include "osm.h" -#define W_ONEWAY (1 << 1) -#define W_ROUNDABOUT (1 << 2) -#define W_LINK (1 << 3) - gint node_cnt=0; +gint noded_cnt=0; gint way_cnt=0; gint seg_cnt=0; +typedef struct _node_data node_data; +struct _node_data { + gchar *name; + node_type_t type; +}; + /* Node type */ typedef struct _node node; struct _node { - gint id; - gdouble lat; - gdouble lon; - gint count; - node_type_t type; - gchar *name; - void *data; + gint32 id; + gfloat lat; + gfloat lon; + node_data *data; }; +/* Segment */ typedef struct _segment segment; struct _segment { - gint id; - node *from; - node *to; + gint32 id; + gint32 from; + gint32 to; +}; + +typedef struct _way_data way_data; +struct _way_data { + gchar *name; + gchar *ref; + gchar *int_ref; + gchar *is_in; }; /* Segment/Way structure */ typedef struct _way way; struct _way { - gint id; + gint32 id; way_type_t type; - gint layer; - gint flags; + gshort layer; gchar *name; - gchar *ref; - gchar *int_ref; - gboolean oneway; - gboolean link; - gboolean area; - GList *nodes; + guint flags; + GSList *nodes; }; typedef enum { @@ -93,6 +101,8 @@ struct _nodeinfo { { "amenity", "supermarket", NODE_AMENITY_SHOP }, { "amenity", "shop", NODE_AMENITY_SHOP }, { "amenity", "place_of_worship",NODE_AMENITY_POW }, + { "railway", "station", NODE_AMENITY_RAILWAY_STATION }, + { "railway", "halt", NODE_AMENITY_RAILWAY_HALT }, { "place", "city", NODE_PLACE_CITY }, { "place", "town", NODE_PLACE_CITY }, @@ -100,6 +110,8 @@ struct _nodeinfo { { "place", "hamlet", NODE_PLACE_VILLAGE }, { "place", "suburb", NODE_PLACE_SUBURB }, + { "highway", "traffic_signals", NODE_TRAFFIC_SIGNALS }, + { NULL, NULL, NODE_PLAIN } }; @@ -109,7 +121,7 @@ struct _wayinfo { way_type_t type; gboolean oneway, link, area; } wayinfo[] = { - { "highway", "motorway",WAY_MOTORWAY, TRUE, FALSE, FALSE }, + { "highway", "motorway",WAY_MOTORWAY, TRUE, FALSE, FALSE }, { "highway", "motorway_link",WAY_MOTORWAY, TRUE, TRUE, FALSE }, { "highway", "trunk",WAY_TRUNK, FALSE, FALSE, FALSE }, { "highway", "trunk_link",WAY_TRUNK, FALSE, TRUE, FALSE }, @@ -120,12 +132,16 @@ struct _wayinfo { { "highway", "tertiary",WAY_TERTIARY, FALSE, FALSE, FALSE }, { "highway", "unclasified",WAY_UNCLASSIFIED, FALSE, FALSE, FALSE }, { "highway", "residential",WAY_RESIDENTIAL, FALSE, FALSE, FALSE }, + { "highway", "service",WAY_SERVICE, FALSE, FALSE, FALSE }, { "highway", "track",WAY_TRACK, FALSE, FALSE, FALSE }, { "highway", "unsurfaced",WAY_TRACK, FALSE, FALSE, FALSE }, { "highway", "footway",WAY_FOOTWAY, FALSE, FALSE, FALSE }, { "highway", "cycleway",WAY_CYCLEWAY, FALSE, FALSE, FALSE }, { "railway", "rail",WAY_RAIL, FALSE, FALSE, FALSE }, { "natural", "water",WAY_WATER, FALSE, FALSE, TRUE }, + { "waterway", "river",WAY_WATER, FALSE, FALSE, FALSE }, + { "waterway", "canal",WAY_WATER, FALSE, FALSE, FALSE }, + { "waterway", "stream",WAY_WATER, FALSE, FALSE, FALSE }, { NULL, NULL, WAY_UNWAYED, FALSE, FALSE, FALSE } }; @@ -142,17 +158,29 @@ way *cway=NULL; struct sql_stmt { sqlite3_stmt *insert_poi; sqlite3_stmt *delete_osm_poi; + sqlite3_stmt *insert_node; + sqlite3_stmt *delete_nodes; + sqlite3_stmt *select_node; + sqlite3_stmt *update_node; + sqlite3_stmt *insert_way_data; sqlite3_stmt *insert_way_ref; sqlite3_stmt *insert_way_name; sqlite3_stmt *insert_way_seg; + sqlite3_stmt *delete_way; + sqlite3_stmt *delete_way_seg; + sqlite3_stmt *delete_way_name; + sqlite3_stmt *delete_way_ref; + sqlite3_stmt *insert_place; }; struct sql_stmt sql; gint wsegcnt; +node *osm_find_node(gint nid); + /****************************************************/ /* Functions */ /****************************************************/ @@ -170,27 +198,39 @@ return TRUE; void db_prepare(void) { -sqlite3_prepare(db, "insert into nodes (id,lat,lon) values (?, ?, ?)", +/* Way nodes */ +sqlite3_prepare(db, "insert into nodes (nid,lat,lon,l) values (?, ?, ?, 0)", -1, &sql.insert_node, NULL); +sqlite3_prepare(db, "select lat,lon,l from nodes where nid=?", + -1, &sql.select_node, NULL); +sqlite3_prepare(db, "delete from nodes", -1, &sql.delete_nodes, NULL); +sqlite3_prepare(db, "update nodes set l=l+1 where nid=?", -1, &sql.update_node, NULL); /* POI nodes */ sqlite3_prepare(db, "insert into poi (osm_id, lat, lon, label, desc, cat_id, public, priority)" - " values (?, ?, ?, ?, ?, ?, 1, ?)", - -1, &sql.insert_poi, NULL); + " values (?, ?, ?, ?, ?, ?, 1, ?)", -1, &sql.insert_poi, NULL); sqlite3_prepare(db, "delete from poi where osm_id>0", -1, &sql.delete_osm_poi, NULL); /* Ways */ sqlite3_prepare(db, "insert into way (wid,nodes,type,flags) values (?, ?, ?, ?)", -1, &sql.insert_way_data, NULL); +sqlite3_prepare(db, "delete from way", -1, + &sql.delete_way, NULL); sqlite3_prepare(db, "insert into way_seg (wsid,num,node) values (?, ?, ?)", -1, &sql.insert_way_seg, NULL); +sqlite3_prepare(db, "delete from way_seg", -1, + &sql.delete_way_seg, NULL); sqlite3_prepare(db, "insert into name (nid,name,lang) values (?, ?, ?)", -1, &sql.insert_way_name, NULL); +sqlite3_prepare(db, "delete from name", -1, + &sql.delete_way_name, NULL); sqlite3_prepare(db, "insert into ref (rid,ref,int_ref) values (?, ?, ?)", -1, &sql.insert_way_ref, NULL); +sqlite3_prepare(db, "delete from ref", -1, + &sql.delete_way_ref, NULL); } /********************************************************************/ @@ -198,9 +238,14 @@ sqlite3_prepare(db, "insert into ref (rid,ref,int_ref) values (?, ?, ?)", gboolean db_insert_node(node *n) { +gint32 lat, lon; + +lat=osm_dlat2ilat(n->lat); +lon=osm_dlon2ilon(n->lon); + sqlite3_bind_int(sql.insert_node, 1, n->id); -sqlite3_bind_int(sql.insert_node, 2, n->lat); -sqlite3_bind_int(sql.insert_node, 3, n->lon); +sqlite3_bind_int(sql.insert_node, 2, lat); +sqlite3_bind_int(sql.insert_node, 3, lon); sqlite3_step(sql.insert_node); sqlite3_reset(sql.insert_node); return TRUE; @@ -212,13 +257,27 @@ db_insert_poi(node *n) sqlite3_bind_int(sql.insert_poi, 1, n->id); sqlite3_bind_double(sql.insert_poi, 2, n->lat); sqlite3_bind_double(sql.insert_poi, 3, n->lon); -sqlite3_bind_text(sql.insert_poi, 4, n->name, -1, SQLITE_TRANSIENT); -/* sqlite3_bind_text(sql.insert_poi, 5, n->desc, -1, SQLITE_TRANSIENT); */ -sqlite3_bind_int(sql.insert_poi, 6, n->type); -sqlite3_bind_int(sql.insert_poi, 7, n->type/100); +if (n->data->name) + sqlite3_bind_text(sql.insert_poi, 4, n->data->name, -1, SQLITE_TRANSIENT); +#if 0 +if (n->data->desc) + sqlite3_bind_text(sql.insert_poi, 5, n->data->desc, -1, SQLITE_TRANSIENT); +#endif +sqlite3_bind_int(sql.insert_poi, 6, n->data->type); +sqlite3_bind_int(sql.insert_poi, 7, n->data->type/100); sqlite3_step(sql.insert_poi); sqlite3_reset(sql.insert_poi); +return TRUE; +} + +void +db_update_node_links(node *n) +{ +sqlite3_bind_int(sql.update_node, 1, n->id); + +sqlite3_step(sql.update_node); +sqlite3_reset(sql.update_node); } void @@ -231,15 +290,27 @@ sqlite3_bind_int(sql.insert_way_seg, 3, n->id); sqlite3_step(sql.insert_way_seg); sqlite3_reset(sql.insert_way_seg); +db_update_node_links(n); + wsegcnt++; } void db_insert_way_segments(segment *s, way *w) { -db_insert_way_seg(s->from,w); -if (g_list_length(w->nodes)==wsegcnt) - db_insert_way_seg(s->to,w); +node *f; +node *t; + +f=osm_find_node(s->from); +t=osm_find_node(s->to); +if (!f) + return; +if (!t) + return; + +db_insert_way_seg(f,w); +if (g_slist_length(w->nodes)==wsegcnt) + db_insert_way_seg(t,w); } gboolean @@ -249,7 +320,7 @@ if (w->type==WAY_UNWAYED) return TRUE; sqlite3_bind_int(sql.insert_way_data, 1, w->id); -sqlite3_bind_int(sql.insert_way_data, 2, g_list_length(w->nodes)); +sqlite3_bind_int(sql.insert_way_data, 2, g_slist_length(w->nodes)); sqlite3_bind_int(sql.insert_way_data, 3, w->type); sqlite3_bind_int(sql.insert_way_data, 4, w->flags); @@ -257,10 +328,14 @@ sqlite3_step(sql.insert_way_data); sqlite3_reset(sql.insert_way_data); wsegcnt=0; -if (w->area==TRUE) { +if (w->flags & W_AREA) { +#if 1 g_print("Areas not handled yet\n"); +#else + g_slist_foreach(w->nodes, db_insert_area_segments, w); +#endif } else { - g_list_foreach(w->nodes, db_insert_way_segments, w); + g_slist_foreach(w->nodes, db_insert_way_segments, w); } return TRUE; @@ -279,7 +354,7 @@ while (*d!=NULL) { } } -gchar *get_attr_key_value(gchar **p, const gchar *key) +gchar *get_attr_key_value(const gchar **p, gchar *key) { char **d; @@ -308,15 +383,15 @@ else return ERROR; /********************************************************************/ -static gint -osm_dlat2ilat(double lat) +gint32 +osm_dlat2ilat(gdouble lat) { return lat > 85.051128779 ? INT_MAX : lat < -85.051128779 ? INT_MIN : lrint(log(tan(M_PI_4l+lat*M_PIl/360))/M_PIl*INT_MAX); } -static gint -osm_dlon2ilon(double lon) +gint32 +osm_dlon2ilon(gdouble lon) { return lrint(lon/180*INT_MAX); } @@ -326,24 +401,51 @@ return lrint(lon/180*INT_MAX); void print_node (node *n) { -if (n->type) - g_printf("ID: %d [%f:%f][%s](%d) %d\n", - n->id, n->lat, n->lon, n->name, n->type, n->count); +g_assert(n); +if (n->data) { + g_printf("N: %d [%f:%f][%s](%d) %d\n", + n->id, n->lat, n->lon, + n->data->name ? n->data->name : "-", + n->data->type); +} else { + g_printf("N: %d [%f:%f]\n", + n->id, n->lat, n->lon); +} + +} + +void +osm_new_node_data(node *n) +{ +if (n==NULL) return; +if (n->data!=NULL) return; +n->data=g_slice_new(node_data); +n->data->name=NULL; +n->data->type=NODE_PLAIN; +noded_cnt++; +} + +void +osm_free_node_data(node *n) +{ +if (n->data->name) + g_free(n->data->name); +g_slice_free(node_data, n->data); +n->data=NULL; +noded_cnt--; } node * -osm_new_node(gint id, double lat, double lon) +osm_new_node(gint id, gdouble lat, gdouble lon) { -node *n; +node *n=NULL; n=g_slice_new(node); +g_assert(n); n->id=id; n->lat=lat; n->lon=lon; -n->type=NODE_PLAIN; -n->count=0; -n->data=(void *)NULL; -n->name=NULL; +n->data=(node_data *)NULL; g_hash_table_insert(osm_nodes, GINT_TO_POINTER(id), n); return n; } @@ -351,14 +453,10 @@ return n; void print_segment(segment *s) { -g_printf("Segment %d:\n", s->id); -if (s->from) - print_node(s->from); -if (s->to) - print_node(s->to); +g_printf("Segment %d %d-%d:\n", s->id, s->from, s->to); } -static void +void _print_segment_helper(segment *s, void *data) { print_segment(s); @@ -367,28 +465,23 @@ print_segment(s); void print_way(way *w) { -g_printf("Way #%d(%d/%d): %s %s %s [%s:%s] %d\n", - w->id, g_list_length(w->nodes), w->type, +g_printf("Way #%d(%d/%d): %s [%s:%s:%s]\n", + w->id, + g_slist_length(w->nodes), + w->type, w->name ? w->name : "", - w->ref ? w->ref : "", - w->int_ref ? w->int_ref : "", - w->oneway==TRUE ? "-" : "=", w->link==TRUE ? "L" : " ", - w->flags); + w->flags & W_ONEWAY ? "-" : "=", + w->flags & W_ROUNDABOUT ? "O" : "-", + w->flags & W_LINK ? "|" : " "); #ifdef VERBOSE -g_list_foreach(w->nodes, _print_segment_helper, NULL); +g_slist_foreach(w->nodes, _print_segment_helper, NULL); #endif } segment * osm_get_segment_for_way(gint sid) { -segment *s; -s=g_hash_table_lookup(osm_segments, GINT_TO_POINTER(sid)); -if (s) { - s->from->count++; - s->to->count++; -} -return s; +return g_hash_table_lookup(osm_segments, GINT_TO_POINTER(sid)); } node * @@ -404,8 +497,8 @@ segment *s; s=g_slice_new(segment); s->id=id; -s->from=osm_find_node(from); -s->to=osm_find_node(to); +s->from=from; +s->to=to; g_hash_table_insert(osm_segments, GINT_TO_POINTER(id), s); } @@ -416,12 +509,9 @@ way *w; w=g_slice_new(way); w->id=id; -w->oneway=FALSE; w->nodes=NULL; w->type=WAY_UNWAYED; w->name=NULL; -w->ref=NULL; -w->int_ref=NULL; w->layer=0; w->flags=0; @@ -435,26 +525,63 @@ osm_way_new_seg(way *w, gint id) segment *s; s=osm_get_segment_for_way(id); -w->nodes=g_list_append(w->nodes, s); +w->nodes=g_slist_append(w->nodes, s); } /***********************************************************************/ +void +osm_node_save_node(gint key, node *value, gpointer user_data) +{ +db_insert_node(value); +} + void osm_node_save_poi(gint key, node *value, gpointer user_data) { -if (value->type>NODE_POI_START && value->typedata==NULL) + return; + +if (value->data->name==NULL) + return; + +if (value->data->type>NODE_POI_START && value->data->typename=g_strdup(v); + if (cnode==NULL) { + g_printf("In node tags but current node is NULL\n"); + return; + } - if (cnode->type==NODE_PLAIN) { + if (cnode->data==NULL) + osm_new_node_data(cnode); + + if (strcasecmp(k, "name")==0) + cnode->data->name=g_strdup(v); + else if (cnode->data->type==NODE_PLAIN) { for (i=0; nodeinfo[i].k; i++) { if (strcasecmp (nodeinfo[i].k, k)==0 && strcasecmp (v, nodeinfo[i].v)==0) { - cnode->type=nodeinfo[i].type; + cnode->data->type=nodeinfo[i].type; } } } @@ -546,26 +682,36 @@ switch (t) { k=get_attr_key_value(atts, "k"); v=get_attr_key_value(atts, "v"); + if (strcasecmp(k, "name")==0) cway->name=g_strdup(v); - if (strcasecmp(k, "ref")==0) +#if 0 + else if (strcasecmp(k, "ref")==0) cway->ref=g_strdup(v); - if (strcasecmp(k, "int_ref")==0) + else if (strcasecmp(k, "int_ref")==0) cway->int_ref=g_strdup(v); +#endif + if (cway->type==WAY_UNWAYED) { for (i=0; wayinfo[i].k; i++) { if (strcasecmp (wayinfo[i].k, k)==0 && strcasecmp (v, wayinfo[i].v)==0) { - cway->oneway=wayinfo[i].oneway; - cway->link=wayinfo[i].link; - cway->area=wayinfo[i].area; + if (wayinfo[i].link==TRUE) + cway->flags|=W_LINK; + if (wayinfo[i].area==TRUE) + cway->flags|=W_AREA; + if (wayinfo[i].oneway==TRUE) + cway->flags|=W_ONEWAY; cway->type=wayinfo[i].type; } } } - if (strcasecmp(k, "oneway")==0) - cway->oneway=TRUE; - if (strcasecmp(k, "layer")==0) + if ((strcasecmp(k, "junction")==0) && (strcasecmp(v, "roundabout")==0)) { + cway->flags|=W_ROUNDABOUT; + cway->flags|=W_ONEWAY; + } else if (strcasecmp(k, "oneway")==0) + cway->flags|=W_ONEWAY; + else if (strcasecmp(k, "layer")==0) cway->layer=atoi(v); } break; @@ -588,17 +734,24 @@ tag_state_t t; t=check_tag(name); switch (t) { case IN_NODE_TAG: - print_node(cnode); + if (node_cnt % 100000==0) { + g_printf("Nodes: %d/%d\n", node_cnt, noded_cnt); + } + if (cnode->data && cnode->data->type==NODE_PLAIN) + osm_free_node_data(cnode); cnode=NULL; break; case IN_WAY_TAG: - if (cway->oneway==TRUE) - cway->flags|=W_ONEWAY; - if (cway->link==TRUE) - cway->flags|=W_LINK; - print_way(cway); + if (way_cnt % 10000==0) { + g_printf("Ways: %d\n", way_cnt); + } cway=NULL; break; + case IN_SEGMENT_TAG: + if (seg_cnt % 50000==0) { + g_printf("Segments: %d\n", seg_cnt); + } + break; default:; } } @@ -640,8 +793,15 @@ g_printf("ERROR: %s\n", msg); return ret; } +/************************************************************************/ + int main (int argc, char **argv) { + +g_printf("Node size: %d\n", sizeof(node)); +g_printf("Seg size: %d\n", sizeof(segment)); +g_printf("Way size: %d\n", sizeof(way)); + if (argc!=2) { return print_fail("No planet XML file given", 1); } else { @@ -658,9 +818,14 @@ osm_ways=g_hash_table_new(g_direct_hash, g_direct_equal); osm_planet_parse_file(argv[1]); -g_printf("Total nodes %d segments %d and ways %d\n", node_cnt, seg_cnt, way_cnt); +g_printf("Total nodes %d/%d segments %d and ways %d\n", + node_cnt, noded_cnt, seg_cnt, way_cnt); osm_planet_save_to_db(); +g_hash_table_unref(osm_nodes); +g_hash_table_unref(osm_segments); +g_hash_table_unref(osm_ways); + sqlite3_close(db); } diff --git a/src/osm.h b/src/osm.h index 5fcf055..747ee15 100644 --- a/src/osm.h +++ b/src/osm.h @@ -1,7 +1,14 @@ /* #define VERBOSE */ -#define FILE_BUFFER 65535 +#define FILE_BUFFER 8192 #define OSM_DB_FILE "osm-planet.db" +/* Flags */ +#define W_ONEWAY (1 << 1) +#define W_ROUNDABOUT (1 << 2) +#define W_LINK (1 << 3) +#define W_AREA (1 << 4) + + /* POI category ID's are special, the # is divied * by 100 and the result is the importance/priority of the poi. * This can be used by the client program to discard @@ -22,6 +29,12 @@ typedef enum { NODE_AMENITY_BANK=206, NODE_AMENITY_POST=210, NODE_AMENITY_POST_BOX=211, + NODE_AMENITY_TAXI=220, + NODE_AMENITY_RAILWAY_STATION=230, + NODE_AMENITY_RAILWAY_HALT=231, + NODE_AMENITY_BUS_STATION=236, + NODE_AMENITY_BOAT=240, + NODE_AMENITY_AIRPORT=250, NODE_AMENITY_FOOD=301, NODE_AMENITY_PUB=302, NODE_AMENITY_CINEMA=303, @@ -36,7 +49,10 @@ typedef enum { NODE_PLACE_VILLAGE=1050, NODE_PLACE_CITY=1100, NODE_PLACE_END=1500, + /* Other */ + NODE_TRAFFIC_SIGNALS=2000, + NODE_TYPE_MAX=9000 } node_type_t; @@ -49,10 +65,16 @@ typedef enum { WAY_TERTIARY, WAY_UNCLASSIFIED, WAY_RESIDENTIAL, + WAY_SERVICE, WAY_TRACK, WAY_FOOTWAY, WAY_CYCLEWAY, + WAY_RAIL, + WAY_WATER, + WAY_WATER_RIVER, + WAY_WATER_STREAM, + WAY_WATER_CANAL, WAY_OTHER } way_type_t; -- 2.39.5