From ff7a56be374000f730f72646229f1643bd96713e Mon Sep 17 00:00:00 2001 From: Kaj-Michael Lang Date: Wed, 4 Jul 2007 15:12:22 +0300 Subject: [PATCH] Add start of a OSM planet to sqlite database parser. Right now it can parse some of the OSM POI nodes and ways. --- src/Makefile.am | 8 +- src/osm.c | 666 ++++++++++++++++++++++++++++++++++++++++++++++++ src/osm.h | 58 +++++ 3 files changed, 731 insertions(+), 1 deletion(-) create mode 100644 src/osm.c create mode 100644 src/osm.h diff --git a/src/Makefile.am b/src/Makefile.am index 4a24c7b..1015b55 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -18,7 +18,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # -bin_PROGRAMS = mapper +bin_PROGRAMS = mapper osm2db mapper_CFLAGS = $(GTK_CFLAGS) $(OSSO_CFLAGS) $(HILDON_CFLAGS) \ $(GNOME_VFS_CFLAGS) $(GCONF_CFLAGS) $(LIBXML2_CFLAGS) $(GSTREAMER_CFLAGS) \ @@ -52,3 +52,9 @@ mapper_SOURCES = utils.c \ ui-common.c \ config.c \ mapper.c + +osm2db_CFLAGS = $(EXPAT_CFLAGS) $(SQLITE_CFLAGS) $(GLIB_CFLAGS) + +osm2db_LDADD = $(EXPAT_LIBS) $(SQLITE_LIBS) $(GLIB_LIBS) + +osm2db_SOURCES = osm.c diff --git a/src/osm.c b/src/osm.c new file mode 100644 index 0000000..4673848 --- /dev/null +++ b/src/osm.c @@ -0,0 +1,666 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include + +#include "osm.h" + +#define W_ONEWAY (1 << 1) +#define W_ROUNDABOUT (1 << 2) +#define W_LINK (1 << 3) + +gint node_cnt=0; +gint way_cnt=0; +gint seg_cnt=0; + +/* Node type */ +typedef struct _node node; +struct _node { + gint id; + gdouble lat; + gdouble lon; + gint count; + node_type_t type; + gchar *name; + void *data; +}; + +typedef struct _segment segment; +struct _segment { + gint id; + node *from; + node *to; +}; + +/* Segment/Way structure */ +typedef struct _way way; +struct _way { + gint id; + way_type_t type; + gint layer; + gint flags; + gchar *name; + gchar *ref; + gchar *int_ref; + gboolean oneway; + gboolean link; + gboolean area; + GList *nodes; +}; + +typedef enum { + START, + IN_OSM_TAG, + IN_NODE_TAG, + IN_SEGMENT_TAG, + IN_SEG_TAG, + IN_WAY_TAG, + IN_KEY_TAG, + END, + ERROR +} tag_state_t; + +typedef enum { + IS_NONE, + IS_NODE, + IS_WAY, + IS_SEGMENT +} tag_parent_t; + +struct _nodeinfo { + gchar *k, *v; + node_type_t type; +} nodeinfo[] = { + { "amenity", "fuel", NODE_AMENITY_FUEL }, + { "amenity", "pub", NODE_AMENITY_PUB }, + { "amenity", "fast_food", NODE_AMENITY_FOOD }, + { "amenity", "restaurant", NODE_AMENITY_FOOD }, + { "amenity", "parking", NODE_AMENITY_PARKING }, + { "amenity", "hospital", NODE_AMENITY_HOSPITAL }, + { "amenity", "pharmacy", NODE_AMENITY_PHARMACY }, + { "amenity", "post_office", NODE_AMENITY_POST }, + { "amenity", "post_box", NODE_AMENITY_POST_BOX }, + { "amenity", "cinema", NODE_AMENITY_CINEMA }, + { "amenity", "theatre", NODE_AMENITY_THEATRE }, + { "amenity", "atm", NODE_AMENITY_ATM }, + { "amenity", "bank", NODE_AMENITY_BANK }, + { "amenity", "police", NODE_AMENITY_POLICE }, + { "amenity", "supermarket", NODE_AMENITY_SHOP }, + { "amenity", "shop", NODE_AMENITY_SHOP }, + { "amenity", "place_of_worship",NODE_AMENITY_POW }, + + { "place", "city", NODE_PLACE_CITY }, + { "place", "town", NODE_PLACE_CITY }, + { "place", "village", NODE_PLACE_VILLAGE }, + { "place", "hamlet", NODE_PLACE_VILLAGE }, + { "place", "suburb", NODE_PLACE_SUBURB }, + + { NULL, NULL, NODE_PLAIN } +}; + +/* Array to get id number and defaults for ways of different types */ +struct _wayinfo { + gchar *k, *v; + way_type_t type; + gboolean oneway, link, area; +} wayinfo[] = { + { "highway", "motorway",WAY_MOTORWAY, TRUE, FALSE, FALSE }, + { "highway", "motorway_link",WAY_MOTORWAY, TRUE, TRUE, FALSE }, + { "highway", "trunk",WAY_TRUNK, FALSE, FALSE, FALSE }, + { "highway", "trunk_link",WAY_TRUNK, FALSE, TRUE, FALSE }, + { "highway", "primary",WAY_PRIMARY, FALSE, FALSE, FALSE }, + { "highway", "primary_link",WAY_PRIMARY, FALSE, TRUE, FALSE }, + { "highway", "secondary",WAY_SECONDARY, FALSE, FALSE, FALSE }, + { "highway", "secondary_link",WAY_SECONDARY, FALSE, TRUE, FALSE }, + { "highway", "tertiary",WAY_TERTIARY, FALSE, FALSE, FALSE }, + { "highway", "unclasified",WAY_UNCLASSIFIED, FALSE, FALSE, FALSE }, + { "highway", "residential",WAY_RESIDENTIAL, FALSE, FALSE, FALSE }, + { "highway", "track",WAY_TRACK, FALSE, FALSE, FALSE }, + { "highway", "unsurfaced",WAY_TRACK, FALSE, FALSE, FALSE }, + { "highway", "footway",WAY_FOOTWAY, FALSE, FALSE, FALSE }, + { "highway", "cycleway",WAY_CYCLEWAY, FALSE, FALSE, FALSE }, + { "railway", "rail",WAY_RAIL, FALSE, FALSE, FALSE }, + { "natural", "water",WAY_WATER, FALSE, FALSE, TRUE }, + { NULL, NULL, WAY_UNWAYED, FALSE, FALSE, FALSE } +}; + +sqlite3 *db; +tag_parent_t tag_parent=IS_NONE; + +GHashTable *osm_nodes=NULL; +GHashTable *osm_segments=NULL; +GHashTable *osm_ways=NULL; + +node *cnode=NULL; +way *cway=NULL; + +struct sql_stmt { + sqlite3_stmt *insert_poi; + sqlite3_stmt *delete_osm_poi; + sqlite3_stmt *insert_node; + sqlite3_stmt *insert_way_data; + sqlite3_stmt *insert_way_ref; + sqlite3_stmt *insert_way_name; + sqlite3_stmt *insert_way_seg; + sqlite3_stmt *insert_place; +}; +struct sql_stmt sql; + +gint wsegcnt; + +/****************************************************/ +/* Functions */ +/****************************************************/ + +gboolean +db_open(void) +{ +if (SQLITE_OK != (sqlite3_open(OSM_DB_FILE, &db))) { + sqlite3_close(db); + return FALSE; +} +return TRUE; +} + +void +db_prepare(void) +{ +sqlite3_prepare(db, "insert into nodes (id,lat,lon) values (?, ?, ?)", + -1, &sql.insert_node, NULL); + +/* POI nodes */ +sqlite3_prepare(db, "insert into poi (osm_id, lat, lon, label, desc, cat_id, public, priority)" + " values (?, ?, ?, ?, ?, ?, 1, ?)", + -1, &sql.insert_poi, NULL); +sqlite3_prepare(db, "delete from poi where osm_id>0", -1, &sql.delete_osm_poi, NULL); + +/* Ways */ +sqlite3_prepare(db, "insert into way (wid,nodes,type,flags) values (?, ?, ?, ?)", + -1, &sql.insert_way_data, NULL); + +sqlite3_prepare(db, "insert into way_seg (wsid,num,node) values (?, ?, ?)", + -1, &sql.insert_way_seg, NULL); + +sqlite3_prepare(db, "insert into name (nid,name,lang) values (?, ?, ?)", + -1, &sql.insert_way_name, NULL); + +sqlite3_prepare(db, "insert into ref (rid,ref,int_ref) values (?, ?, ?)", + -1, &sql.insert_way_ref, NULL); +} + +/********************************************************************/ + +gboolean +db_insert_node(node *n) +{ +sqlite3_bind_int(sql.insert_node, 1, n->id); +sqlite3_bind_int(sql.insert_node, 2, n->lat); +sqlite3_bind_int(sql.insert_node, 3, n->lon); +sqlite3_step(sql.insert_node); +sqlite3_reset(sql.insert_node); +return TRUE; +} + +gboolean +db_insert_poi(node *n) +{ +sqlite3_bind_int(sql.insert_poi, 1, n->id); +sqlite3_bind_double(sql.insert_poi, 2, n->lat); +sqlite3_bind_double(sql.insert_poi, 3, n->lon); +sqlite3_bind_text(sql.insert_poi, 4, n->name, -1, SQLITE_TRANSIENT); +/* sqlite3_bind_text(sql.insert_poi, 5, n->desc, -1, SQLITE_TRANSIENT); */ +sqlite3_bind_int(sql.insert_poi, 6, n->type); +sqlite3_bind_int(sql.insert_poi, 7, n->type/100); + +sqlite3_step(sql.insert_poi); +sqlite3_reset(sql.insert_poi); +} + +void +db_insert_way_seg(node *n, way *w) +{ +sqlite3_bind_int(sql.insert_way_seg, 1, w->id); +sqlite3_bind_int(sql.insert_way_seg, 2, wsegcnt); +sqlite3_bind_int(sql.insert_way_seg, 3, n->id); + +sqlite3_step(sql.insert_way_seg); +sqlite3_reset(sql.insert_way_seg); + +wsegcnt++; +} + +void +db_insert_way_segments(segment *s, way *w) +{ +db_insert_way_seg(s->from,w); +if (g_list_length(w->nodes)==wsegcnt) + db_insert_way_seg(s->to,w); +} + +gboolean +db_insert_way(way *w) +{ +if (w->type==WAY_UNWAYED) + return TRUE; + +sqlite3_bind_int(sql.insert_way_data, 1, w->id); +sqlite3_bind_int(sql.insert_way_data, 2, g_list_length(w->nodes)); +sqlite3_bind_int(sql.insert_way_data, 3, w->type); +sqlite3_bind_int(sql.insert_way_data, 4, w->flags); + +sqlite3_step(sql.insert_way_data); +sqlite3_reset(sql.insert_way_data); + +wsegcnt=0; +if (w->area==TRUE) { + g_print("Areas not handled yet\n"); +} else { + g_list_foreach(w->nodes, db_insert_way_segments, w); +} + +return TRUE; +} + +/********************************************************************/ + +void dump_array(const gchar **p) +{ +char **d; + +d=p; +while (*d!=NULL) { + printf("[%s]", *d); + d++; +} +} + +gchar *get_attr_key_value(gchar **p, const gchar *key) +{ +char **d; + +d=p; +while (*d!=NULL) { + if (strncmp(*d, key, strlen(key))==0) { + d++; + return *d; + } + d++; + d++; +} +return NULL; +} + +tag_state_t check_tag(const gchar *tag) +{ +if (strcmp(tag,"node")==0) return IN_NODE_TAG; +else if (strcmp(tag,"segment")==0) return IN_SEGMENT_TAG; +else if (strcmp(tag,"seg")==0) return IN_SEG_TAG; +else if (strcmp(tag,"way")==0) return IN_WAY_TAG; +else if (strcmp(tag,"tag")==0) return IN_KEY_TAG; +else if (strcmp(tag,"osm")==0) return IN_OSM_TAG; +else return ERROR; +} + +/********************************************************************/ + +static gint +osm_dlat2ilat(double lat) +{ +return lat > 85.051128779 ? INT_MAX : lat < -85.051128779 ? INT_MIN : + lrint(log(tan(M_PI_4l+lat*M_PIl/360))/M_PIl*INT_MAX); +} + +static gint +osm_dlon2ilon(double lon) +{ +return lrint(lon/180*INT_MAX); +} + +/********************************************************************/ + +void +print_node (node *n) +{ +if (n->type) + g_printf("ID: %d [%f:%f][%s](%d) %d\n", + n->id, n->lat, n->lon, n->name, n->type, n->count); +} + +node * +osm_new_node(gint id, double lat, double lon) +{ +node *n; + +n=g_slice_new(node); +n->id=id; +n->lat=lat; +n->lon=lon; +n->type=NODE_PLAIN; +n->count=0; +n->data=(void *)NULL; +n->name=NULL; +g_hash_table_insert(osm_nodes, GINT_TO_POINTER(id), n); +return n; +} + +void +print_segment(segment *s) +{ +g_printf("Segment %d:\n", s->id); +if (s->from) + print_node(s->from); +if (s->to) + print_node(s->to); +} + +static void +_print_segment_helper(segment *s, void *data) +{ +print_segment(s); +} + +void +print_way(way *w) +{ +g_printf("Way #%d(%d/%d): %s %s %s [%s:%s] %d\n", + w->id, g_list_length(w->nodes), w->type, + w->name ? w->name : "", + w->ref ? w->ref : "", + w->int_ref ? w->int_ref : "", + w->oneway==TRUE ? "-" : "=", w->link==TRUE ? "L" : " ", + w->flags); +#ifdef VERBOSE +g_list_foreach(w->nodes, _print_segment_helper, NULL); +#endif +} + +segment * +osm_get_segment_for_way(gint sid) +{ +segment *s; +s=g_hash_table_lookup(osm_segments, GINT_TO_POINTER(sid)); +if (s) { + s->from->count++; + s->to->count++; +} +return s; +} + +node * +osm_find_node(gint nid) +{ +return g_hash_table_lookup(osm_nodes, GINT_TO_POINTER(nid)); +} + +void +osm_new_segment(gint id, gint from, gint to) +{ +segment *s; + +s=g_slice_new(segment); +s->id=id; +s->from=osm_find_node(from); +s->to=osm_find_node(to); +g_hash_table_insert(osm_segments, GINT_TO_POINTER(id), s); +} + +way * +osm_new_way(gint id) +{ +way *w; + +w=g_slice_new(way); +w->id=id; +w->oneway=FALSE; +w->nodes=NULL; +w->type=WAY_UNWAYED; +w->name=NULL; +w->ref=NULL; +w->int_ref=NULL; +w->layer=0; +w->flags=0; + +g_hash_table_insert(osm_ways, GINT_TO_POINTER(id), w); +return w; +} + +void +osm_way_new_seg(way *w, gint id) +{ +segment *s; + +s=osm_get_segment_for_way(id); +w->nodes=g_list_append(w->nodes, s); +} + +/***********************************************************************/ + +void +osm_node_save_poi(gint key, node *value, gpointer user_data) +{ +if (value->type>NODE_POI_START && value->typename=g_strdup(v); + + if (cnode->type==NODE_PLAIN) { + for (i=0; nodeinfo[i].k; i++) { + if (strcasecmp (nodeinfo[i].k, k)==0 && strcasecmp (v, nodeinfo[i].v)==0) { + cnode->type=nodeinfo[i].type; + } + } + } + } + break; + case IS_WAY: + { + gint i; + + k=get_attr_key_value(atts, "k"); + v=get_attr_key_value(atts, "v"); + if (strcasecmp(k, "name")==0) + cway->name=g_strdup(v); + if (strcasecmp(k, "ref")==0) + cway->ref=g_strdup(v); + if (strcasecmp(k, "int_ref")==0) + cway->int_ref=g_strdup(v); + if (cway->type==WAY_UNWAYED) { + for (i=0; wayinfo[i].k; i++) { + if (strcasecmp (wayinfo[i].k, k)==0 && strcasecmp (v, wayinfo[i].v)==0) { + cway->oneway=wayinfo[i].oneway; + cway->link=wayinfo[i].link; + cway->area=wayinfo[i].area; + cway->type=wayinfo[i].type; + } + } + } + + if (strcasecmp(k, "oneway")==0) + cway->oneway=TRUE; + if (strcasecmp(k, "layer")==0) + cway->layer=atoi(v); + } + break; + case IS_SEGMENT: + /* Just ignore tagged segments */ + break; + } + break; + default: + tag_parent=IS_NONE; + g_printf("Unknown tag: %s\n", name); + break; +} +} + +static void +_osm_tag_end(void *userData, const char *name) +{ +tag_state_t t; +t=check_tag(name); +switch (t) { + case IN_NODE_TAG: + print_node(cnode); + cnode=NULL; + break; + case IN_WAY_TAG: + if (cway->oneway==TRUE) + cway->flags|=W_ONEWAY; + if (cway->link==TRUE) + cway->flags|=W_LINK; + print_way(cway); + cway=NULL; + break; + default:; +} +} + +gint osm_planet_parse_file(gchar *pfile) +{ +int f, r, res; +gchar buffer[FILE_BUFFER]; +XML_Parser xp; + +f=open(pfile, O_RDONLY); +if (f==-1) return 1; +res=0; + +xp=XML_ParserCreate(NULL); +XML_SetElementHandler(xp, _osm_tag_start, _osm_tag_end); + +do { + r=read(f, buffer, FILE_BUFFER); + if (XML_Parse(xp, buffer, r, r>0 ? 0:1) == XML_STATUS_ERROR) { + g_printf("Parse error at line %d:\n%s\n", + XML_GetCurrentLineNumber(xp), + XML_ErrorString(XML_GetErrorCode(xp))); + res=2; + goto end; + } +} while (r>0); + +end:; + +close(f); +return res; +} + +inline gint +print_fail(const gchar *msg, gint ret) +{ +g_printf("ERROR: %s\n", msg); +return ret; +} + +int main (int argc, char **argv) +{ +if (argc!=2) { + return print_fail("No planet XML file given", 1); +} else { + g_printf("Using file: %s\n", argv[1]); +} + +if (db_open()!=TRUE) + return print_fail("Database open failed", 2); +db_prepare(); + +osm_nodes=g_hash_table_new(g_direct_hash, g_direct_equal); +osm_segments=g_hash_table_new(g_direct_hash, g_direct_equal); +osm_ways=g_hash_table_new(g_direct_hash, g_direct_equal); + +osm_planet_parse_file(argv[1]); + +g_printf("Total nodes %d segments %d and ways %d\n", node_cnt, seg_cnt, way_cnt); + +osm_planet_save_to_db(); + +sqlite3_close(db); +} diff --git a/src/osm.h b/src/osm.h new file mode 100644 index 0000000..5fcf055 --- /dev/null +++ b/src/osm.h @@ -0,0 +1,58 @@ +/* #define VERBOSE */ +#define FILE_BUFFER 65535 +#define OSM_DB_FILE "osm-planet.db" + +/* POI category ID's are special, the # is divied + * by 100 and the result is the importance/priority of the poi. + * This can be used by the client program to discard + * unimportant poi at different zoom levels. + */ +typedef enum { + NODE_PLAIN=0, + /* Amenities */ + NODE_POI_START=1, + NODE_AMENITY_PARKING=100, + NODE_AMENITY_FUEL=101, + NODE_AMENITY_HOSPITAL=120, + NODE_AMENITY_PHARMACY=121, + NODE_AMENITY_POLICE=130, + NODE_AMENITY_HOTEL=200, + NODE_AMENITY_HOSTEL=201, + NODE_AMENITY_ATM=205, + NODE_AMENITY_BANK=206, + NODE_AMENITY_POST=210, + NODE_AMENITY_POST_BOX=211, + NODE_AMENITY_FOOD=301, + NODE_AMENITY_PUB=302, + NODE_AMENITY_CINEMA=303, + NODE_AMENITY_THEATRE=304, + NODE_AMENITY_SHOP=305, + NODE_AMENITY_POW=310, + /* */ + NODE_POI_END=999, + /* Places */ + NODE_PLACE_START=1000, + NODE_PLACE_SUBURB=1001, + NODE_PLACE_VILLAGE=1050, + NODE_PLACE_CITY=1100, + NODE_PLACE_END=1500, + /* Other */ + NODE_TYPE_MAX=9000 +} node_type_t; + +typedef enum { + WAY_UNWAYED=0, + WAY_MOTORWAY, + WAY_TRUNK, + WAY_PRIMARY, + WAY_SECONDARY, + WAY_TERTIARY, + WAY_UNCLASSIFIED, + WAY_RESIDENTIAL, + WAY_TRACK, + WAY_FOOTWAY, + WAY_CYCLEWAY, + WAY_RAIL, + WAY_WATER, + WAY_OTHER +} way_type_t; -- 2.39.5