From 606a7adf44e2ede97d0925c0a669e10f9d59f363 Mon Sep 17 00:00:00 2001 From: Kaj-Michael Lang Date: Tue, 30 Oct 2007 22:07:08 +0200 Subject: [PATCH] A lot of changes to OSM importer: - Use common db.c for database connection, close, exec - Handle secondary languages for ways - Mark functions static --- src/osm.c | 223 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 130 insertions(+), 93 deletions(-) diff --git a/src/osm.c b/src/osm.c index 9cebb34..57356ce 100644 --- a/src/osm.c +++ b/src/osm.c @@ -51,6 +51,7 @@ #include "osm.h" #include "latlon.h" +#include "db.h" #if 0 #define VERBOSE @@ -99,6 +100,7 @@ struct _node { typedef struct _way_data way_data; struct _way_data { gchar *name; + GHashTable *names; gchar *ref; gchar *int_ref; guint32 isin; @@ -126,6 +128,7 @@ typedef enum { IN_WAY_TAG, IN_KEY_TAG, IN_BOUND_TAG, + IN_RELATION_TAG, END, ERROR } tag_state_t; @@ -134,7 +137,8 @@ typedef enum { typedef enum { IS_NONE, IS_NODE, - IS_WAY + IS_WAY, + IS_RELATION } tag_parent_t; /* Node types table */ @@ -371,7 +375,7 @@ struct sql_stmt { sqlite3_stmt *insert_place; sqlite3_stmt *delete_place; }; -struct sql_stmt sql; +static struct sql_stmt sql; struct map_bbox { gdouble lat_min; @@ -379,13 +383,11 @@ struct map_bbox { gdouble lat_max; gdouble lon_max; }; -struct map_bbox bbox; -gboolean use_bbox; +static struct map_bbox bbox; +static gboolean use_bbox; void osm_free_way_data(way *w); void print_way(way *w); -gboolean db_open(void); -gboolean db_close(void); void db_prepare(void); gboolean db_insert_node(node *n); @@ -395,24 +397,8 @@ guint32 osm_find_way_place(way *w); /* Functions */ /****************************************************/ -gboolean -db_open(void) -{ -if (SQLITE_OK != (sqlite3_open(OSM_DB_FILE, &db))) { - sqlite3_close(db); - return FALSE; -} -/* Turn of syncing to speed up data import */ -sqlite3_exec(db, "PRAGMA synchronous = OFF;", NULL, NULL, NULL); - -/* Use more cache memory */ -sqlite3_exec(db, "PRAGMA cache_size = 8000;", NULL, NULL, NULL); - -return TRUE; -} - -gboolean -db_close(void) +static void +db_finalize(void) { sqlite3_finalize(sql.insert_poi); sqlite3_finalize(sql.delete_osm_poi); @@ -436,8 +422,6 @@ sqlite3_finalize(sql.insert_way_n2n); sqlite3_finalize(sql.delete_way_names_nls); sqlite3_finalize(sql.insert_way_names_nls); - -return sqlite3_close(db)==SQLITE_OK ? TRUE : FALSE; } void @@ -542,8 +526,6 @@ return TRUE; static gboolean db_insert_place(node *n) { -gint r; - if (!n->data) return FALSE; if (!n->data->name) @@ -552,18 +534,13 @@ sqlite3_bind_int(sql.insert_place, 1, n->id); sqlite3_bind_int(sql.insert_place, 2, n->type); sqlite3_bind_text(sql.insert_place, 3, n->data->name, -1, SQLITE_TRANSIENT); sqlite3_bind_int(sql.insert_place, 4, n->data->isin); -r=sqlite3_step(sql.insert_place); -sqlite3_reset(sql.insert_place); -sqlite3_clear_bindings(sql.insert_place); -return (r==SQLITE_OK) ? TRUE : FALSE; +return db_exec(sql.insert_place); } static gboolean db_insert_poi(node *n) { -gint r; - sqlite3_bind_int(sql.insert_poi, 1, n->id); sqlite3_bind_double(sql.insert_poi, 2, n->lat); sqlite3_bind_double(sql.insert_poi, 3, n->lon); @@ -582,11 +559,7 @@ if (n->data->url) sqlite3_bind_text(sql.insert_poi, 9, n->data->url, -1, SQLITE_TRANSIENT); #endif -r=sqlite3_step(sql.insert_poi); -sqlite3_reset(sql.insert_poi); -sqlite3_clear_bindings(sql.insert_poi); - -return (r==SQLITE_OK) ? TRUE : FALSE; +return db_exec(sql.insert_poi); } /** @@ -597,10 +570,7 @@ db_update_node_links(node *n) { sqlite3_bind_int(sql.update_node, 1, n->id); -sqlite3_step(sql.update_node); -sqlite3_reset(sql.update_node); -sqlite3_clear_bindings(sql.update_node); -return TRUE; +return db_exec(sql.update_node); } /** @@ -632,9 +602,7 @@ sqlite3_bind_int(sql.insert_way_n2n, 3, nt->id); g_printf("%d [%d - %d]\n", w->id, nf->id, nt->id); #endif -sqlite3_step(sql.insert_way_n2n); -sqlite3_reset(sql.insert_way_n2n); -sqlite3_clear_bindings(sql.insert_way_n2n); +db_exec(sql.insert_way_n2n); db_update_node_links(nf); db_update_node_links(nt); return TRUE; @@ -660,13 +628,11 @@ if (w->data->ref) if (w->data->int_ref) sqlite3_bind_text(sql.insert_way_ref, 3, w->data->int_ref, -1, SQLITE_TRANSIENT); -sqlite3_step(sql.insert_way_ref); -sqlite3_reset(sql.insert_way_ref); -sqlite3_clear_bindings(sql.insert_way_ref); +db_exec(sql.insert_way_ref); } /** - * Insert way name (nls names are not handled yet, ideas ?) + * Insert way name */ static void db_insert_way_name(way *w) @@ -681,9 +647,40 @@ way_names++; sqlite3_bind_int(sql.insert_way_name, 1, w->id); sqlite3_bind_text(sql.insert_way_name, 2, w->data->name, -1, SQLITE_TRANSIENT); -sqlite3_step(sql.insert_way_name); -sqlite3_reset(sql.insert_way_name); -sqlite3_clear_bindings(sql.insert_way_name); +db_exec(sql.insert_way_name); +} + +static void +db_delete_way_names_nls(way *w) +{ +sqlite3_bind_int(sql.delete_way_names_nls, 1, w->id); + +sqlite3_step(sql.delete_way_names_nls); +sqlite3_reset(sql.delete_way_names_nls); +sqlite3_clear_bindings(sql.delete_way_names_nls); +} + +static void +db_insert_way_names_nls_cb(gpointer key, gpointer value, gpointer user_data) +{ +way *w=(way *)user_data; + +sqlite3_bind_int(sql.insert_way_names_nls, 1, w->id); +sqlite3_bind_text(sql.insert_way_names_nls, 2, (gchar *)key, -1, SQLITE_TRANSIENT); +sqlite3_bind_text(sql.insert_way_names_nls, 3, (gchar *)value, -1, SQLITE_TRANSIENT); + +db_exec(sql.insert_way_names_nls); +} + +static void +db_insert_way_names_nls(way *w) +{ +if (!w->data) + return; +if (!w->data->names) + return; + +g_hash_table_foreach(w->data->names, db_insert_way_names_nls_cb, w); } /** @@ -729,12 +726,11 @@ if (w->data) { sqlite3_bind_int(sql.insert_way_data, 6, w->data->isin); } -sqlite3_step(sql.insert_way_data); -sqlite3_reset(sql.insert_way_data); -sqlite3_clear_bindings(sql.insert_way_data); +db_exec(sql.insert_way_data); db_insert_way_ref(w); db_insert_way_name(w); +db_insert_way_names_nls(w); osm_free_way_data(w); return TRUE; @@ -759,7 +755,8 @@ while (*d!=NULL) { return NULL; } -static tag_state_t check_tag(const gchar *tag) +static tag_state_t +check_tag(const gchar *tag) { if (strcmp(tag,"node")==0) return IN_NODE_TAG; else if (strcmp(tag,"nd")==0) return IN_WNODE_TAG; @@ -770,6 +767,33 @@ else if (strcmp(tag,"bound")==0) return IN_BOUND_TAG; else return ERROR; } +static void +find_nls_names(gpointer key, gpointer value, gpointer user_data) +{ +gchar *k, *v; +gchar *tmp; +GHashTable *nls; + +k=(gchar *)key; +v=(gchar *)value; +nls=(GHashTable *)user_data; + +/* Check if it is a name key, return if not. */ +if (g_str_has_prefix(k, "name:")==FALSE) + return; + +tmp=g_strrstr(k, ":"); +if (!tmp) + return; +tmp++; /* skip : */ +if (*tmp==0) + return; +g_hash_table_insert(nls, g_strdup(tmp), g_strdup(v)); +#ifdef VERBOSE +g_printf("NLS(%s): [%s]\n", tmp, v); +#endif +} + /********************************************************************/ static void @@ -810,7 +834,7 @@ if (use_bbox==FALSE) return (nlat > bbox.lat_min && nlat < bbox.lat_max && nlon > bbox.lon_min && nlon < bbox.lon_max) ? TRUE : FALSE; } -void +static void osm_new_node_data(node *n) { if (n==NULL) return; @@ -821,7 +845,7 @@ n->type=NODE_PLAIN; noded_cnt++; } -void +static void osm_free_node_data(node *n) { g_assert(n); @@ -833,7 +857,7 @@ n->data=NULL; noded_cnt--; } -node * +static node * osm_new_node(gint id, gdouble lat, gdouble lon) { node *n=NULL; @@ -860,7 +884,7 @@ if (!n) return n; } -void +static void osm_new_way_data(way *w) { if (w==NULL) return; @@ -868,13 +892,14 @@ if (w->data!=NULL) return; w->data=g_slice_new(way_data); w->data->name=NULL; +w->data->names=NULL; w->data->ref=NULL; w->data->int_ref=NULL; w->data->layer=0; w->data->speed=0; } -void +static void osm_free_way_data(way *w) { if (!w->data) @@ -889,7 +914,7 @@ g_slice_free(way_data, w->data); w->data=NULL; } -way * +static way * osm_new_way(gint id) { way *w; @@ -906,13 +931,13 @@ w->flags=0; return w; } -void +static void osm_way_add_to_list(way *w) { osm_ways=g_slist_prepend(osm_ways, w); } -void +static void osm_way_new_node(way *w, gint nid) { node *n; @@ -926,7 +951,7 @@ w->ncnt++; * Search the place hash table for the location of the node. * */ -guint32 +static guint32 osm_find_node_place(node *n) { node *t; @@ -1012,7 +1037,7 @@ return 0; /***********************************************************************/ -void +static void osm_node_save_node(gint key, gpointer value, gpointer user_data) { node *n=(node *)value; @@ -1027,7 +1052,7 @@ if (dbnode_cnt % 26214==0) * Check node type and insert as POI or Place * Discard extra data after insert. */ -gboolean +static gboolean osm_node_save_poi(node *n, gpointer user_data) { if (!n) { @@ -1060,7 +1085,7 @@ osm_free_node_data(n); return TRUE; } -void +static void osm_planet_poi_clear_nodes(void) { g_print("Removing old OSM POIs...\n"); @@ -1070,7 +1095,7 @@ sqlite3_step(sql.delete_place); sqlite3_exec(db, "commit;", NULL, NULL, NULL); } -void +static void osm_planet_poi_save_nodes(void) { g_print("Storing new POIs...\n"); @@ -1082,14 +1107,14 @@ g_slist_free(osm_poi); /*********************************************************************/ -void +static void osm_planet_clear_nodes(void) { g_print("Clearing old nodes\n"); sqlite3_step(sql.delete_nodes); } -void +static void osm_planet_save_nodes(void) { g_print("Storing nodes...\n"); @@ -1101,7 +1126,7 @@ sqlite3_exec(db, "commit;", NULL, NULL, NULL); /*********************************************************************/ -void +static void osm_way_save(way *value, gpointer user_data) { dbway_cnt++; @@ -1112,7 +1137,7 @@ if (dbway_cnt % 8192==0 && dbway_cnt>0) { } } -void +static void osm_planet_clear_ways(void) { g_print("Clearing old data:\n"); @@ -1122,7 +1147,7 @@ sqlite3_step(sql.delete_way_ref); sqlite3_step(sql.delete_way_n2n); } -void +static void osm_planet_save_ways(void) { g_print("Inserting new ways:\n"); @@ -1133,10 +1158,10 @@ sqlite3_exec(db, "commit;", NULL, NULL, NULL); /*********************************************************************/ -void -osm_planet_save_to_db(void) +static void +osm_planet_save_all_nodes(void) { -g_printf("Saving planet to database:\n"); +g_printf("Saving planet nodes to database:\n"); osm_planet_poi_clear_nodes(); osm_planet_poi_save_nodes(); @@ -1145,11 +1170,15 @@ if (!is_update) { osm_planet_clear_nodes(); osm_planet_clear_ways(); } - osm_planet_save_nodes(); -osm_planet_save_ways(); +} + +static void +osm_planet_save_all_ways(void) +{ +g_printf("Saving planet way to database:\n"); -g_printf("Data saved.\n"); +osm_planet_save_ways(); } /***********************************************************************/ @@ -1232,17 +1261,16 @@ switch (t) { break; case IS_WAY: { - gint i; - if (cway==NULL) { g_printerr("In way tags but way is NULL!\n"); return; } - g_hash_table_insert(osm_way_tags, g_strdup(k), g_strdup(v)); osm_new_way_data(cway); } break; + case IS_RELATION: + break; } break; case IN_BOUND_TAG: @@ -1357,8 +1385,16 @@ switch (t) { } v=g_hash_table_lookup(osm_way_tags, "name"); - if (v) + if (v) { cway->data->name=g_strdup(v); + /* Try to find other language names */ + cway->data->names=g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free); + g_hash_table_foreach(osm_way_tags, find_nls_names, cway->data->names); + if (g_hash_table_size(cway->data->names)==0) { + g_hash_table_destroy(cway->data->names); + cway->data->names=NULL; + } + } v=g_hash_table_lookup(osm_way_tags, "ref"); if (v) @@ -1395,6 +1431,8 @@ switch (t) { cway->flags|=W_ROUNDABOUT; cway->flags|=W_ONEWAY; } + + /* XXX: Should check keys */ v=g_hash_table_lookup(osm_way_tags, "access"); if (v && (strcasecmp(v, "private")==0)) { cway->flags|=W_NOACCESS; @@ -1565,28 +1603,27 @@ if (argc==6) { g_print("Note: missing nodes can't be reported.\n"); } else use_bbox=FALSE; -if (db_open()!=TRUE) +if (db_connect(&db, OSM_DB_FILE)!=TRUE) return print_fail("Database open failed", 2); db_prepare(); print_memory_usage(); osm_planet_parser_init(); -osm_planet_parse_file(argv[1]); +if (osm_planet_parse_file(argv[1])==FALSE) + return 1; -g_printf("Total nodes %d, POIs: %d and Ways %d.\n", - node_cnt, noded_cnt, way_cnt); +g_printf("Total nodes %d, POIs: %d and Ways %d.\n", node_cnt, noded_cnt, way_cnt); g_printf("Cities/Towns: %d\n", g_hash_table_size(osm_place_city)); g_printf("Villages/Hamlets: %d\n", g_hash_table_size(osm_place_village)); g_printf("Suburbs: %d\n", g_hash_table_size(osm_place_suburb)); -osm_planet_save_to_db(); +osm_planet_save_all_nodes(); +osm_planet_save_all_ways(); osm_planet_parser_deinit(); +db_close(&db); g_printf("Named ways: %d, Numbered ways: %d\n", way_names, way_refs); -db_close(); - -sync(); return 0; } -- 2.39.5