From 4cd8829c6885ec9179a0fd0fffe5a0a0ca6e0494 Mon Sep 17 00:00:00 2001
From: Kaj-Michael Lang <milang@angel.tal.org>
Date: Tue, 17 Jul 2007 15:13:44 +0300
Subject: [PATCH] Be a bit more verbose. Use sqlite pragma to not sync so much.
 Put POIs in a linked list so we don't need to go trough the whole node hash
 table when adding POI nodes.

---
 src/osm.c | 52 +++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 41 insertions(+), 11 deletions(-)

diff --git a/src/osm.c b/src/osm.c
index 26e28ae..c0252e2 100644
--- a/src/osm.c
+++ b/src/osm.c
@@ -24,6 +24,10 @@ static gint noded_cnt=0;
 static gint way_cnt=0;
 static gint seg_cnt=0;
 
+static gint dbnode_cnt=0;
+static gint dbnoded_cnt=0;
+static gint dbway_cnt=0;
+
 typedef struct _node_data node_data;
 struct _node_data {
 	gchar *name;
@@ -62,7 +66,7 @@ struct _way {
 	gint32 id;
 	way_type_t type;
 	guint16 ncnt;
-	guint flags;
+	guint8 flags;
 	way_data *data;
 	GSList *nodes;
 };
@@ -157,6 +161,7 @@ tag_parent_t tag_parent=IS_NONE;
 static GHashTable *osm_nodes=NULL;
 static GHashTable *osm_segments=NULL;
 static GHashTable *osm_ways=NULL;
+static GSList *osm_poi=NULL;
 
 static node *cnode=NULL;
 static way *cway=NULL;
@@ -245,6 +250,10 @@ sqlite3_prepare_v2(db, "insert into way_ref (rid,ref,int_ref) values (?, ?, ?)",
 		    -1, &sql.insert_way_ref, NULL);
 sqlite3_prepare_v2(db, "delete from way_ref", -1, 
 	&sql.delete_way_ref, NULL);
+
+/* Turn of syncing to speed up data import */
+sqlite3_exec(db, "PRAGMA synchronous = OFF;", NULL, NULL, NULL);
+
 }
 
 /********************************************************************/
@@ -387,6 +396,9 @@ if (w->ncnt==wsegcnt)
 gboolean
 db_insert_way(way *w)
 {
+if (!w)
+	return FALSE;
+
 if (w->type==WAY_UNWAYED)
 	return TRUE;
 
@@ -404,7 +416,6 @@ db_insert_way_names(w);
 
 wsegcnt=0;
 g_slist_foreach(w->nodes, db_insert_way_segments, w);
-print_way(w);
 
 osm_free_way_data(w);
 return TRUE;
@@ -615,24 +626,32 @@ w->ncnt++;
 void
 osm_node_save_node(gint key, node *value, gpointer user_data)
 {
+dbnode_cnt++;
 db_insert_node(value);
+if (dbnode_cnt % 262140==0)
+		g_printf("Nodes: %d\n", dbnode_cnt);
 }
 
 void
-osm_node_save_poi(gint key, node *value, gpointer user_data)
+osm_node_save_poi(node *value, gpointer user_data)
 {
-if (value->data==NULL)
+if (!value) {
+	g_printf("ERROR: null poi\n");
+	return;
+}
+
+if (!value->data)
 	return;
 
 if (value->data->type>NODE_POI_START && value->data->type<NODE_POI_END) {
 	db_insert_poi(value);
-	/* Free the extra info, we don't need it anymore */
-	osm_free_node_data(value);
 } else if (value->data->type>NODE_PLACE_START && value->data->type<NODE_PLACE_END) {
 	db_insert_place(value);
-	osm_free_node_data(value);
 }
 
+/* Free the extra info, we don't need it anymore */
+osm_free_node_data(value);
+
 }
 
 void
@@ -646,7 +665,9 @@ void
 osm_planet_poi_save_nodes(void)
 {
 g_print("Storing new POIs...\n");
-g_hash_table_foreach(osm_nodes, osm_node_save_poi, NULL);
+sqlite3_exec(db, "begin;", NULL, NULL, NULL);
+g_slist_foreach(osm_poi, osm_node_save_poi, NULL);
+sqlite3_exec(db, "commit;", NULL, NULL, NULL);
 }
 
 /*********************************************************************/
@@ -667,7 +688,10 @@ sqlite3_exec(db, "commit;", NULL, NULL, NULL);
 void
 osm_node_save_way(gint key, way *value, gpointer user_data)
 {
+dbway_cnt++;
 db_insert_way(value);
+if (dbway_cnt % 26214==0)
+		g_printf("Ways: %d\n", dbway_cnt);
 }
 
 void
@@ -689,10 +713,12 @@ sqlite3_exec(db, "commit;", NULL, NULL, NULL);
 void
 osm_planet_save_to_db(void)
 {
+g_printf("Saving planet to database:\n");
 osm_planet_poi_clear_nodes();
 osm_planet_poi_save_nodes();
 osm_planet_save_nodes();
 osm_planet_save_ways();
+g_printf("Data saved.\n");
 }
 
 /***********************************************************************/
@@ -823,15 +849,18 @@ tag_state_t t;
 t=check_tag(name);
 switch (t) {
 	case IN_NODE_TAG:
-		if (node_cnt % 100000==0) {
+		if (node_cnt % 262140==0) {
 			g_printf("Nodes: %d/%d\n", node_cnt, noded_cnt);
 		}
 		if (cnode->data && cnode->data->type==NODE_PLAIN)
 			osm_free_node_data(cnode);
+		else if (cnode->data) {
+			osm_poi=g_slist_append(osm_poi, cnode);
+		}
 		cnode=NULL;
 	break;
 	case IN_WAY_TAG:
-		if (way_cnt % 10000==0) {
+		if (way_cnt % 32767==0) {
 			g_printf("Ways: %d\n", way_cnt);
 		}
 		if (cway->data && cway->data->name==NULL && cway->data->ref==NULL && 
@@ -910,7 +939,7 @@ osm_ways=g_hash_table_new(g_direct_hash, g_direct_equal);
 
 osm_planet_parse_file(argv[1]);
 
-g_printf("Total nodes %d/%d segments %d and ways %d\n", 
+g_printf("Planet loaded.\nTotal nodes %d, POIs: %d, Segments %d and Ways %d.\n",
 	node_cnt, noded_cnt, seg_cnt, way_cnt);
 
 osm_planet_save_to_db();
@@ -920,4 +949,5 @@ g_hash_table_unref(osm_segments);
 g_hash_table_unref(osm_ways);
 
 sqlite3_close(db);
+sync();
 }
-- 
2.39.5