]> err.no Git - mapper/blob - src/map-download.c
MapWidgetIntegration:
[mapper] / src / map-download.c
1 /*
2  * This file is part of mapper
3  *
4  * Copyright (C) 2007 Kaj-Michael Lang
5  * Copyright (C) 2006-2007 John Costigan.
6  *
7  * POI and GPS-Info code originally written by Cezary Jackiewicz.
8  *
9  * Default map data provided by http://www.openstreetmap.org/
10  *
11  * This program is free software; you can redistribute it and/or modify
12  * it under the terms of the GNU General Public License as published by
13  * the Free Software Foundation; either version 2 of the License, or
14  * (at your option) any later version.
15  *
16  * This program is distributed in the hope that it will be useful,
17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19  * GNU General Public License for more details.
20  *
21  * You should have received a copy of the GNU General Public License along
22  * with this program; if not, write to the Free Software Foundation, Inc.,
23  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24  */
25
26 #include "config.h"
27
28 #include <unistd.h>
29 #include <stdlib.h>
30 #include <string.h>
31 #include <strings.h>
32 #include <stddef.h>
33 #include <locale.h>
34 #include <math.h>
35 #include <errno.h>
36 #include <sys/wait.h>
37 #include <glib/gstdio.h>
38 #include <fcntl.h>
39 #include <curl/multi.h>
40 #include <libintl.h>
41 #include <locale.h>
42
43 #include "hildon-mapper.h"
44
45 #include "utils.h"
46 #include "osm.h"
47 #include "db.h"
48 #include "osm-db.h"
49 #include "poi.h"
50 #include "route.h"
51 #include "gps.h"
52 #include "mapper-types.h"
53 #include "ui-common.h"
54 #include "settings.h"
55 #include "latlon.h"
56 #include "gpx.h"
57 #include "map-download.h"
58 #include "iap.h"
59 #include "map-repo.h"
60 #include "gtkmap.h"
61
62 static guint _num_downloads=0;
63 static guint _curr_download=0;
64
65 static GQueue *curl_easy_queue=NULL;
66 static GTree *pui_tree=NULL;
67 static GTree *downloading_tree=NULL;
68 static GHashTable *pui_by_easy=NULL;
69
70 static gchar *map_construct_url(guint tilex, guint tiley, guint zoom);
71
72 static gboolean 
73 get_next_pui(gpointer key, gpointer value, ProgressUpdateInfo ** data)
74 {
75 *data = key;
76 return TRUE;
77 }
78
79 static gint
80 download_comparefunc(const ProgressUpdateInfo * a, const ProgressUpdateInfo * b, gpointer user_data)
81 {
82 gint diff = (a->priority - b->priority);
83 if (diff)
84         return diff;
85 diff = (a->tilex - b->tilex);
86 if (diff)
87         return diff;
88 diff = (a->tiley - b->tiley);
89 if (diff)
90         return diff;
91 diff = (a->zoom - b->zoom);
92 if (diff)
93         return diff;
94 diff = (a->repo - b->repo);
95 if (diff)
96         return diff;
97 /* Otherwise, deletes are "greatest" (least priority). */
98 if (!a->retries)
99         return (b->retries ? -1 : 0);
100 else if (!b->retries)
101         return (a->retries ? 1 : 0);
102 /* Do updates after non-updates (because they'll both be done anyway). */
103 return (a->retries - b->retries);
104 }
105
106 /**
107  * Free a ProgressUpdateInfo data structure that was allocated during the
108  * auto-map-download process.
109  */
110 static void 
111 progress_update_info_free(ProgressUpdateInfo * pui)
112 {
113 g_free(pui->src_str);
114 g_free(pui->dest_str);
115 g_slice_free(ProgressUpdateInfo, pui);
116 }
117
118 gboolean 
119 map_download_timeout()
120 {
121 static guint destroy_counter = 50;
122 gint num_transfers = 0, num_msgs = 0;
123 gint deletes_left = 50; /* only do 50 deletes at a time. */
124 CURLMsg *msg;
125
126 if (_curl_multi && CURLM_CALL_MULTI_PERFORM == curl_multi_perform(_curl_multi, &num_transfers))
127         return TRUE;    /* Give UI a chance first. */
128
129 while (_curl_multi && (msg = curl_multi_info_read(_curl_multi, &num_msgs))) {
130         if (msg->msg == CURLMSG_DONE) {
131                 if (msg->easy_handle == _autoroute_data.curl_easy) {
132                         /* This is the autoroute download. */
133                         /* Now, parse the autoroute and update the display. */
134                         if (_autoroute_data.enabled && gpx_parse(&_route, _autoroute_data.rdl_data.bytes, _autoroute_data.rdl_data.bytes_read, 0)) {
135                                 /* Find the nearest route point, if we're connected. */
136                                 route_find_nearest_point(_route);
137                                 map_force_redraw();
138                         }
139                         route_cancel_autoroute(_route, TRUE);   /* We're done. Clean up. */
140                 } else {
141                         ProgressUpdateInfo *pui = g_hash_table_lookup(pui_by_easy, msg->easy_handle);
142                         g_queue_push_head(curl_easy_queue, msg->easy_handle);
143                         g_hash_table_remove(pui_by_easy, msg->easy_handle);
144                         fclose(pui->file);
145                         if (msg->data.result != CURLE_OK)
146                                 g_unlink(pui->dest_str);        /* Delete so we try again. */
147                         curl_multi_remove_handle(_curl_multi, msg->easy_handle);
148                         g_idle_add_full(G_PRIORITY_HIGH_IDLE, (GSourceFunc)map_download_idle_refresh, pui, NULL);
149                 }
150         }
151 }
152
153 /* Up to 1 transfer per tile. */
154 while (num_transfers < (4*4) && g_tree_nnodes(pui_tree)) {
155         ProgressUpdateInfo *pui;
156         g_tree_foreach(pui_tree, (GTraverseFunc)get_next_pui, &pui);
157
158         if (pui->retries) {
159                 /* This is a download. */
160                 FILE *f;
161                 g_tree_steal(pui_tree, pui);
162                 g_tree_insert(downloading_tree, pui, pui);
163
164                 pui->src_str=map_construct_url(pui->tilex, pui->tiley, pui->zoom);
165
166                 if (!pui->src_str) {
167                         /* Failed to generate URL. */
168                         g_debug("Failed to generate tile download URL");
169                         g_idle_add_full(G_PRIORITY_HIGH_IDLE,(GSourceFunc)map_download_idle_refresh, pui, NULL);
170                         continue;
171                 }
172
173                 pui->dest_str=g_strdup_printf("%s/%u/%u/%u.jpg", pui->repo->cache_dir, pui->zoom, pui->tilex, pui->tiley);
174
175                 /* Check to see if we need to overwrite. */
176                 if (pui->retries > 0) {
177                         /* We're not updating - check if file already exists. */
178                         if (g_file_test(pui->dest_str, G_FILE_TEST_EXISTS)) {
179                                 g_idle_add_full(G_PRIORITY_HIGH_IDLE, (GSourceFunc)map_download_idle_refresh, pui, NULL);
180                                 continue;
181                         }
182                 }
183
184                 /* Attempt to open the file for writing. */
185                 if (!(f=g_fopen(pui->dest_str, "w")) && errno == ENOENT) {
186                         /* Directory doesn't exist yet - create it, then we'll retry */
187                         gchar buffer[BUFFER_SIZE];
188                         g_snprintf(buffer, sizeof(buffer), "%s/%u/%u", pui->repo->cache_dir, pui->zoom, pui->tilex);
189                         g_mkdir_with_parents(buffer, 0775);
190                         f=g_fopen(pui->dest_str, "w");
191                 }
192
193                 if (f) {
194                         CURL *curl_easy;
195                         pui->file = f;
196                         curl_easy = g_queue_pop_tail(curl_easy_queue);
197                         if (!curl_easy) {
198                                 /* Need a new curl_easy. */
199                                 MACRO_CURL_EASY_INIT(curl_easy);
200                         }
201                         curl_easy_setopt(curl_easy, CURLOPT_URL, pui->src_str);
202                         curl_easy_setopt(curl_easy, CURLOPT_WRITEDATA, f);
203                         g_hash_table_insert(pui_by_easy, curl_easy, pui);
204                         curl_multi_add_handle(_curl_multi, curl_easy);
205                         num_transfers++;
206                 } else {
207                         /* Unable to open tile file for writing. */
208                         gchar buffer[BUFFER_SIZE];
209                         g_snprintf(buffer, sizeof(buffer), "%s:\n%s", _("Failed to open file for writing"), pui->dest_str);
210                         MACRO_BANNER_SHOW_INFO(_window, buffer);
211                         g_idle_add_full(G_PRIORITY_HIGH_IDLE, (GSourceFunc)map_download_idle_refresh, pui, NULL);
212                         continue;
213                 }
214         } else if (--deletes_left) {
215                 /* This is a delete. */
216                 gchar buffer[BUFFER_SIZE];
217                 g_tree_steal(pui_tree, pui);
218                 g_tree_insert(downloading_tree, pui, pui);
219
220                 g_snprintf(buffer, sizeof(buffer), "%s/%u/%u/%u.jpg", pui->repo->cache_dir, pui->zoom, pui->tilex, pui->tiley);
221                 g_unlink(buffer);
222                 g_idle_add_full(G_PRIORITY_HIGH_IDLE,(GSourceFunc)map_download_idle_refresh, pui, NULL);
223         } else
224                 break;
225 }
226
227 if (!(num_transfers || g_tree_nnodes(pui_tree))) {
228         /* Destroy curl after 50 counts (5 seconds). */
229         if (--destroy_counter) {
230                 /* Clean up curl. */
231                 CURL *curr;
232                 while ((curr = g_queue_pop_tail(curl_easy_queue)))
233                         curl_easy_cleanup(curr);
234
235                 _curl_sid = 0;
236                 return FALSE;
237         }
238 } else
239         destroy_counter = 50;
240
241 return TRUE;
242 }
243
244 #ifdef MAP_DOWNLOAD_WMS
245 /**
246  * Given a wms uri pattern, compute the coordinate transformation and
247  * trimming.
248  * 'proj' is used for the conversion
249  */
250
251 #define WMS_TILE_SIZE_PIXELS (256)
252
253 static gchar *
254 map_convert_wms_to_wms(gint tilex, gint tiley, gint zoomlevel, gchar * uri)
255 {
256 gint system_retcode;
257 gchar cmd[BUFFER_SIZE], srs[BUFFER_SIZE];
258 gchar *ret = NULL;
259 FILE *in;
260 gdouble lon1, lat1, lon2, lat2;
261
262 gchar *widthstr = strcasestr(uri, "WIDTH=");
263 gchar *heightstr = strcasestr(uri, "HEIGHT=");
264 gchar *srsstr = strcasestr(uri, "SRS=EPSG");
265 gchar *srsstre = strchr(srsstr, '&');
266
267 /* missing: test if found */
268 strcpy(srs, "epsg");
269 strncpy(srs + 4, srsstr + 8, 256);
270 /* missing: test srsstre-srsstr < 526 */
271 srs[srsstre - srsstr - 4] = 0;
272 /* convert to lower, as WMC is EPSG and cs2cs is epsg */
273
274 gint dwidth = widthstr ? atoi(widthstr + 6) - WMS_TILE_SIZE_PIXELS : 0;
275 gint dheight = heightstr ? atoi(heightstr + 7) - WMS_TILE_SIZE_PIXELS : 0;
276
277 unit2latlon(tile2zunit(tilex, zoomlevel) - pixel2zunit(dwidth / 2, zoomlevel),
278             tile2zunit(tiley + 1, zoomlevel) + pixel2zunit((dheight + 1) / 2, zoomlevel), &lat1, &lon1);
279
280 unit2latlon(tile2zunit(tilex + 1, zoomlevel) + pixel2zunit((dwidth + 1) / 2, zoomlevel), 
281                 tile2zunit(tiley, zoomlevel) - pixel2zunit(dheight / 2, zoomlevel), &lat2, &lon2);
282
283 setlocale(LC_NUMERIC, "C");
284
285 g_snprintf(cmd, sizeof(cmd),
286          "(echo \"%.6f %.6f\"; echo \"%.6f %.6f\") | "
287          "/usr/bin/cs2cs +proj=longlat +datum=WGS84 +to +init=%s -f %%.6f "
288          " > /tmp/tmpcs2cs ", lon1, lat1, lon2, lat2, srs);
289 vprintf("Running command: %s\n", cmd);
290 system_retcode = system(cmd);
291
292 if (system_retcode)
293         g_printerr("cs2cs returned error code %d\n", WEXITSTATUS(system_retcode));
294 else if (!(in = g_fopen("/tmp/tmpcs2cs", "r")))
295         g_printerr("Cannot open results of conversion\n");
296 else if (5 != fscanf(in, "%lf %lf %s %lf %lf", &lon1, &lat1, cmd, &lon2, &lat2)) {
297         g_printerr("Wrong conversion\n");
298         fclose(in);
299 } else {
300         fclose(in);
301         ret = g_strdup_printf(uri, lon1, lat1, lon2, lat2);
302 }
303
304 setlocale(LC_NUMERIC, "");
305
306 return ret;
307 }
308 #endif
309
310 /**
311  * Given the xyz coordinates of our map coordinate system, write the qrst
312  * quadtree coordinates to buffer.
313  */
314 static void
315 map_convert_coords_to_quadtree_string(gint x, gint y, gint zoomlevel,
316                                       gchar * buffer, const gchar initial,
317                                       const gchar * const quadrant)
318 {
319 gchar *ptr = buffer;
320 gint n;
321
322 if (initial)
323         *ptr++ = initial;
324
325 for (n = 16 - zoomlevel; n >= 0; n--) {
326         gint xbit = (x >> n) & 1;
327         gint ybit = (y >> n) & 1;
328         *ptr++ = quadrant[xbit + 2 * ybit];
329 }
330 *ptr++ = '\0';
331 }
332
333 /**
334  * Construct the URL that we should fetch, based on the current URI format.
335  * This method works differently depending on if a "%s" string is present in
336  * the URI format, since that would indicate a quadtree-based map coordinate
337  * system.
338  */
339 static gchar *
340 map_construct_url(guint tilex, guint tiley, guint zoom)
341 {
342 switch (_curr_repo->type) {
343 case REPOTYPE_XYZ:
344         return g_strdup_printf(_curr_repo->url, tilex, tiley, zoom);
345 case REPOTYPE_XYZ_INV:
346         return g_strdup_printf(_curr_repo->url, 17 - zoom, tilex, tiley);
347 case REPOTYPE_QUAD_QRST: {
348                 gchar location[32];
349                 map_convert_coords_to_quadtree_string(tilex, tiley, zoom, location, 't', "qrts");
350                 return g_strdup_printf(_curr_repo->url, location);
351         }
352 case REPOTYPE_QUAD_ZERO: {
353                 /* This is a zero-based quadtree URI. */
354                 gchar location[32];
355                 map_convert_coords_to_quadtree_string(tilex, tiley, zoom, location, '\0', "0123");
356                 return g_strdup_printf(_curr_repo->url, location);
357         }
358 #ifdef MAP_DOWNLOAD_WMD
359 case REPOTYPE_WMS:
360         return map_convert_wms_to_wms(tilex, tiley, zoom, _curr_repo->url);
361 #endif
362 default:
363         return NULL;
364 }
365 return "";
366 }
367
368 void 
369 map_download_progress_clear(void) 
370 {
371 g_assert(_progress_item);
372 gtk_progress_bar_set_fraction(_progress_item, 0.0);
373 gtk_progress_bar_set_text(_progress_item, "");
374 }
375
376 void 
377 map_download_progress_set(gint currd, gint numd)
378 {
379 gchar buffer[64];
380
381 g_assert(_progress_item);
382 g_snprintf(buffer, sizeof(buffer), _("Downloading maps (%d/%d)"), currd, numd);
383 gtk_progress_bar_set_text(_progress_item, buffer);      
384 gtk_progress_bar_set_fraction(_progress_item, currd/(double)numd);
385 }
386
387 gboolean 
388 map_download_idle_refresh(ProgressUpdateInfo * pui)
389 {
390 guint tmp=0;
391 /* Test if download succeeded (only if retries != 0). */
392 if (!pui->retries || g_file_test(pui->dest_str, G_FILE_TEST_EXISTS)) {
393         gint zoom_diff = pui->zoom - gtk_map_get_zoom(_map);
394
395 #if 0
396         /* Only refresh at same or "lower" (more detailed) zoom level. */
397         if (zoom_diff >= 0) {
398                 guint tilex, tiley, tilex_end, tiley_end;
399
400                 /* If zoom has changed since we first put in the request for
401                  * this tile, then we may have to update more than one tile. */
402                 for (tilex = pui->tilex << zoom_diff, tilex_end = tilex + (1 << zoom_diff); tilex < tilex_end; tilex++) {
403                         for (tiley = pui->tiley << zoom_diff, tiley_end = tiley + (1 << zoom_diff); tiley < tiley_end; tiley++) {
404                                 tmp++;
405                                 if (map_render_tile(tilex, tiley, ((tilex - _base_tilex) << TILE_SIZE_P2), ((tiley - _base_tiley) << TILE_SIZE_P2), TRUE)==TRUE) {
406                                         map_render_data();
407                                         gtk_widget_queue_draw_area(_map_widget,
408                                              ((tilex - _base_tilex) << TILE_SIZE_P2) - _offsetx,
409                                              ((tiley - _base_tiley) << TILE_SIZE_P2) - _offsety,
410                                              TILE_SIZE_PIXELS, TILE_SIZE_PIXELS);
411                                 }
412                         }
413                 }
414         g_debug("*** MDIR LOOP: %u", tmp);
415         }
416 #else
417         g_debug("XXX: Add cb or signal or something to inform map widget about fresh tile");
418 #endif
419 }
420 /* Else the download failed. Update retries and maybe try again. */
421 else {
422         if (pui->retries > 0)
423                 --pui->retries;
424         else if (pui->retries < 0)
425                 ++pui->retries;
426
427         if (pui->retries) {
428                 /* removal automatically calls progress_update_info_free(). */
429                 g_tree_steal(downloading_tree, pui);
430                 g_tree_insert(pui_tree, pui, pui);
431                 if (iap_is_connected() && !_curl_sid)
432                         _curl_sid = g_timeout_add(100,(GSourceFunc)map_download_timeout, NULL);
433                 /* Don't do anything else. */
434                 return FALSE;
435         } else {
436                 /* No more retries left - something must be wrong. */
437                 MACRO_BANNER_SHOW_INFO(_window, _("Error in download. Check internet connection and/or Map Repository URL Format."));
438         }
439 }
440
441 /* removal automatically calls progress_update_info_free(). */
442 g_tree_remove(downloading_tree, pui);
443
444 if (++_curr_download == _num_downloads) {
445         _num_downloads = _curr_download = 0;
446         map_download_progress_clear();
447 } else {
448         map_download_progress_set(_curr_download, _num_downloads);
449 }
450 return FALSE;
451 }
452
453 /**
454  * Initiate a download of the given xyz coordinates using the given buffer
455  * as the URL.  If the map already exists on disk, or if we are already
456  * downloading the map, then this method does nothing.
457  */
458 void 
459 map_initiate_download(guint tilex, guint tiley, guint zoom, gint retries)
460 {
461 ProgressUpdateInfo *pui;
462
463 iap_connect();
464
465 pui = g_slice_new(ProgressUpdateInfo);
466 pui->tilex = tilex;
467 pui->tiley = tiley;
468 pui->zoom = zoom;
469 pui->priority = 1;
470 if (!retries)
471         pui->priority = -pui->priority; /* "Negative" makes them lowest pri. */
472 pui->retries = retries;
473 pui->repo = _curr_repo;
474
475 if (g_tree_lookup(pui_tree, pui) || g_tree_lookup(downloading_tree, pui)) {
476         /* Already downloading. */
477         g_slice_free(ProgressUpdateInfo, pui);
478         return;
479 }
480 pui->src_str = NULL;
481 pui->dest_str = NULL;
482 pui->file = NULL;
483
484 g_tree_insert(pui_tree, pui, pui);
485 if (iap_is_connected() && !_curl_sid)
486         _curl_sid = g_timeout_add(100, (GSourceFunc) map_download_timeout, NULL);
487
488 if (!_num_downloads++)
489         gtk_progress_bar_set_text(_progress_item, _("Downloading maps..."));    
490 }
491
492 void
493 map_download_init(void)
494 {
495 curl_easy_queue = g_queue_new();
496 pui_tree = g_tree_new_full((GCompareDataFunc)download_comparefunc, NULL, (GDestroyNotify) progress_update_info_free, NULL);
497 downloading_tree = g_tree_new_full((GCompareDataFunc)download_comparefunc, NULL, (GDestroyNotify) progress_update_info_free, NULL);
498 pui_by_easy = g_hash_table_new(g_direct_hash, g_direct_equal);
499 _curl_multi = curl_multi_init();
500 /* curl_multi_setopt(_curl_multi, CURLMOPT_PIPELINING, 1); */
501 }
502
503 gboolean
504 map_download_stop(void)
505 {
506 if (_curl_multi) {
507         CURL *curr;
508         CURLMsg *msg;
509         gint num_msgs;
510
511         /* Clear the tree and create it again */
512         if (pui_tree) {
513                 g_tree_destroy(pui_tree);
514                 pui_tree = g_tree_new_full((GCompareDataFunc)download_comparefunc, NULL, (GDestroyNotify) progress_update_info_free, NULL);
515         }
516
517         /* Close all finished files. */
518         while ((msg = curl_multi_info_read(_curl_multi, &num_msgs))) {
519                 if (msg->msg == CURLMSG_DONE) {
520                         /* This is a map download. */
521                         ProgressUpdateInfo *pui = g_hash_table_lookup(pui_by_easy, msg->easy_handle);
522                         g_queue_push_head(curl_easy_queue, msg->easy_handle);
523                         g_hash_table_remove(pui_by_easy, msg->easy_handle);
524                         if (pui->file)
525                                 fclose(pui->file);
526                         curl_multi_remove_handle(_curl_multi, msg->easy_handle);
527                 }
528         }
529
530         g_debug("Stopping downloads");
531         while ((curr = g_queue_pop_tail(curl_easy_queue)))
532                 curl_easy_cleanup(curr);
533         return TRUE;
534 }
535 g_debug("No downloads to stop");
536 return FALSE;
537 }
538
539 /*
540  * Clean up CURL and structures
541  */
542 void
543 map_download_deinit(void) 
544 {
545 if (_curl_multi) {
546         CURL *curr;
547         gint num_transfers;
548
549         /* Finish up all downloads. */
550         while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(_curl_multi, &num_transfers) || num_transfers) {
551                 g_debug("NT: %d", num_transfers);
552         }
553
554         /* Stop all transfers and cleanup */
555         map_download_stop();
556
557         if (pui_tree)
558                 g_tree_destroy(pui_tree);
559         if (downloading_tree)
560                 g_tree_destroy(downloading_tree);
561         if (pui_by_easy)
562                 g_hash_table_destroy(pui_by_easy);
563         if (curl_easy_queue)
564                 g_queue_free(curl_easy_queue);
565         curl_multi_cleanup(_curl_multi);
566         _curl_multi = NULL;
567 }
568
569 if (_curl_sid) {
570     g_source_remove(_curl_sid);
571     _curl_sid = 0;
572 }
573 }