err.no Git - mapper/blob - src/map-download.c

   1 /*
   2  * This file is part of mapper
   3  *
   4  * Copyright (C) 2007 Kaj-Michael Lang
   5  * Copyright (C) 2006-2007 John Costigan.
   6  *
   7  * POI and GPS-Info code originally written by Cezary Jackiewicz.
   8  *
   9  * Default map data provided by http://www.openstreetmap.org/
  10  *
  11  * This program is free software; you can redistribute it and/or modify
  12  * it under the terms of the GNU General Public License as published by
  13  * the Free Software Foundation; either version 2 of the License, or
  14  * (at your option) any later version.
  15  *
  16  * This program is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  * GNU General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU General Public License along
  22  * with this program; if not, write to the Free Software Foundation, Inc.,
  23  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  24  */
  25
  26 #include "config.h"
  27
  28 #include <unistd.h>
  29 #include <stdlib.h>
  30 #include <string.h>
  31 #include <strings.h>
  32 #include <stddef.h>
  33 #include <locale.h>
  34 #include <math.h>
  35 #include <errno.h>
  36 #include <sys/wait.h>
  37 #include <glib/gstdio.h>
  38 #include <fcntl.h>
  39 #include <curl/multi.h>
  40 #include <libintl.h>
  41 #include <locale.h>
  42
  43 #include "hildon-mapper.h"
  44
  45 #include "utils.h"
  46 #include "osm.h"
  47 #include "db.h"
  48 #include "osm-db.h"
  49 #include "poi.h"
  50 #include "route.h"
  51 #include "gps.h"
  52 #include "mapper-types.h"
  53 #include "ui-common.h"
  54 #include "settings.h"
  55 #include "latlon.h"
  56 #include "map-download.h"
  57 #include "iap.h"
  58 #include "map-repo.h"
  59 #include "gtkmap.h"
  60
  61 static guint _num_downloads=0;
  62 static guint _curr_download=0;
  63
  64 static GQueue *curl_easy_queue=NULL;
  65 static GTree *pui_tree=NULL;
  66 static GTree *downloading_tree=NULL;
  67 static GHashTable *pui_by_easy=NULL;
  68
  69 static gchar *map_construct_url(guint tilex, guint tiley, guint zoom);
  70
  71 static gboolean
  72 get_next_pui(gpointer key, gpointer value, ProgressUpdateInfo ** data)
  73 {
  74 *data = key;
  75 return TRUE;
  76 }
  77
  78 static gint
  79 download_comparefunc(const ProgressUpdateInfo * a, const ProgressUpdateInfo * b, gpointer user_data)
  80 {
  81 gint diff = (a->priority - b->priority);
  82 if (diff)
  83         return diff;
  84 diff = (a->tilex - b->tilex);
  85 if (diff)
  86         return diff;
  87 diff = (a->tiley - b->tiley);
  88 if (diff)
  89         return diff;
  90 diff = (a->zoom - b->zoom);
  91 if (diff)
  92         return diff;
  93 diff = (a->repo - b->repo);
  94 if (diff)
  95         return diff;
  96 /* Otherwise, deletes are "greatest" (least priority). */
  97 if (!a->retries)
  98         return (b->retries ? -1 : 0);
  99 else if (!b->retries)
 100         return (a->retries ? 1 : 0);
 101 /* Do updates after non-updates (because they'll both be done anyway). */
 102 return (a->retries - b->retries);
 103 }
 104
 105 /**
 106  * Free a ProgressUpdateInfo data structure that was allocated during the
 107  * auto-map-download process.
 108  */
 109 static void
 110 progress_update_info_free(ProgressUpdateInfo * pui)
 111 {
 112 g_free(pui->src_str);
 113 g_free(pui->dest_str);
 114 g_slice_free(ProgressUpdateInfo, pui);
 115 }
 116
 117 gboolean
 118 map_download_timeout()
 119 {
 120 static guint destroy_counter = 50;
 121 gint num_transfers = 0, num_msgs = 0;
 122 gint deletes_left = 50; /* only do 50 deletes at a time. */
 123 CURLMsg *msg;
 124
 125 if (_curl_multi && CURLM_CALL_MULTI_PERFORM == curl_multi_perform(_curl_multi, &num_transfers))
 126         return TRUE;    /* Give UI a chance first. */
 127
 128 while (_curl_multi && (msg = curl_multi_info_read(_curl_multi, &num_msgs))) {
 129         if (msg->msg == CURLMSG_DONE) {
 130                 if (msg->easy_handle == _autoroute_data.curl_easy) {
 131                         /* This is the autoroute download. */
 132                         /* Now, parse the autoroute and update the display. */
 133                         if (_autoroute_data.enabled && gpx_parse(&_route, _autoroute_data.rdl_data.bytes, _autoroute_data.rdl_data.bytes_read, 0)) {
 134                                 /* Find the nearest route point, if we're connected. */
 135                                 path_find_nearest_point(_route);
 136                         }
 137                         route_cancel_autoroute(_route, TRUE);   /* We're done. Clean up. */
 138                 } else {
 139                         ProgressUpdateInfo *pui = g_hash_table_lookup(pui_by_easy, msg->easy_handle);
 140                         g_queue_push_head(curl_easy_queue, msg->easy_handle);
 141                         g_hash_table_remove(pui_by_easy, msg->easy_handle);
 142                         fclose(pui->file);
 143                         if (msg->data.result != CURLE_OK)
 144                                 g_unlink(pui->dest_str);        /* Delete so we try again. */
 145                         curl_multi_remove_handle(_curl_multi, msg->easy_handle);
 146                         g_idle_add_full(G_PRIORITY_HIGH_IDLE, (GSourceFunc)map_download_idle_refresh, pui, NULL);
 147                 }
 148         }
 149 }
 150
 151 /* Up to 1 transfer per tile. */
 152 while (num_transfers < (4*4) && g_tree_nnodes(pui_tree)) {
 153         ProgressUpdateInfo *pui;
 154         g_tree_foreach(pui_tree, (GTraverseFunc)get_next_pui, &pui);
 155
 156         if (pui->retries) {
 157                 /* This is a download. */
 158                 FILE *f;
 159                 g_tree_steal(pui_tree, pui);
 160                 g_tree_insert(downloading_tree, pui, pui);
 161
 162                 pui->src_str=map_construct_url(pui->tilex, pui->tiley, pui->zoom);
 163
 164                 if (!pui->src_str) {
 165                         /* Failed to generate URL. */
 166                         g_debug("Failed to generate tile download URL");
 167                         g_idle_add_full(G_PRIORITY_HIGH_IDLE,(GSourceFunc)map_download_idle_refresh, pui, NULL);
 168                         continue;
 169                 }
 170
 171                 pui->dest_str=g_strdup_printf("%s/%u/%u/%u.jpg", pui->repo->cache_dir, pui->zoom, pui->tilex, pui->tiley);
 172
 173                 /* Check to see if we need to overwrite. */
 174                 if (pui->retries > 0) {
 175                         /* We're not updating - check if file already exists. */
 176                         if (g_file_test(pui->dest_str, G_FILE_TEST_EXISTS)) {
 177                                 g_idle_add_full(G_PRIORITY_HIGH_IDLE, (GSourceFunc)map_download_idle_refresh, pui, NULL);
 178                                 continue;
 179                         }
 180                 }
 181
 182                 /* Attempt to open the file for writing. */
 183                 if (!(f=g_fopen(pui->dest_str, "w")) && errno == ENOENT) {
 184                         /* Directory doesn't exist yet - create it, then we'll retry */
 185                         gchar buffer[BUFFER_SIZE];
 186                         g_snprintf(buffer, sizeof(buffer), "%s/%u/%u", pui->repo->cache_dir, pui->zoom, pui->tilex);
 187                         g_mkdir_with_parents(buffer, 0775);
 188                         f=g_fopen(pui->dest_str, "w");
 189                 }
 190
 191                 if (f) {
 192                         CURL *curl_easy;
 193                         pui->file = f;
 194                         curl_easy = g_queue_pop_tail(curl_easy_queue);
 195                         if (!curl_easy) {
 196                                 /* Need a new curl_easy. */
 197                                 MACRO_CURL_EASY_INIT(curl_easy);
 198                         }
 199                         curl_easy_setopt(curl_easy, CURLOPT_URL, pui->src_str);
 200                         curl_easy_setopt(curl_easy, CURLOPT_WRITEDATA, f);
 201                         g_hash_table_insert(pui_by_easy, curl_easy, pui);
 202                         curl_multi_add_handle(_curl_multi, curl_easy);
 203                         num_transfers++;
 204                 } else {
 205                         /* Unable to open tile file for writing. */
 206                         gchar buffer[BUFFER_SIZE];
 207                         g_snprintf(buffer, sizeof(buffer), "%s:\n%s", _("Failed to open file for writing"), pui->dest_str);
 208                         MACRO_BANNER_SHOW_INFO(_window, buffer);
 209                         g_idle_add_full(G_PRIORITY_HIGH_IDLE, (GSourceFunc)map_download_idle_refresh, pui, NULL);
 210                         continue;
 211                 }
 212         } else if (--deletes_left) {
 213                 /* This is a delete. */
 214                 gchar buffer[BUFFER_SIZE];
 215                 g_tree_steal(pui_tree, pui);
 216                 g_tree_insert(downloading_tree, pui, pui);
 217
 218                 g_snprintf(buffer, sizeof(buffer), "%s/%u/%u/%u.jpg", pui->repo->cache_dir, pui->zoom, pui->tilex, pui->tiley);
 219                 g_unlink(buffer);
 220                 g_idle_add_full(G_PRIORITY_HIGH_IDLE,(GSourceFunc)map_download_idle_refresh, pui, NULL);
 221         } else
 222                 break;
 223 }
 224
 225 if (!(num_transfers || g_tree_nnodes(pui_tree))) {
 226         /* Destroy curl after 50 counts (5 seconds). */
 227         if (--destroy_counter) {
 228                 /* Clean up curl. */
 229                 CURL *curr;
 230                 while ((curr = g_queue_pop_tail(curl_easy_queue)))
 231                         curl_easy_cleanup(curr);
 232
 233                 _curl_sid = 0;
 234                 return FALSE;
 235         }
 236 } else
 237         destroy_counter = 50;
 238
 239 return TRUE;
 240 }
 241
 242 #ifdef MAP_DOWNLOAD_WMS
 243 /**
 244  * Given a wms uri pattern, compute the coordinate transformation and
 245  * trimming.
 246  * 'proj' is used for the conversion
 247  */
 248
 249 #define WMS_TILE_SIZE_PIXELS (256)
 250
 251 static gchar *
 252 map_convert_wms_to_wms(gint tilex, gint tiley, gint zoomlevel, gchar * uri)
 253 {
 254 gint system_retcode;
 255 gchar cmd[BUFFER_SIZE], srs[BUFFER_SIZE];
 256 gchar *ret = NULL;
 257 FILE *in;
 258 gdouble lon1, lat1, lon2, lat2;
 259
 260 gchar *widthstr = strcasestr(uri, "WIDTH=");
 261 gchar *heightstr = strcasestr(uri, "HEIGHT=");
 262 gchar *srsstr = strcasestr(uri, "SRS=EPSG");
 263 gchar *srsstre = strchr(srsstr, '&');
 264
 265 /* missing: test if found */
 266 strcpy(srs, "epsg");
 267 strncpy(srs + 4, srsstr + 8, 256);
 268 /* missing: test srsstre-srsstr < 526 */
 269 srs[srsstre - srsstr - 4] = 0;
 270 /* convert to lower, as WMC is EPSG and cs2cs is epsg */
 271
 272 gint dwidth = widthstr ? atoi(widthstr + 6) - WMS_TILE_SIZE_PIXELS : 0;
 273 gint dheight = heightstr ? atoi(heightstr + 7) - WMS_TILE_SIZE_PIXELS : 0;
 274
 275 unit2latlon(tile2zunit(tilex, zoomlevel) - pixel2zunit(dwidth / 2, zoomlevel),
 276             tile2zunit(tiley + 1, zoomlevel) + pixel2zunit((dheight + 1) / 2, zoomlevel), &lat1, &lon1);
 277
 278 unit2latlon(tile2zunit(tilex + 1, zoomlevel) + pixel2zunit((dwidth + 1) / 2, zoomlevel),
 279                 tile2zunit(tiley, zoomlevel) - pixel2zunit(dheight / 2, zoomlevel), &lat2, &lon2);
 280
 281 setlocale(LC_NUMERIC, "C");
 282
 283 g_snprintf(cmd, sizeof(cmd),
 284          "(echo \"%.6f %.6f\"; echo \"%.6f %.6f\") | "
 285          "/usr/bin/cs2cs +proj=longlat +datum=WGS84 +to +init=%s -f %%.6f "
 286          " > /tmp/tmpcs2cs ", lon1, lat1, lon2, lat2, srs);
 287 vprintf("Running command: %s\n", cmd);
 288 system_retcode = system(cmd);
 289
 290 if (system_retcode)
 291         g_printerr("cs2cs returned error code %d\n", WEXITSTATUS(system_retcode));
 292 else if (!(in = g_fopen("/tmp/tmpcs2cs", "r")))
 293         g_printerr("Cannot open results of conversion\n");
 294 else if (5 != fscanf(in, "%lf %lf %s %lf %lf", &lon1, &lat1, cmd, &lon2, &lat2)) {
 295         g_printerr("Wrong conversion\n");
 296         fclose(in);
 297 } else {
 298         fclose(in);
 299         ret = g_strdup_printf(uri, lon1, lat1, lon2, lat2);
 300 }
 301
 302 setlocale(LC_NUMERIC, "");
 303
 304 return ret;
 305 }
 306 #endif
 307
 308 /**
 309  * Given the xyz coordinates of our map coordinate system, write the qrst
 310  * quadtree coordinates to buffer.
 311  */
 312 static void
 313 map_convert_coords_to_quadtree_string(gint x, gint y, gint zoomlevel,
 314                                       gchar * buffer, const gchar initial,
 315                                       const gchar * const quadrant)
 316 {
 317 gchar *ptr = buffer;
 318 gint n;
 319
 320 if (initial)
 321         *ptr++ = initial;
 322
 323 for (n = 16 - zoomlevel; n >= 0; n--) {
 324         gint xbit = (x >> n) & 1;
 325         gint ybit = (y >> n) & 1;
 326         *ptr++ = quadrant[xbit + 2 * ybit];
 327 }
 328 *ptr++ = '\0';
 329 }
 330
 331 /**
 332  * Construct the URL that we should fetch, based on the current URI format.
 333  * This method works differently depending on if a "%s" string is present in
 334  * the URI format, since that would indicate a quadtree-based map coordinate
 335  * system.
 336  */
 337 static gchar *
 338 map_construct_url(guint tilex, guint tiley, guint zoom)
 339 {
 340 switch (_curr_repo->type) {
 341 case REPOTYPE_XYZ:
 342         return g_strdup_printf(_curr_repo->url, tilex, tiley, zoom);
 343 case REPOTYPE_XYZ_INV:
 344         return g_strdup_printf(_curr_repo->url, 17 - zoom, tilex, tiley);
 345 case REPOTYPE_QUAD_QRST: {
 346                 gchar location[32];
 347                 map_convert_coords_to_quadtree_string(tilex, tiley, zoom, location, 't', "qrts");
 348                 return g_strdup_printf(_curr_repo->url, location);
 349         }
 350 case REPOTYPE_QUAD_ZERO: {
 351                 /* This is a zero-based quadtree URI. */
 352                 gchar location[32];
 353                 map_convert_coords_to_quadtree_string(tilex, tiley, zoom, location, '\0', "0123");
 354                 return g_strdup_printf(_curr_repo->url, location);
 355         }
 356 #ifdef MAP_DOWNLOAD_WMD
 357 case REPOTYPE_WMS:
 358         return map_convert_wms_to_wms(tilex, tiley, zoom, _curr_repo->url);
 359 #endif
 360 default:
 361         return NULL;
 362 }
 363 return "";
 364 }
 365
 366 void
 367 map_download_progress_clear(void)
 368 {
 369 g_assert(_progress_item);
 370 gtk_progress_bar_set_fraction(_progress_item, 0.0);
 371 gtk_progress_bar_set_text(_progress_item, "");
 372 }
 373
 374 void
 375 map_download_progress_set(gint currd, gint numd)
 376 {
 377 gchar buffer[64];
 378
 379 g_assert(_progress_item);
 380 g_snprintf(buffer, sizeof(buffer), _("Downloading maps (%d/%d)"), currd, numd);
 381 gtk_progress_bar_set_text(_progress_item, buffer);
 382 gtk_progress_bar_set_fraction(_progress_item, currd/(double)numd);
 383 }
 384
 385 gboolean
 386 map_download_idle_refresh(ProgressUpdateInfo * pui)
 387 {
 388 guint tmp=0;
 389 /* Test if download succeeded (only if retries != 0). */
 390 if (!pui->retries || g_file_test(pui->dest_str, G_FILE_TEST_EXISTS)) {
 391         gint zoom_diff = pui->zoom - gtk_map_get_zoom(_map);
 392
 393 #if 0
 394         /* Only refresh at same or "lower" (more detailed) zoom level. */
 395         if (zoom_diff >= 0) {
 396                 guint tilex, tiley, tilex_end, tiley_end;
 397
 398                 /* If zoom has changed since we first put in the request for
 399                  * this tile, then we may have to update more than one tile. */
 400                 for (tilex = pui->tilex << zoom_diff, tilex_end = tilex + (1 << zoom_diff); tilex < tilex_end; tilex++) {
 401                         for (tiley = pui->tiley << zoom_diff, tiley_end = tiley + (1 << zoom_diff); tiley < tiley_end; tiley++) {
 402                                 tmp++;
 403                                 if (map_render_tile(tilex, tiley, ((tilex - _base_tilex) << TILE_SIZE_P2), ((tiley - _base_tiley) << TILE_SIZE_P2), TRUE)==TRUE) {
 404                                         map_render_data();
 405                                         gtk_widget_queue_draw_area(_map_widget,
 406                                              ((tilex - _base_tilex) << TILE_SIZE_P2) - _offsetx,
 407                                              ((tiley - _base_tiley) << TILE_SIZE_P2) - _offsety,
 408                                              TILE_SIZE_PIXELS, TILE_SIZE_PIXELS);
 409                                 }
 410                         }
 411                 }
 412         g_debug("*** MDIR LOOP: %u", tmp);
 413         }
 414 #else
 415         g_debug("XXX: Add cb or signal or something to inform map widget about fresh tile");
 416 #endif
 417 }
 418 /* Else the download failed. Update retries and maybe try again. */
 419 else {
 420         if (pui->retries > 0)
 421                 --pui->retries;
 422         else if (pui->retries < 0)
 423                 ++pui->retries;
 424
 425         if (pui->retries) {
 426                 /* removal automatically calls progress_update_info_free(). */
 427                 g_tree_steal(downloading_tree, pui);
 428                 g_tree_insert(pui_tree, pui, pui);
 429                 if (iap_is_connected() && !_curl_sid)
 430                         _curl_sid = g_timeout_add(100,(GSourceFunc)map_download_timeout, NULL);
 431                 /* Don't do anything else. */
 432                 return FALSE;
 433         } else {
 434                 /* No more retries left - something must be wrong. */
 435                 MACRO_BANNER_SHOW_INFO(_window, _("Error in download. Check internet connection and/or Map Repository URL Format."));
 436         }
 437 }
 438
 439 /* removal automatically calls progress_update_info_free(). */
 440 g_tree_remove(downloading_tree, pui);
 441
 442 if (++_curr_download == _num_downloads) {
 443         _num_downloads = _curr_download = 0;
 444         map_download_progress_clear();
 445 } else {
 446         map_download_progress_set(_curr_download, _num_downloads);
 447 }
 448 return FALSE;
 449 }
 450
 451 /**
 452  * Initiate a download of the given xyz coordinates using the given buffer
 453  * as the URL.  If the map already exists on disk, or if we are already
 454  * downloading the map, then this method does nothing.
 455  */
 456 void
 457 map_initiate_download(guint tilex, guint tiley, guint zoom, gint retries)
 458 {
 459 ProgressUpdateInfo *pui;
 460
 461 iap_connect();
 462
 463 pui = g_slice_new(ProgressUpdateInfo);
 464 pui->tilex = tilex;
 465 pui->tiley = tiley;
 466 pui->zoom = zoom;
 467 pui->priority = 1;
 468 if (!retries)
 469         pui->priority = -pui->priority; /* "Negative" makes them lowest pri. */
 470 pui->retries = retries;
 471 pui->repo = _curr_repo;
 472
 473 if (g_tree_lookup(pui_tree, pui) || g_tree_lookup(downloading_tree, pui)) {
 474         /* Already downloading. */
 475         g_slice_free(ProgressUpdateInfo, pui);
 476         return;
 477 }
 478 pui->src_str = NULL;
 479 pui->dest_str = NULL;
 480 pui->file = NULL;
 481
 482 g_tree_insert(pui_tree, pui, pui);
 483 if (iap_is_connected() && !_curl_sid)
 484         _curl_sid = g_timeout_add(100, (GSourceFunc) map_download_timeout, NULL);
 485
 486 if (!_num_downloads++)
 487         gtk_progress_bar_set_text(_progress_item, _("Downloading maps..."));
 488 }
 489
 490 void
 491 map_download_init(void)
 492 {
 493 curl_easy_queue = g_queue_new();
 494 pui_tree = g_tree_new_full((GCompareDataFunc)download_comparefunc, NULL, (GDestroyNotify) progress_update_info_free, NULL);
 495 downloading_tree = g_tree_new_full((GCompareDataFunc)download_comparefunc, NULL, (GDestroyNotify) progress_update_info_free, NULL);
 496 pui_by_easy = g_hash_table_new(g_direct_hash, g_direct_equal);
 497 _curl_multi = curl_multi_init();
 498 /* curl_multi_setopt(_curl_multi, CURLMOPT_PIPELINING, 1); */
 499 }
 500
 501 gboolean
 502 map_download_stop(void)
 503 {
 504 if (_curl_multi) {
 505         CURL *curr;
 506         CURLMsg *msg;
 507         gint num_msgs;
 508
 509         /* Clear the tree and create it again */
 510         if (pui_tree) {
 511                 g_tree_destroy(pui_tree);
 512                 pui_tree = g_tree_new_full((GCompareDataFunc)download_comparefunc, NULL, (GDestroyNotify) progress_update_info_free, NULL);
 513         }
 514
 515         /* Close all finished files. */
 516         while ((msg = curl_multi_info_read(_curl_multi, &num_msgs))) {
 517                 if (msg->msg == CURLMSG_DONE) {
 518                         /* This is a map download. */
 519                         ProgressUpdateInfo *pui = g_hash_table_lookup(pui_by_easy, msg->easy_handle);
 520                         g_queue_push_head(curl_easy_queue, msg->easy_handle);
 521                         g_hash_table_remove(pui_by_easy, msg->easy_handle);
 522                         if (pui->file)
 523                                 fclose(pui->file);
 524                         curl_multi_remove_handle(_curl_multi, msg->easy_handle);
 525                 }
 526         }
 527
 528         g_debug("Stopping downloads");
 529         while ((curr = g_queue_pop_tail(curl_easy_queue)))
 530                 curl_easy_cleanup(curr);
 531         return TRUE;
 532 }
 533 g_debug("No downloads to stop");
 534 return FALSE;
 535 }
 536
 537 /*
 538  * Clean up CURL and structures
 539  */
 540 void
 541 map_download_deinit(void)
 542 {
 543 if (_curl_multi) {
 544         CURL *curr;
 545         gint num_transfers;
 546
 547         /* Finish up all downloads. */
 548         while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(_curl_multi, &num_transfers) || num_transfers) {
 549                 g_debug("NT: %d", num_transfers);
 550         }
 551
 552         /* Stop all transfers and cleanup */
 553         map_download_stop();
 554
 555         if (pui_tree)
 556                 g_tree_destroy(pui_tree);
 557         if (downloading_tree)
 558                 g_tree_destroy(downloading_tree);
 559         if (pui_by_easy)
 560                 g_hash_table_destroy(pui_by_easy);
 561         if (curl_easy_queue)
 562                 g_queue_free(curl_easy_queue);
 563         curl_multi_cleanup(_curl_multi);
 564         _curl_multi = NULL;
 565 }
 566
 567 if (_curl_sid) {
 568     g_source_remove(_curl_sid);
 569     _curl_sid = 0;
 570 }
 571 }