http-fetch.c

   1 #include "cache.h"
   2 #include "commit.h"
   3 #include "pack.h"
   4 #include "fetch.h"
   5
   6 #include <curl/curl.h>
   7 #include <curl/easy.h>
   8
   9 #if LIBCURL_VERSION_NUM >= 0x070908
  10 #define USE_CURL_MULTI
  11 #define DEFAULT_MAX_REQUESTS 5
  12 #endif
  13
  14 #if LIBCURL_VERSION_NUM < 0x070704
  15 #define curl_global_cleanup() do { /* nothing */ } while(0)
  16 #endif
  17 #if LIBCURL_VERSION_NUM < 0x070800
  18 #define curl_global_init(a) do { /* nothing */ } while(0)
  19 #endif
  20
  21 #if LIBCURL_VERSION_NUM < 0x070c04
  22 #define NO_CURL_EASY_DUPHANDLE
  23 #endif
  24
  25 #define PREV_BUF_SIZE 4096
  26 #define RANGE_HEADER_SIZE 30
  27
  28 static int active_requests = 0;
  29 static int data_received;
  30
  31 #ifdef USE_CURL_MULTI
  32 static int max_requests = -1;
  33 static CURLM *curlm;
  34 #endif
  35 #ifndef NO_CURL_EASY_DUPHANDLE
  36 static CURL *curl_default;
  37 #endif
  38 static struct curl_slist *pragma_header;
  39 static struct curl_slist *no_pragma_header;
  40 static struct curl_slist *no_range_header;
  41 static char curl_errorstr[CURL_ERROR_SIZE];
  42
  43 struct alt_base
  44 {
  45         char *base;
  46         int got_indices;
  47         struct packed_git *packs;
  48         struct alt_base *next;
  49 };
  50
  51 static struct alt_base *alt = NULL;
  52
  53 enum transfer_state {
  54         WAITING,
  55         ABORTED,
  56         ACTIVE,
  57         COMPLETE,
  58 };
  59
  60 struct transfer_request
  61 {
  62         unsigned char sha1[20];
  63         struct alt_base *repo;
  64         char *url;
  65         char filename[PATH_MAX];
  66         char tmpfile[PATH_MAX];
  67         int local;
  68         enum transfer_state state;
  69         CURLcode curl_result;
  70         char errorstr[CURL_ERROR_SIZE];
  71         long http_code;
  72         unsigned char real_sha1[20];
  73         SHA_CTX c;
  74         z_stream stream;
  75         int zret;
  76         int rename;
  77         struct active_request_slot *slot;
  78         struct transfer_request *next;
  79 };
  80
  81 struct active_request_slot
  82 {
  83         CURL *curl;
  84         FILE *local;
  85         int in_use;
  86         int done;
  87         CURLcode curl_result;
  88         struct active_request_slot *next;
  89 };
  90
  91 static struct transfer_request *request_queue_head = NULL;
  92 static struct active_request_slot *active_queue_head = NULL;
  93
  94 static int curl_ssl_verify = -1;
  95 static char *ssl_cert = NULL;
  96 #if LIBCURL_VERSION_NUM >= 0x070902
  97 static char *ssl_key = NULL;
  98 #endif
  99 #if LIBCURL_VERSION_NUM >= 0x070908
 100 static char *ssl_capath = NULL;
 101 #endif
 102 static char *ssl_cainfo = NULL;
 103 static long curl_low_speed_limit = -1;
 104 static long curl_low_speed_time = -1;
 105
 106 struct buffer
 107 {
 108         size_t posn;
 109         size_t size;
 110         void *buffer;
 111 };
 112
 113 static int http_options(const char *var, const char *value)
 114 {
 115         if (!strcmp("http.sslverify", var)) {
 116                 if (curl_ssl_verify == -1) {
 117                         curl_ssl_verify = git_config_bool(var, value);
 118                 }
 119                 return 0;
 120         }
 121
 122         if (!strcmp("http.sslcert", var)) {
 123                 if (ssl_cert == NULL) {
 124                         ssl_cert = xmalloc(strlen(value)+1);
 125                         strcpy(ssl_cert, value);
 126                 }
 127                 return 0;
 128         }
 129 #if LIBCURL_VERSION_NUM >= 0x070902
 130         if (!strcmp("http.sslkey", var)) {
 131                 if (ssl_key == NULL) {
 132                         ssl_key = xmalloc(strlen(value)+1);
 133                         strcpy(ssl_key, value);
 134                 }
 135                 return 0;
 136         }
 137 #endif
 138 #if LIBCURL_VERSION_NUM >= 0x070908
 139         if (!strcmp("http.sslcapath", var)) {
 140                 if (ssl_capath == NULL) {
 141                         ssl_capath = xmalloc(strlen(value)+1);
 142                         strcpy(ssl_capath, value);
 143                 }
 144                 return 0;
 145         }
 146 #endif
 147         if (!strcmp("http.sslcainfo", var)) {
 148                 if (ssl_cainfo == NULL) {
 149                         ssl_cainfo = xmalloc(strlen(value)+1);
 150                         strcpy(ssl_cainfo, value);
 151                 }
 152                 return 0;
 153         }
 154
 155 #ifdef USE_CURL_MULTI
 156         if (!strcmp("http.maxrequests", var)) {
 157                 if (max_requests == -1)
 158                         max_requests = git_config_int(var, value);
 159                 return 0;
 160         }
 161 #endif
 162
 163         if (!strcmp("http.lowspeedlimit", var)) {
 164                 if (curl_low_speed_limit == -1)
 165                         curl_low_speed_limit = (long)git_config_int(var, value);
 166                 return 0;
 167         }
 168         if (!strcmp("http.lowspeedtime", var)) {
 169                 if (curl_low_speed_time == -1)
 170                         curl_low_speed_time = (long)git_config_int(var, value);
 171                 return 0;
 172         }
 173
 174         /* Fall back on the default ones */
 175         return git_default_config(var, value);
 176 }
 177
 178 static size_t fwrite_buffer(void *ptr, size_t eltsize, size_t nmemb,
 179                             struct buffer *buffer)
 180 {
 181         size_t size = eltsize * nmemb;
 182         if (size > buffer->size - buffer->posn)
 183                 size = buffer->size - buffer->posn;
 184         memcpy(buffer->buffer + buffer->posn, ptr, size);
 185         buffer->posn += size;
 186         data_received++;
 187         return size;
 188 }
 189
 190 static size_t fwrite_buffer_dynamic(const void *ptr, size_t eltsize,
 191                                     size_t nmemb, struct buffer *buffer)
 192 {
 193         size_t size = eltsize * nmemb;
 194         if (size > buffer->size - buffer->posn) {
 195                 buffer->size = buffer->size * 3 / 2;
 196                 if (buffer->size < buffer->posn + size)
 197                         buffer->size = buffer->posn + size;
 198                 buffer->buffer = xrealloc(buffer->buffer, buffer->size);
 199         }
 200         memcpy(buffer->buffer + buffer->posn, ptr, size);
 201         buffer->posn += size;
 202         data_received++;
 203         return size;
 204 }
 205
 206 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
 207                                void *data)
 208 {
 209         unsigned char expn[4096];
 210         size_t size = eltsize * nmemb;
 211         int posn = 0;
 212         struct transfer_request *request = (struct transfer_request *)data;
 213         do {
 214                 ssize_t retval = write(request->local,
 215                                        ptr + posn, size - posn);
 216                 if (retval < 0)
 217                         return posn;
 218                 posn += retval;
 219         } while (posn < size);
 220
 221         request->stream.avail_in = size;
 222         request->stream.next_in = ptr;
 223         do {
 224                 request->stream.next_out = expn;
 225                 request->stream.avail_out = sizeof(expn);
 226                 request->zret = inflate(&request->stream, Z_SYNC_FLUSH);
 227                 SHA1_Update(&request->c, expn,
 228                             sizeof(expn) - request->stream.avail_out);
 229         } while (request->stream.avail_in && request->zret == Z_OK);
 230         data_received++;
 231         return size;
 232 }
 233
 234 #ifdef USE_CURL_MULTI
 235 static void process_curl_messages(void);
 236 static void process_request_queue(void);
 237 #endif
 238
 239 static CURL* get_curl_handle(void)
 240 {
 241         CURL* result = curl_easy_init();
 242
 243         curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify);
 244 #if LIBCURL_VERSION_NUM >= 0x070907
 245         curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
 246 #endif
 247
 248         if (ssl_cert != NULL)
 249                 curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
 250 #if LIBCURL_VERSION_NUM >= 0x070902
 251         if (ssl_key != NULL)
 252                 curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
 253 #endif
 254 #if LIBCURL_VERSION_NUM >= 0x070908
 255         if (ssl_capath != NULL)
 256                 curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
 257 #endif
 258         if (ssl_cainfo != NULL)
 259                 curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
 260         curl_easy_setopt(result, CURLOPT_FAILONERROR, 1);
 261
 262         if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
 263                 curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
 264                                  curl_low_speed_limit);
 265                 curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
 266                                  curl_low_speed_time);
 267         }
 268
 269         return result;
 270 }
 271
 272 static struct active_request_slot *get_active_slot(void)
 273 {
 274         struct active_request_slot *slot = active_queue_head;
 275         struct active_request_slot *newslot;
 276
 277 #ifdef USE_CURL_MULTI
 278         int num_transfers;
 279
 280         /* Wait for a slot to open up if the queue is full */
 281         while (active_requests >= max_requests) {
 282                 curl_multi_perform(curlm, &num_transfers);
 283                 if (num_transfers < active_requests) {
 284                         process_curl_messages();
 285                 }
 286         }
 287 #endif
 288
 289         while (slot != NULL && slot->in_use) {
 290                 slot = slot->next;
 291         }
 292         if (slot == NULL) {
 293                 newslot = xmalloc(sizeof(*newslot));
 294                 newslot->curl = NULL;
 295                 newslot->in_use = 0;
 296                 newslot->next = NULL;
 297
 298                 slot = active_queue_head;
 299                 if (slot == NULL) {
 300                         active_queue_head = newslot;
 301                 } else {
 302                         while (slot->next != NULL) {
 303                                 slot = slot->next;
 304                         }
 305                         slot->next = newslot;
 306                 }
 307                 slot = newslot;
 308         }
 309
 310         if (slot->curl == NULL) {
 311 #ifdef NO_CURL_EASY_DUPHANDLE
 312                 slot->curl = get_curl_handle();
 313 #else
 314                 slot->curl = curl_easy_duphandle(curl_default);
 315 #endif
 316         }
 317
 318         active_requests++;
 319         slot->in_use = 1;
 320         slot->done = 0;
 321         slot->local = NULL;
 322         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
 323         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_range_header);
 324         curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
 325
 326         return slot;
 327 }
 328
 329 static int start_active_slot(struct active_request_slot *slot)
 330 {
 331 #ifdef USE_CURL_MULTI
 332         CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
 333
 334         if (curlm_result != CURLM_OK &&
 335             curlm_result != CURLM_CALL_MULTI_PERFORM) {
 336                 active_requests--;
 337                 slot->in_use = 0;
 338                 return 0;
 339         }
 340 #endif
 341         return 1;
 342 }
 343
 344 static void run_active_slot(struct active_request_slot *slot)
 345 {
 346 #ifdef USE_CURL_MULTI
 347         int num_transfers;
 348         long last_pos = 0;
 349         long current_pos;
 350         fd_set readfds;
 351         fd_set writefds;
 352         fd_set excfds;
 353         int max_fd;
 354         struct timeval select_timeout;
 355         CURLMcode curlm_result;
 356
 357         while (!slot->done) {
 358                 data_received = 0;
 359                 do {
 360                         curlm_result = curl_multi_perform(curlm,
 361                                                           &num_transfers);
 362                 } while (curlm_result == CURLM_CALL_MULTI_PERFORM);
 363                 if (num_transfers < active_requests) {
 364                         process_curl_messages();
 365                         process_request_queue();
 366                 }
 367
 368                 if (!data_received && slot->local != NULL) {
 369                         current_pos = ftell(slot->local);
 370                         if (current_pos > last_pos)
 371                                 data_received++;
 372                         last_pos = current_pos;
 373                 }
 374
 375                 if (!slot->done && !data_received) {
 376                         max_fd = 0;
 377                         FD_ZERO(&readfds);
 378                         FD_ZERO(&writefds);
 379                         FD_ZERO(&excfds);
 380                         select_timeout.tv_sec = 0;
 381                         select_timeout.tv_usec = 50000;
 382                         select(max_fd, &readfds, &writefds,
 383                                &excfds, &select_timeout);
 384                 }
 385         }
 386 #else
 387         slot->curl_result = curl_easy_perform(slot->curl);
 388         active_requests--;
 389 #endif
 390 }
 391
 392 static void start_request(struct transfer_request *request)
 393 {
 394         char *hex = sha1_to_hex(request->sha1);
 395         char prevfile[PATH_MAX];
 396         char *url;
 397         char *posn;
 398         int prevlocal;
 399         unsigned char prev_buf[PREV_BUF_SIZE];
 400         ssize_t prev_read = 0;
 401         long prev_posn = 0;
 402         char range[RANGE_HEADER_SIZE];
 403         struct curl_slist *range_header = NULL;
 404         struct active_request_slot *slot;
 405
 406         snprintf(prevfile, sizeof(prevfile), "%s.prev", request->filename);
 407         unlink(prevfile);
 408         rename(request->tmpfile, prevfile);
 409         unlink(request->tmpfile);
 410
 411         request->local = open(request->tmpfile,
 412                               O_WRONLY | O_CREAT | O_EXCL, 0666);
 413         /* This could have failed due to the "lazy directory creation";
 414          * try to mkdir the last path component.
 415          */
 416         if (request->local < 0 && errno == ENOENT) {
 417                 char *dir = strrchr(request->tmpfile, '/');
 418                 if (dir) {
 419                         *dir = 0;
 420                         mkdir(request->tmpfile, 0777);
 421                         *dir = '/';
 422                 }
 423                 request->local = open(request->tmpfile,
 424                                       O_WRONLY | O_CREAT | O_EXCL, 0666);
 425         }
 426
 427         if (request->local < 0) {
 428                 request->state = ABORTED;
 429                 error("Couldn't create temporary file %s for %s: %s\n",
 430                       request->tmpfile, request->filename, strerror(errno));
 431                 return;
 432         }
 433
 434         memset(&request->stream, 0, sizeof(request->stream));
 435
 436         inflateInit(&request->stream);
 437
 438         SHA1_Init(&request->c);
 439
 440         url = xmalloc(strlen(request->repo->base) + 50);
 441         request->url = xmalloc(strlen(request->repo->base) + 50);
 442         strcpy(url, request->repo->base);
 443         posn = url + strlen(request->repo->base);
 444         strcpy(posn, "objects/");
 445         posn += 8;
 446         memcpy(posn, hex, 2);
 447         posn += 2;
 448         *(posn++) = '/';
 449         strcpy(posn, hex + 2);
 450         strcpy(request->url, url);
 451
 452         /* If a previous temp file is present, process what was already
 453            fetched. */
 454         prevlocal = open(prevfile, O_RDONLY);
 455         if (prevlocal != -1) {
 456                 do {
 457                         prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE);
 458                         if (prev_read>0) {
 459                                 if (fwrite_sha1_file(prev_buf,
 460                                                      1,
 461                                                      prev_read,
 462                                                      request) == prev_read) {
 463                                         prev_posn += prev_read;
 464                                 } else {
 465                                         prev_read = -1;
 466                                 }
 467                         }
 468                 } while (prev_read > 0);
 469                 close(prevlocal);
 470         }
 471         unlink(prevfile);
 472
 473         /* Reset inflate/SHA1 if there was an error reading the previous temp
 474            file; also rewind to the beginning of the local file. */
 475         if (prev_read == -1) {
 476                 memset(&request->stream, 0, sizeof(request->stream));
 477                 inflateInit(&request->stream);
 478                 SHA1_Init(&request->c);
 479                 if (prev_posn>0) {
 480                         prev_posn = 0;
 481                         lseek(request->local, SEEK_SET, 0);
 482                         ftruncate(request->local, 0);
 483                 }
 484         }
 485
 486         slot = get_active_slot();
 487         curl_easy_setopt(slot->curl, CURLOPT_FILE, request);
 488         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
 489         curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr);
 490         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 491         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 492
 493         /* If we have successfully processed data from a previous fetch
 494            attempt, only fetch the data we don't already have. */
 495         if (prev_posn>0) {
 496                 if (get_verbosely)
 497                         fprintf(stderr,
 498                                 "Resuming fetch of object %s at byte %ld\n",
 499                                 hex, prev_posn);
 500                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 501                 range_header = curl_slist_append(range_header, range);
 502                 curl_easy_setopt(slot->curl,
 503                                  CURLOPT_HTTPHEADER, range_header);
 504         }
 505
 506         /* Try to get the request started, abort the request on error */
 507         if (!start_active_slot(slot)) {
 508                 request->state = ABORTED;
 509                 close(request->local);
 510                 free(request->url);
 511                 return;
 512         }
 513
 514         request->slot = slot;
 515         request->state = ACTIVE;
 516 }
 517
 518 static void finish_request(struct transfer_request *request)
 519 {
 520         fchmod(request->local, 0444);
 521         close(request->local);
 522
 523         if (request->http_code == 416) {
 524                 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
 525         } else if (request->curl_result != CURLE_OK) {
 526                 return;
 527         }
 528
 529         inflateEnd(&request->stream);
 530         SHA1_Final(request->real_sha1, &request->c);
 531         if (request->zret != Z_STREAM_END) {
 532                 unlink(request->tmpfile);
 533                 return;
 534         }
 535         if (memcmp(request->sha1, request->real_sha1, 20)) {
 536                 unlink(request->tmpfile);
 537                 return;
 538         }
 539         request->rename =
 540                 move_temp_to_file(request->tmpfile, request->filename);
 541
 542         if (request->rename == 0)
 543                 pull_say("got %s\n", sha1_to_hex(request->sha1));
 544 }
 545
 546 static void release_request(struct transfer_request *request)
 547 {
 548         struct transfer_request *entry = request_queue_head;
 549
 550         if (request == request_queue_head) {
 551                 request_queue_head = request->next;
 552         } else {
 553                 while (entry->next != NULL && entry->next != request)
 554                         entry = entry->next;
 555                 if (entry->next == request)
 556                         entry->next = entry->next->next;
 557         }
 558
 559         free(request->url);
 560         free(request);
 561 }
 562
 563 #ifdef USE_CURL_MULTI
 564 void process_curl_messages(void)
 565 {
 566         int num_messages;
 567         struct active_request_slot *slot;
 568         struct transfer_request *request = NULL;
 569         CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
 570
 571         while (curl_message != NULL) {
 572                 if (curl_message->msg == CURLMSG_DONE) {
 573                         slot = active_queue_head;
 574                         while (slot != NULL &&
 575                                slot->curl != curl_message->easy_handle)
 576                                 slot = slot->next;
 577                         if (slot != NULL) {
 578                                 curl_multi_remove_handle(curlm, slot->curl);
 579                                 active_requests--;
 580                                 slot->done = 1;
 581                                 slot->in_use = 0;
 582                                 slot->curl_result = curl_message->data.result;
 583                                 request = request_queue_head;
 584                                 while (request != NULL &&
 585                                        request->slot != slot)
 586                                         request = request->next;
 587                         } else {
 588                                 fprintf(stderr, "Received DONE message for unknown request!\n");
 589                         }
 590                         if (request != NULL) {
 591                                 request->curl_result =
 592                                         curl_message->data.result;
 593                                 curl_easy_getinfo(slot->curl,
 594                                                   CURLINFO_HTTP_CODE,
 595                                                   &request->http_code);
 596                                 request->slot = NULL;
 597
 598                                 /* Use alternates if necessary */
 599                                 if (request->http_code == 404 &&
 600                                     request->repo->next != NULL) {
 601                                         request->repo = request->repo->next;
 602                                         start_request(request);
 603                                 } else {
 604                                         finish_request(request);
 605                                         request->state = COMPLETE;
 606                                 }
 607                         }
 608                 } else {
 609                         fprintf(stderr, "Unknown CURL message received: %d\n",
 610                                 (int)curl_message->msg);
 611                 }
 612                 curl_message = curl_multi_info_read(curlm, &num_messages);
 613         }
 614 }
 615
 616 void process_request_queue(void)
 617 {
 618         struct transfer_request *request = request_queue_head;
 619         struct active_request_slot *slot = active_queue_head;
 620         int num_transfers;
 621
 622         while (active_requests < max_requests && request != NULL) {
 623                 if (request->state == WAITING) {
 624                         if (has_sha1_file(request->sha1))
 625                                 release_request(request);
 626                         else
 627                                 start_request(request);
 628                         curl_multi_perform(curlm, &num_transfers);
 629                 }
 630                 request = request->next;
 631         }
 632
 633         while (slot != NULL) {
 634                 if (!slot->in_use && slot->curl != NULL) {
 635                         curl_easy_cleanup(slot->curl);
 636                         slot->curl = NULL;
 637                 }
 638                 slot = slot->next;
 639         }
 640 }
 641 #endif
 642
 643 void prefetch(unsigned char *sha1)
 644 {
 645         struct transfer_request *newreq;
 646         struct transfer_request *tail;
 647         char *filename = sha1_file_name(sha1);
 648
 649         newreq = xmalloc(sizeof(*newreq));
 650         memcpy(newreq->sha1, sha1, 20);
 651         newreq->repo = alt;
 652         newreq->url = NULL;
 653         newreq->local = -1;
 654         newreq->state = WAITING;
 655         snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
 656         snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
 657                  "%s.temp", filename);
 658         newreq->next = NULL;
 659
 660         if (request_queue_head == NULL) {
 661                 request_queue_head = newreq;
 662         } else {
 663                 tail = request_queue_head;
 664                 while (tail->next != NULL) {
 665                         tail = tail->next;
 666                 }
 667                 tail->next = newreq;
 668         }
 669 #ifdef USE_CURL_MULTI
 670         process_request_queue();
 671         process_curl_messages();
 672 #endif
 673 }
 674
 675 static int fetch_index(struct alt_base *repo, unsigned char *sha1)
 676 {
 677         char *hex = sha1_to_hex(sha1);
 678         char *filename;
 679         char *url;
 680         char tmpfile[PATH_MAX];
 681         long prev_posn = 0;
 682         char range[RANGE_HEADER_SIZE];
 683         struct curl_slist *range_header = NULL;
 684
 685         FILE *indexfile;
 686         struct active_request_slot *slot;
 687
 688         if (has_pack_index(sha1))
 689                 return 0;
 690
 691         if (get_verbosely)
 692                 fprintf(stderr, "Getting index for pack %s\n", hex);
 693
 694         url = xmalloc(strlen(repo->base) + 64);
 695         sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
 696
 697         filename = sha1_pack_index_name(sha1);
 698         snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 699         indexfile = fopen(tmpfile, "a");
 700         if (!indexfile)
 701                 return error("Unable to open local file %s for pack index",
 702                              filename);
 703
 704         slot = get_active_slot();
 705         curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
 706         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 707         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 708         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 709         slot->local = indexfile;
 710
 711         /* If there is data present from a previous transfer attempt,
 712            resume where it left off */
 713         prev_posn = ftell(indexfile);
 714         if (prev_posn>0) {
 715                 if (get_verbosely)
 716                         fprintf(stderr,
 717                                 "Resuming fetch of index for pack %s at byte %ld\n",
 718                                 hex, prev_posn);
 719                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 720                 range_header = curl_slist_append(range_header, range);
 721                 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 722         }
 723
 724         if (start_active_slot(slot)) {
 725                 run_active_slot(slot);
 726                 if (slot->curl_result != CURLE_OK) {
 727                         fclose(indexfile);
 728                         return error("Unable to get pack index %s\n%s", url,
 729                                      curl_errorstr);
 730                 }
 731         } else {
 732                 return error("Unable to start request");
 733         }
 734
 735         fclose(indexfile);
 736
 737         return move_temp_to_file(tmpfile, filename);
 738 }
 739
 740 static int setup_index(struct alt_base *repo, unsigned char *sha1)
 741 {
 742         struct packed_git *new_pack;
 743         if (has_pack_file(sha1))
 744                 return 0; // don't list this as something we can get
 745
 746         if (fetch_index(repo, sha1))
 747                 return -1;
 748
 749         new_pack = parse_pack_index(sha1);
 750         new_pack->next = repo->packs;
 751         repo->packs = new_pack;
 752         return 0;
 753 }
 754
 755 static int fetch_alternates(char *base)
 756 {
 757         int ret = 0;
 758         struct buffer buffer;
 759         char *url;
 760         char *data;
 761         int i = 0;
 762         int http_specific = 1;
 763         struct alt_base *tail = alt;
 764         static const char null_byte = '\0';
 765
 766         struct active_request_slot *slot;
 767
 768         data = xmalloc(4096);
 769         buffer.size = 4096;
 770         buffer.posn = 0;
 771         buffer.buffer = data;
 772
 773         if (get_verbosely)
 774                 fprintf(stderr, "Getting alternates list\n");
 775
 776         url = xmalloc(strlen(base) + 31);
 777         sprintf(url, "%s/objects/info/http-alternates", base);
 778
 779         slot = get_active_slot();
 780         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 781         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
 782                          fwrite_buffer_dynamic);
 783         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 784         if (start_active_slot(slot)) {
 785                 run_active_slot(slot);
 786                 if (slot->curl_result != CURLE_OK || !buffer.posn) {
 787                         http_specific = 0;
 788
 789                         sprintf(url, "%s/objects/info/alternates", base);
 790
 791                         slot = get_active_slot();
 792                         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 793                         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
 794                                          fwrite_buffer_dynamic);
 795                         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 796                         if (start_active_slot(slot)) {
 797                                 run_active_slot(slot);
 798                                 if (slot->curl_result != CURLE_OK) {
 799                                         free(buffer.buffer);
 800                                         return 0;
 801                                 }
 802                         }
 803                 }
 804         } else {
 805                 free(buffer.buffer);
 806                 return 0;
 807         }
 808
 809         fwrite_buffer_dynamic(&null_byte, 1, 1, &buffer);
 810         buffer.posn--;
 811         data = buffer.buffer;
 812
 813         while (i < buffer.posn) {
 814                 int posn = i;
 815                 while (posn < buffer.posn && data[posn] != '\n')
 816                         posn++;
 817                 if (data[posn] == '\n') {
 818                         int okay = 0;
 819                         int serverlen = 0;
 820                         struct alt_base *newalt;
 821                         char *target = NULL;
 822                         if (data[i] == '/') {
 823                                 serverlen = strchr(base + 8, '/') - base;
 824                                 okay = 1;
 825                         } else if (!memcmp(data + i, "../", 3)) {
 826                                 i += 3;
 827                                 serverlen = strlen(base);
 828                                 while (i + 2 < posn &&
 829                                        !memcmp(data + i, "../", 3)) {
 830                                         do {
 831                                                 serverlen--;
 832                                         } while (serverlen &&
 833                                                  base[serverlen - 1] != '/');
 834                                         i += 3;
 835                                 }
 836                                 // If the server got removed, give up.
 837                                 okay = strchr(base, ':') - base + 3 <
 838                                         serverlen;
 839                         } else if (http_specific) {
 840                                 char *colon = strchr(data + i, ':');
 841                                 char *slash = strchr(data + i, '/');
 842                                 if (colon && slash && colon < data + posn &&
 843                                     slash < data + posn && colon < slash) {
 844                                         okay = 1;
 845                                 }
 846                         }
 847                         // skip 'objects' at end
 848                         if (okay) {
 849                                 target = xmalloc(serverlen + posn - i - 6);
 850                                 strncpy(target, base, serverlen);
 851                                 strncpy(target + serverlen, data + i,
 852                                         posn - i - 7);
 853                                 target[serverlen + posn - i - 7] = '\0';
 854                                 if (get_verbosely)
 855                                         fprintf(stderr,
 856                                                 "Also look at %s\n", target);
 857                                 newalt = xmalloc(sizeof(*newalt));
 858                                 newalt->next = NULL;
 859                                 newalt->base = target;
 860                                 newalt->got_indices = 0;
 861                                 newalt->packs = NULL;
 862                                 while (tail->next != NULL)
 863                                         tail = tail->next;
 864                                 tail->next = newalt;
 865                                 ret++;
 866                         }
 867                 }
 868                 i = posn + 1;
 869         }
 870
 871         free(buffer.buffer);
 872         return ret;
 873 }
 874
 875 static int fetch_indices(struct alt_base *repo)
 876 {
 877         unsigned char sha1[20];
 878         char *url;
 879         struct buffer buffer;
 880         char *data;
 881         int i = 0;
 882
 883         struct active_request_slot *slot;
 884
 885         if (repo->got_indices)
 886                 return 0;
 887
 888         data = xmalloc(4096);
 889         buffer.size = 4096;
 890         buffer.posn = 0;
 891         buffer.buffer = data;
 892
 893         if (get_verbosely)
 894                 fprintf(stderr, "Getting pack list\n");
 895
 896         url = xmalloc(strlen(repo->base) + 21);
 897         sprintf(url, "%s/objects/info/packs", repo->base);
 898
 899         slot = get_active_slot();
 900         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 901         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
 902                          fwrite_buffer_dynamic);
 903         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 904         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
 905         if (start_active_slot(slot)) {
 906                 run_active_slot(slot);
 907                 if (slot->curl_result != CURLE_OK) {
 908                         free(buffer.buffer);
 909                         return error("%s", curl_errorstr);
 910                 }
 911         } else {
 912                 free(buffer.buffer);
 913                 return error("Unable to start request");
 914         }
 915
 916         data = buffer.buffer;
 917         while (i < buffer.posn) {
 918                 switch (data[i]) {
 919                 case 'P':
 920                         i++;
 921                         if (i + 52 < buffer.posn &&
 922                             !strncmp(data + i, " pack-", 6) &&
 923                             !strncmp(data + i + 46, ".pack\n", 6)) {
 924                                 get_sha1_hex(data + i + 6, sha1);
 925                                 setup_index(repo, sha1);
 926                                 i += 51;
 927                                 break;
 928                         }
 929                 default:
 930                         while (data[i] != '\n')
 931                                 i++;
 932                 }
 933                 i++;
 934         }
 935
 936         free(buffer.buffer);
 937         repo->got_indices = 1;
 938         return 0;
 939 }
 940
 941 static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
 942 {
 943         char *url;
 944         struct packed_git *target;
 945         struct packed_git **lst;
 946         FILE *packfile;
 947         char *filename;
 948         char tmpfile[PATH_MAX];
 949         int ret;
 950         long prev_posn = 0;
 951         char range[RANGE_HEADER_SIZE];
 952         struct curl_slist *range_header = NULL;
 953
 954         struct active_request_slot *slot;
 955
 956         if (fetch_indices(repo))
 957                 return -1;
 958         target = find_sha1_pack(sha1, repo->packs);
 959         if (!target)
 960                 return -1;
 961
 962         if (get_verbosely) {
 963                 fprintf(stderr, "Getting pack %s\n",
 964                         sha1_to_hex(target->sha1));
 965                 fprintf(stderr, " which contains %s\n",
 966                         sha1_to_hex(sha1));
 967         }
 968
 969         url = xmalloc(strlen(repo->base) + 65);
 970         sprintf(url, "%s/objects/pack/pack-%s.pack",
 971                 repo->base, sha1_to_hex(target->sha1));
 972
 973         filename = sha1_pack_name(target->sha1);
 974         snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 975         packfile = fopen(tmpfile, "a");
 976         if (!packfile)
 977                 return error("Unable to open local file %s for pack",
 978                              filename);
 979
 980         slot = get_active_slot();
 981         curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
 982         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 983         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 984         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 985         slot->local = packfile;
 986
 987         /* If there is data present from a previous transfer attempt,
 988            resume where it left off */
 989         prev_posn = ftell(packfile);
 990         if (prev_posn>0) {
 991                 if (get_verbosely)
 992                         fprintf(stderr,
 993                                 "Resuming fetch of pack %s at byte %ld\n",
 994                                 sha1_to_hex(target->sha1), prev_posn);
 995                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 996                 range_header = curl_slist_append(range_header, range);
 997                 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 998         }
 999
1000         if (start_active_slot(slot)) {
1001                 run_active_slot(slot);
1002                 if (slot->curl_result != CURLE_OK) {
1003                         fclose(packfile);
1004                         return error("Unable to get pack file %s\n%s", url,
1005                                      curl_errorstr);
1006                 }
1007         } else {
1008                 return error("Unable to start request");
1009         }
1010
1011         fclose(packfile);
1012
1013         ret = move_temp_to_file(tmpfile, filename);
1014         if (ret)
1015                 return ret;
1016
1017         lst = &repo->packs;
1018         while (*lst != target)
1019                 lst = &((*lst)->next);
1020         *lst = (*lst)->next;
1021
1022         if (verify_pack(target, 0))
1023                 return -1;
1024         install_packed_git(target);
1025
1026         return 0;
1027 }
1028
1029 static int fetch_object(struct alt_base *repo, unsigned char *sha1)
1030 {
1031         char *hex = sha1_to_hex(sha1);
1032         int ret;
1033         struct transfer_request *request = request_queue_head;
1034
1035         while (request != NULL && memcmp(request->sha1, sha1, 20))
1036                 request = request->next;
1037         if (request == NULL)
1038                 return error("Couldn't find request for %s in the queue", hex);
1039
1040         if (has_sha1_file(request->sha1)) {
1041                 release_request(request);
1042                 return 0;
1043         }
1044
1045 #ifdef USE_CURL_MULTI
1046         while (request->state == WAITING) {
1047                 int num_transfers;
1048                 curl_multi_perform(curlm, &num_transfers);
1049                 if (num_transfers < active_requests) {
1050                         process_curl_messages();
1051                         process_request_queue();
1052                 }
1053         }
1054 #else
1055         start_request(request);
1056 #endif
1057
1058         while (request->state == ACTIVE) {
1059                 run_active_slot(request->slot);
1060 #ifndef USE_CURL_MULTI
1061                 request->curl_result = request->slot->curl_result;
1062                 curl_easy_getinfo(request->slot->curl,
1063                                   CURLINFO_HTTP_CODE,
1064                                   &request->http_code);
1065                 request->slot = NULL;
1066
1067                 /* Use alternates if necessary */
1068                 if (request->http_code == 404 &&
1069                     request->repo->next != NULL) {
1070                         request->repo = request->repo->next;
1071                         start_request(request);
1072                 } else {
1073                         finish_request(request);
1074                         request->state = COMPLETE;
1075                 }
1076 #endif
1077         }
1078
1079         if (request->state == ABORTED) {
1080                 release_request(request);
1081                 return error("Request for %s aborted", hex);
1082         }
1083
1084         if (request->curl_result != CURLE_OK && request->http_code != 416) {
1085                 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
1086                             request->errorstr, request->curl_result,
1087                             request->http_code, hex);
1088                 release_request(request);
1089                 return ret;
1090         }
1091
1092         if (request->zret != Z_STREAM_END) {
1093                 ret = error("File %s (%s) corrupt\n", hex, request->url);
1094                 release_request(request);
1095                 return ret;
1096         }
1097
1098         if (memcmp(request->sha1, request->real_sha1, 20)) {
1099                 release_request(request);
1100                 return error("File %s has bad hash\n", hex);
1101         }
1102
1103         if (request->rename < 0) {
1104                 ret = error("unable to write sha1 filename %s: %s",
1105                             request->filename,
1106                             strerror(request->rename));
1107                 release_request(request);
1108                 return ret;
1109         }
1110
1111         release_request(request);
1112         return 0;
1113 }
1114
1115 int fetch(unsigned char *sha1)
1116 {
1117         struct alt_base *altbase = alt;
1118
1119         if (!fetch_object(altbase, sha1))
1120                 return 0;
1121         while (altbase) {
1122                 if (!fetch_pack(altbase, sha1))
1123                         return 0;
1124                 altbase = altbase->next;
1125         }
1126         return error("Unable to find %s under %s\n", sha1_to_hex(sha1),
1127                      alt->base);
1128 }
1129
1130 static inline int needs_quote(int ch)
1131 {
1132         switch (ch) {
1133         case '/': case '-': case '.':
1134         case 'A'...'Z': case 'a'...'z': case '0'...'9':
1135                 return 0;
1136         default:
1137                 return 1;
1138         }
1139 }
1140
1141 static inline int hex(int v)
1142 {
1143         if (v < 10) return '0' + v;
1144         else return 'A' + v - 10;
1145 }
1146
1147 static char *quote_ref_url(const char *base, const char *ref)
1148 {
1149         const char *cp;
1150         char *dp, *qref;
1151         int len, baselen, ch;
1152
1153         baselen = strlen(base);
1154         len = baselen + 6; /* "refs/" + NUL */
1155         for (cp = ref; (ch = *cp) != 0; cp++, len++)
1156                 if (needs_quote(ch))
1157                         len += 2; /* extra two hex plus replacement % */
1158         qref = xmalloc(len);
1159         memcpy(qref, base, baselen);
1160         memcpy(qref + baselen, "refs/", 5);
1161         for (cp = ref, dp = qref + baselen + 5; (ch = *cp) != 0; cp++) {
1162                 if (needs_quote(ch)) {
1163                         *dp++ = '%';
1164                         *dp++ = hex((ch >> 4) & 0xF);
1165                         *dp++ = hex(ch & 0xF);
1166                 }
1167                 else
1168                         *dp++ = ch;
1169         }
1170         *dp = 0;
1171
1172         return qref;
1173 }
1174
1175 int fetch_ref(char *ref, unsigned char *sha1)
1176 {
1177         char *url;
1178         char hex[42];
1179         struct buffer buffer;
1180         char *base = alt->base;
1181         struct active_request_slot *slot;
1182         buffer.size = 41;
1183         buffer.posn = 0;
1184         buffer.buffer = hex;
1185         hex[41] = '\0';
1186
1187         url = quote_ref_url(base, ref);
1188         slot = get_active_slot();
1189         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
1190         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
1191         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
1192         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
1193         if (start_active_slot(slot)) {
1194                 run_active_slot(slot);
1195                 if (slot->curl_result != CURLE_OK)
1196                         return error("Couldn't get %s for %s\n%s",
1197                                      url, ref, curl_errorstr);
1198         } else {
1199                 return error("Unable to start request");
1200         }
1201
1202         hex[40] = '\0';
1203         get_sha1_hex(hex, sha1);
1204         return 0;
1205 }
1206
1207 int main(int argc, char **argv)
1208 {
1209         char *commit_id;
1210         char *url;
1211         int arg = 1;
1212         struct active_request_slot *slot;
1213         char *low_speed_limit;
1214         char *low_speed_time;
1215
1216         while (arg < argc && argv[arg][0] == '-') {
1217                 if (argv[arg][1] == 't') {
1218                         get_tree = 1;
1219                 } else if (argv[arg][1] == 'c') {
1220                         get_history = 1;
1221                 } else if (argv[arg][1] == 'a') {
1222                         get_all = 1;
1223                         get_tree = 1;
1224                         get_history = 1;
1225                 } else if (argv[arg][1] == 'v') {
1226                         get_verbosely = 1;
1227                 } else if (argv[arg][1] == 'w') {
1228                         write_ref = argv[arg + 1];
1229                         arg++;
1230                 } else if (!strcmp(argv[arg], "--recover")) {
1231                         get_recover = 1;
1232                 }
1233                 arg++;
1234         }
1235         if (argc < arg + 2) {
1236                 usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url");
1237                 return 1;
1238         }
1239         commit_id = argv[arg];
1240         url = argv[arg + 1];
1241
1242         curl_global_init(CURL_GLOBAL_ALL);
1243
1244 #ifdef USE_CURL_MULTI
1245         {
1246                 char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
1247                 if (http_max_requests != NULL)
1248                         max_requests = atoi(http_max_requests);
1249         }
1250
1251         curlm = curl_multi_init();
1252         if (curlm == NULL) {
1253                 fprintf(stderr, "Error creating curl multi handle.\n");
1254                 return 1;
1255         }
1256 #endif
1257
1258         if (getenv("GIT_SSL_NO_VERIFY"))
1259                 curl_ssl_verify = 0;
1260
1261         ssl_cert = getenv("GIT_SSL_CERT");
1262 #if LIBCURL_VERSION_NUM >= 0x070902
1263         ssl_key = getenv("GIT_SSL_KEY");
1264 #endif
1265 #if LIBCURL_VERSION_NUM >= 0x070908
1266         ssl_capath = getenv("GIT_SSL_CAPATH");
1267 #endif
1268         ssl_cainfo = getenv("GIT_SSL_CAINFO");
1269
1270         low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT");
1271         if (low_speed_limit != NULL)
1272                 curl_low_speed_limit = strtol(low_speed_limit, NULL, 10);
1273         low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
1274         if (low_speed_time != NULL)
1275                 curl_low_speed_time = strtol(low_speed_time, NULL, 10);
1276
1277         git_config(http_options);
1278
1279         if (curl_ssl_verify == -1)
1280                 curl_ssl_verify = 1;
1281
1282 #ifdef USE_CURL_MULTI
1283         if (max_requests < 1)
1284                 max_requests = DEFAULT_MAX_REQUESTS;
1285 #endif
1286
1287         pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache");
1288         no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
1289         no_range_header = curl_slist_append(no_range_header, "Range:");
1290
1291 #ifndef NO_CURL_EASY_DUPHANDLE
1292         curl_default = get_curl_handle();
1293 #endif
1294
1295         alt = xmalloc(sizeof(*alt));
1296         alt->base = url;
1297         alt->got_indices = 0;
1298         alt->packs = NULL;
1299         alt->next = NULL;
1300         fetch_alternates(alt->base);
1301
1302         if (pull(commit_id))
1303                 return 1;
1304
1305         curl_slist_free_all(pragma_header);
1306         curl_slist_free_all(no_pragma_header);
1307         curl_slist_free_all(no_range_header);
1308 #ifndef NO_CURL_EASY_DUPHANDLE
1309         curl_easy_cleanup(curl_default);
1310 #endif
1311         slot = active_queue_head;
1312         while (slot != NULL) {
1313                 if (slot->curl != NULL)
1314                         curl_easy_cleanup(slot->curl);
1315                 slot = slot->next;
1316         }
1317 #ifdef USE_CURL_MULTI
1318         curl_multi_cleanup(curlm);
1319 #endif
1320         curl_global_cleanup();
1321         return 0;
1322 }