9 #if LIBCURL_VERSION_NUM >= 0x070908
10 #define USE_CURL_MULTI
11 #define DEFAULT_MAX_REQUESTS 5
14 #if LIBCURL_VERSION_NUM < 0x070704
15 #define curl_global_cleanup() do { /* nothing */ } while(0)
17 #if LIBCURL_VERSION_NUM < 0x070800
18 #define curl_global_init(a) do { /* nothing */ } while(0)
21 #if LIBCURL_VERSION_NUM < 0x070c04
22 #define NO_CURL_EASY_DUPHANDLE
25 #define PREV_BUF_SIZE 4096
26 #define RANGE_HEADER_SIZE 30
28 static int got_alternates = -1;
29 static int active_requests = 0;
30 static int data_received;
33 static int max_requests = -1;
36 #ifndef NO_CURL_EASY_DUPHANDLE
37 static CURL *curl_default;
39 static struct curl_slist *pragma_header;
40 static struct curl_slist *no_pragma_header;
41 static struct curl_slist *no_range_header;
42 static char curl_errorstr[CURL_ERROR_SIZE];
48 struct packed_git *packs;
49 struct alt_base *next;
52 static struct alt_base *alt = NULL;
61 struct transfer_request
63 unsigned char sha1[20];
64 struct alt_base *repo;
66 char filename[PATH_MAX];
67 char tmpfile[PATH_MAX];
69 enum transfer_state state;
71 char errorstr[CURL_ERROR_SIZE];
73 unsigned char real_sha1[20];
78 struct active_request_slot *slot;
79 struct transfer_request *next;
82 struct active_request_slot
91 void (*callback_func)(void *data);
92 struct active_request_slot *next;
98 struct buffer *buffer;
99 struct active_request_slot *slot;
103 static struct transfer_request *request_queue_head = NULL;
104 static struct active_request_slot *active_queue_head = NULL;
106 static int curl_ssl_verify = -1;
107 static char *ssl_cert = NULL;
108 #if LIBCURL_VERSION_NUM >= 0x070902
109 static char *ssl_key = NULL;
111 #if LIBCURL_VERSION_NUM >= 0x070908
112 static char *ssl_capath = NULL;
114 static char *ssl_cainfo = NULL;
115 static long curl_low_speed_limit = -1;
116 static long curl_low_speed_time = -1;
125 static int http_options(const char *var, const char *value)
127 if (!strcmp("http.sslverify", var)) {
128 if (curl_ssl_verify == -1) {
129 curl_ssl_verify = git_config_bool(var, value);
134 if (!strcmp("http.sslcert", var)) {
135 if (ssl_cert == NULL) {
136 ssl_cert = xmalloc(strlen(value)+1);
137 strcpy(ssl_cert, value);
141 #if LIBCURL_VERSION_NUM >= 0x070902
142 if (!strcmp("http.sslkey", var)) {
143 if (ssl_key == NULL) {
144 ssl_key = xmalloc(strlen(value)+1);
145 strcpy(ssl_key, value);
150 #if LIBCURL_VERSION_NUM >= 0x070908
151 if (!strcmp("http.sslcapath", var)) {
152 if (ssl_capath == NULL) {
153 ssl_capath = xmalloc(strlen(value)+1);
154 strcpy(ssl_capath, value);
159 if (!strcmp("http.sslcainfo", var)) {
160 if (ssl_cainfo == NULL) {
161 ssl_cainfo = xmalloc(strlen(value)+1);
162 strcpy(ssl_cainfo, value);
167 #ifdef USE_CURL_MULTI
168 if (!strcmp("http.maxrequests", var)) {
169 if (max_requests == -1)
170 max_requests = git_config_int(var, value);
175 if (!strcmp("http.lowspeedlimit", var)) {
176 if (curl_low_speed_limit == -1)
177 curl_low_speed_limit = (long)git_config_int(var, value);
180 if (!strcmp("http.lowspeedtime", var)) {
181 if (curl_low_speed_time == -1)
182 curl_low_speed_time = (long)git_config_int(var, value);
186 /* Fall back on the default ones */
187 return git_default_config(var, value);
190 static size_t fwrite_buffer(void *ptr, size_t eltsize, size_t nmemb,
191 struct buffer *buffer)
193 size_t size = eltsize * nmemb;
194 if (size > buffer->size - buffer->posn)
195 size = buffer->size - buffer->posn;
196 memcpy(buffer->buffer + buffer->posn, ptr, size);
197 buffer->posn += size;
202 static size_t fwrite_buffer_dynamic(const void *ptr, size_t eltsize,
203 size_t nmemb, struct buffer *buffer)
205 size_t size = eltsize * nmemb;
206 if (size > buffer->size - buffer->posn) {
207 buffer->size = buffer->size * 3 / 2;
208 if (buffer->size < buffer->posn + size)
209 buffer->size = buffer->posn + size;
210 buffer->buffer = xrealloc(buffer->buffer, buffer->size);
212 memcpy(buffer->buffer + buffer->posn, ptr, size);
213 buffer->posn += size;
218 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
221 unsigned char expn[4096];
222 size_t size = eltsize * nmemb;
224 struct transfer_request *request = (struct transfer_request *)data;
226 ssize_t retval = write(request->local,
227 ptr + posn, size - posn);
231 } while (posn < size);
233 request->stream.avail_in = size;
234 request->stream.next_in = ptr;
236 request->stream.next_out = expn;
237 request->stream.avail_out = sizeof(expn);
238 request->zret = inflate(&request->stream, Z_SYNC_FLUSH);
239 SHA1_Update(&request->c, expn,
240 sizeof(expn) - request->stream.avail_out);
241 } while (request->stream.avail_in && request->zret == Z_OK);
246 #ifdef USE_CURL_MULTI
247 static void process_curl_messages(void);
248 static void process_request_queue(void);
250 static void fetch_alternates(char *base);
252 static CURL* get_curl_handle(void)
254 CURL* result = curl_easy_init();
256 curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify);
257 #if LIBCURL_VERSION_NUM >= 0x070907
258 curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
261 if (ssl_cert != NULL)
262 curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
263 #if LIBCURL_VERSION_NUM >= 0x070902
265 curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
267 #if LIBCURL_VERSION_NUM >= 0x070908
268 if (ssl_capath != NULL)
269 curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
271 if (ssl_cainfo != NULL)
272 curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
273 curl_easy_setopt(result, CURLOPT_FAILONERROR, 1);
275 if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
276 curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
277 curl_low_speed_limit);
278 curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
279 curl_low_speed_time);
282 curl_easy_setopt(result, CURLOPT_FOLLOWLOCATION, 1);
287 static struct active_request_slot *get_active_slot(void)
289 struct active_request_slot *slot = active_queue_head;
290 struct active_request_slot *newslot;
292 #ifdef USE_CURL_MULTI
295 /* Wait for a slot to open up if the queue is full */
296 while (active_requests >= max_requests) {
297 curl_multi_perform(curlm, &num_transfers);
298 if (num_transfers < active_requests) {
299 process_curl_messages();
304 while (slot != NULL && slot->in_use) {
308 newslot = xmalloc(sizeof(*newslot));
309 newslot->curl = NULL;
311 newslot->next = NULL;
313 slot = active_queue_head;
315 active_queue_head = newslot;
317 while (slot->next != NULL) {
320 slot->next = newslot;
325 if (slot->curl == NULL) {
326 #ifdef NO_CURL_EASY_DUPHANDLE
327 slot->curl = get_curl_handle();
329 slot->curl = curl_easy_duphandle(curl_default);
337 slot->callback_data = NULL;
338 slot->callback_func = NULL;
339 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
340 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_range_header);
341 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
346 static int start_active_slot(struct active_request_slot *slot)
348 #ifdef USE_CURL_MULTI
349 CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
351 if (curlm_result != CURLM_OK &&
352 curlm_result != CURLM_CALL_MULTI_PERFORM) {
361 static void run_active_slot(struct active_request_slot *slot)
363 #ifdef USE_CURL_MULTI
371 struct timeval select_timeout;
372 CURLMcode curlm_result;
374 while (!slot->done) {
377 curlm_result = curl_multi_perform(curlm,
379 } while (curlm_result == CURLM_CALL_MULTI_PERFORM);
380 if (num_transfers < active_requests) {
381 process_curl_messages();
382 process_request_queue();
385 if (!data_received && slot->local != NULL) {
386 current_pos = ftell(slot->local);
387 if (current_pos > last_pos)
389 last_pos = current_pos;
392 if (!slot->done && !data_received) {
397 select_timeout.tv_sec = 0;
398 select_timeout.tv_usec = 50000;
399 select(max_fd, &readfds, &writefds,
400 &excfds, &select_timeout);
404 slot->curl_result = curl_easy_perform(slot->curl);
409 static void start_request(struct transfer_request *request)
411 char *hex = sha1_to_hex(request->sha1);
412 char prevfile[PATH_MAX];
416 unsigned char prev_buf[PREV_BUF_SIZE];
417 ssize_t prev_read = 0;
419 char range[RANGE_HEADER_SIZE];
420 struct curl_slist *range_header = NULL;
421 struct active_request_slot *slot;
423 snprintf(prevfile, sizeof(prevfile), "%s.prev", request->filename);
425 rename(request->tmpfile, prevfile);
426 unlink(request->tmpfile);
428 request->local = open(request->tmpfile,
429 O_WRONLY | O_CREAT | O_EXCL, 0666);
430 /* This could have failed due to the "lazy directory creation";
431 * try to mkdir the last path component.
433 if (request->local < 0 && errno == ENOENT) {
434 char *dir = strrchr(request->tmpfile, '/');
437 mkdir(request->tmpfile, 0777);
440 request->local = open(request->tmpfile,
441 O_WRONLY | O_CREAT | O_EXCL, 0666);
444 if (request->local < 0) {
445 request->state = ABORTED;
446 error("Couldn't create temporary file %s for %s: %s\n",
447 request->tmpfile, request->filename, strerror(errno));
451 memset(&request->stream, 0, sizeof(request->stream));
453 inflateInit(&request->stream);
455 SHA1_Init(&request->c);
457 url = xmalloc(strlen(request->repo->base) + 50);
458 request->url = xmalloc(strlen(request->repo->base) + 50);
459 strcpy(url, request->repo->base);
460 posn = url + strlen(request->repo->base);
461 strcpy(posn, "objects/");
463 memcpy(posn, hex, 2);
466 strcpy(posn, hex + 2);
467 strcpy(request->url, url);
469 /* If a previous temp file is present, process what was already
471 prevlocal = open(prevfile, O_RDONLY);
472 if (prevlocal != -1) {
474 prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE);
476 if (fwrite_sha1_file(prev_buf,
479 request) == prev_read) {
480 prev_posn += prev_read;
485 } while (prev_read > 0);
490 /* Reset inflate/SHA1 if there was an error reading the previous temp
491 file; also rewind to the beginning of the local file. */
492 if (prev_read == -1) {
493 memset(&request->stream, 0, sizeof(request->stream));
494 inflateInit(&request->stream);
495 SHA1_Init(&request->c);
498 lseek(request->local, SEEK_SET, 0);
499 ftruncate(request->local, 0);
503 slot = get_active_slot();
504 curl_easy_setopt(slot->curl, CURLOPT_FILE, request);
505 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
506 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr);
507 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
508 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
510 /* If we have successfully processed data from a previous fetch
511 attempt, only fetch the data we don't already have. */
515 "Resuming fetch of object %s at byte %ld\n",
517 sprintf(range, "Range: bytes=%ld-", prev_posn);
518 range_header = curl_slist_append(range_header, range);
519 curl_easy_setopt(slot->curl,
520 CURLOPT_HTTPHEADER, range_header);
523 /* Try to get the request started, abort the request on error */
524 if (!start_active_slot(slot)) {
525 request->state = ABORTED;
526 close(request->local);
531 request->slot = slot;
532 request->state = ACTIVE;
535 static void finish_request(struct transfer_request *request)
539 fchmod(request->local, 0444);
540 close(request->local);
542 if (request->http_code == 416) {
543 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
544 } else if (request->curl_result != CURLE_OK) {
545 if (stat(request->tmpfile, &st) == 0)
547 unlink(request->tmpfile);
551 inflateEnd(&request->stream);
552 SHA1_Final(request->real_sha1, &request->c);
553 if (request->zret != Z_STREAM_END) {
554 unlink(request->tmpfile);
557 if (memcmp(request->sha1, request->real_sha1, 20)) {
558 unlink(request->tmpfile);
562 move_temp_to_file(request->tmpfile, request->filename);
564 if (request->rename == 0)
565 pull_say("got %s\n", sha1_to_hex(request->sha1));
568 static void release_request(struct transfer_request *request)
570 struct transfer_request *entry = request_queue_head;
572 if (request == request_queue_head) {
573 request_queue_head = request->next;
575 while (entry->next != NULL && entry->next != request)
577 if (entry->next == request)
578 entry->next = entry->next->next;
585 #ifdef USE_CURL_MULTI
586 static void process_curl_messages(void)
589 struct active_request_slot *slot;
590 struct transfer_request *request = NULL;
591 CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
593 while (curl_message != NULL) {
594 if (curl_message->msg == CURLMSG_DONE) {
595 int curl_result = curl_message->data.result;
596 slot = active_queue_head;
597 while (slot != NULL &&
598 slot->curl != curl_message->easy_handle)
601 curl_multi_remove_handle(curlm, slot->curl);
605 slot->curl_result = curl_result;
606 curl_easy_getinfo(slot->curl,
609 request = request_queue_head;
610 while (request != NULL &&
611 request->slot != slot)
612 request = request->next;
614 fprintf(stderr, "Received DONE message for unknown request!\n");
617 /* Process slot callback if appropriate */
618 if (slot->callback_func != NULL) {
619 slot->callback_func(slot->callback_data);
622 if (request != NULL) {
623 request->curl_result = curl_result;
624 request->http_code = slot->http_code;
625 request->slot = NULL;
626 request->state = COMPLETE;
628 /* Use alternates if necessary */
629 if (request->http_code == 404) {
630 fetch_alternates(alt->base);
631 if (request->repo->next != NULL) {
634 start_request(request);
636 finish_request(request);
639 finish_request(request);
643 fprintf(stderr, "Unknown CURL message received: %d\n",
644 (int)curl_message->msg);
646 curl_message = curl_multi_info_read(curlm, &num_messages);
650 static void process_request_queue(void)
652 struct transfer_request *request = request_queue_head;
653 struct active_request_slot *slot = active_queue_head;
656 while (active_requests < max_requests && request != NULL) {
657 if (request->state == WAITING) {
658 if (has_sha1_file(request->sha1))
659 release_request(request);
661 start_request(request);
662 curl_multi_perform(curlm, &num_transfers);
664 request = request->next;
667 while (slot != NULL) {
668 if (!slot->in_use && slot->curl != NULL) {
669 curl_easy_cleanup(slot->curl);
677 void prefetch(unsigned char *sha1)
679 struct transfer_request *newreq;
680 struct transfer_request *tail;
681 char *filename = sha1_file_name(sha1);
683 newreq = xmalloc(sizeof(*newreq));
684 memcpy(newreq->sha1, sha1, 20);
688 newreq->state = WAITING;
689 snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
690 snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
691 "%s.temp", filename);
694 if (request_queue_head == NULL) {
695 request_queue_head = newreq;
697 tail = request_queue_head;
698 while (tail->next != NULL) {
703 #ifdef USE_CURL_MULTI
704 process_request_queue();
705 process_curl_messages();
709 static int fetch_index(struct alt_base *repo, unsigned char *sha1)
711 char *hex = sha1_to_hex(sha1);
714 char tmpfile[PATH_MAX];
716 char range[RANGE_HEADER_SIZE];
717 struct curl_slist *range_header = NULL;
720 struct active_request_slot *slot;
722 if (has_pack_index(sha1))
726 fprintf(stderr, "Getting index for pack %s\n", hex);
728 url = xmalloc(strlen(repo->base) + 64);
729 sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
731 filename = sha1_pack_index_name(sha1);
732 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
733 indexfile = fopen(tmpfile, "a");
735 return error("Unable to open local file %s for pack index",
738 slot = get_active_slot();
739 curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
740 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
741 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
742 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
743 slot->local = indexfile;
745 /* If there is data present from a previous transfer attempt,
746 resume where it left off */
747 prev_posn = ftell(indexfile);
751 "Resuming fetch of index for pack %s at byte %ld\n",
753 sprintf(range, "Range: bytes=%ld-", prev_posn);
754 range_header = curl_slist_append(range_header, range);
755 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
758 if (start_active_slot(slot)) {
759 run_active_slot(slot);
760 if (slot->curl_result != CURLE_OK) {
762 return error("Unable to get pack index %s\n%s", url,
766 return error("Unable to start request");
771 return move_temp_to_file(tmpfile, filename);
774 static int setup_index(struct alt_base *repo, unsigned char *sha1)
776 struct packed_git *new_pack;
777 if (has_pack_file(sha1))
778 return 0; // don't list this as something we can get
780 if (fetch_index(repo, sha1))
783 new_pack = parse_pack_index(sha1);
784 new_pack->next = repo->packs;
785 repo->packs = new_pack;
789 static void process_alternates(void *callback_data)
791 struct alt_request *alt_req = (struct alt_request *)callback_data;
792 struct active_request_slot *slot = alt_req->slot;
793 struct alt_base *tail = alt;
794 char *base = alt_req->base;
795 static const char null_byte = '\0';
799 if (alt_req->http_specific) {
800 if (slot->curl_result != CURLE_OK ||
801 !alt_req->buffer->posn) {
803 /* Try reusing the slot to get non-http alternates */
804 alt_req->http_specific = 0;
805 sprintf(alt_req->url, "%s/objects/info/alternates",
807 curl_easy_setopt(slot->curl, CURLOPT_URL,
812 if (start_active_slot(slot)) {
820 } else if (slot->curl_result != CURLE_OK) {
821 if (slot->http_code != 404) {
827 fwrite_buffer_dynamic(&null_byte, 1, 1, alt_req->buffer);
828 alt_req->buffer->posn--;
829 data = alt_req->buffer->buffer;
831 while (i < alt_req->buffer->posn) {
833 while (posn < alt_req->buffer->posn && data[posn] != '\n')
835 if (data[posn] == '\n') {
838 struct alt_base *newalt;
840 if (data[i] == '/') {
841 serverlen = strchr(base + 8, '/') - base;
843 } else if (!memcmp(data + i, "../", 3)) {
845 serverlen = strlen(base);
846 while (i + 2 < posn &&
847 !memcmp(data + i, "../", 3)) {
850 } while (serverlen &&
851 base[serverlen - 1] != '/');
854 // If the server got removed, give up.
855 okay = strchr(base, ':') - base + 3 <
857 } else if (alt_req->http_specific) {
858 char *colon = strchr(data + i, ':');
859 char *slash = strchr(data + i, '/');
860 if (colon && slash && colon < data + posn &&
861 slash < data + posn && colon < slash) {
865 // skip 'objects' at end
867 target = xmalloc(serverlen + posn - i - 6);
868 strncpy(target, base, serverlen);
869 strncpy(target + serverlen, data + i,
871 target[serverlen + posn - i - 7] = '\0';
874 "Also look at %s\n", target);
875 newalt = xmalloc(sizeof(*newalt));
877 newalt->base = target;
878 newalt->got_indices = 0;
879 newalt->packs = NULL;
880 while (tail->next != NULL)
891 static void fetch_alternates(char *base)
893 struct buffer buffer;
896 struct active_request_slot *slot;
897 static struct alt_request alt_req;
900 /* If another request has already started fetching alternates,
901 wait for them to arrive and return to processing this request's
903 while (got_alternates == 0) {
904 curl_multi_perform(curlm, &num_transfers);
905 process_curl_messages();
906 process_request_queue();
909 /* Nothing to do if they've already been fetched */
910 if (got_alternates == 1)
913 /* Start the fetch */
916 data = xmalloc(4096);
919 buffer.buffer = data;
922 fprintf(stderr, "Getting alternates list for %s\n", base);
924 url = xmalloc(strlen(base) + 31);
925 sprintf(url, "%s/objects/info/http-alternates", base);
927 /* Use a callback to process the result, since another request
928 may fail and need to have alternates loaded before continuing */
929 slot = get_active_slot();
930 slot->callback_func = process_alternates;
931 slot->callback_data = &alt_req;
933 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
934 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
935 fwrite_buffer_dynamic);
936 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
940 alt_req.buffer = &buffer;
941 alt_req.http_specific = 1;
944 if (start_active_slot(slot))
945 run_active_slot(slot);
953 static int fetch_indices(struct alt_base *repo)
955 unsigned char sha1[20];
957 struct buffer buffer;
961 struct active_request_slot *slot;
963 if (repo->got_indices)
966 data = xmalloc(4096);
969 buffer.buffer = data;
972 fprintf(stderr, "Getting pack list for %s\n", repo->base);
974 url = xmalloc(strlen(repo->base) + 21);
975 sprintf(url, "%s/objects/info/packs", repo->base);
977 slot = get_active_slot();
978 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
979 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
980 fwrite_buffer_dynamic);
981 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
982 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
983 if (start_active_slot(slot)) {
984 run_active_slot(slot);
985 if (slot->curl_result != CURLE_OK) {
987 return error("%s", curl_errorstr);
991 return error("Unable to start request");
994 data = buffer.buffer;
995 while (i < buffer.posn) {
999 if (i + 52 < buffer.posn &&
1000 !strncmp(data + i, " pack-", 6) &&
1001 !strncmp(data + i + 46, ".pack\n", 6)) {
1002 get_sha1_hex(data + i + 6, sha1);
1003 setup_index(repo, sha1);
1008 while (data[i] != '\n')
1014 free(buffer.buffer);
1015 repo->got_indices = 1;
1019 static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
1022 struct packed_git *target;
1023 struct packed_git **lst;
1026 char tmpfile[PATH_MAX];
1029 char range[RANGE_HEADER_SIZE];
1030 struct curl_slist *range_header = NULL;
1032 struct active_request_slot *slot;
1034 if (fetch_indices(repo))
1036 target = find_sha1_pack(sha1, repo->packs);
1040 if (get_verbosely) {
1041 fprintf(stderr, "Getting pack %s\n",
1042 sha1_to_hex(target->sha1));
1043 fprintf(stderr, " which contains %s\n",
1047 url = xmalloc(strlen(repo->base) + 65);
1048 sprintf(url, "%s/objects/pack/pack-%s.pack",
1049 repo->base, sha1_to_hex(target->sha1));
1051 filename = sha1_pack_name(target->sha1);
1052 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
1053 packfile = fopen(tmpfile, "a");
1055 return error("Unable to open local file %s for pack",
1058 slot = get_active_slot();
1059 curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
1060 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
1061 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
1062 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1063 slot->local = packfile;
1065 /* If there is data present from a previous transfer attempt,
1066 resume where it left off */
1067 prev_posn = ftell(packfile);
1071 "Resuming fetch of pack %s at byte %ld\n",
1072 sha1_to_hex(target->sha1), prev_posn);
1073 sprintf(range, "Range: bytes=%ld-", prev_posn);
1074 range_header = curl_slist_append(range_header, range);
1075 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
1078 if (start_active_slot(slot)) {
1079 run_active_slot(slot);
1080 if (slot->curl_result != CURLE_OK) {
1082 return error("Unable to get pack file %s\n%s", url,
1086 return error("Unable to start request");
1091 ret = move_temp_to_file(tmpfile, filename);
1096 while (*lst != target)
1097 lst = &((*lst)->next);
1098 *lst = (*lst)->next;
1100 if (verify_pack(target, 0))
1102 install_packed_git(target);
1107 static int fetch_object(struct alt_base *repo, unsigned char *sha1)
1109 char *hex = sha1_to_hex(sha1);
1111 struct transfer_request *request = request_queue_head;
1113 while (request != NULL && memcmp(request->sha1, sha1, 20))
1114 request = request->next;
1115 if (request == NULL)
1116 return error("Couldn't find request for %s in the queue", hex);
1118 if (has_sha1_file(request->sha1)) {
1119 release_request(request);
1123 #ifdef USE_CURL_MULTI
1124 while (request->state == WAITING) {
1126 curl_multi_perform(curlm, &num_transfers);
1127 if (num_transfers < active_requests) {
1128 process_curl_messages();
1129 process_request_queue();
1133 start_request(request);
1136 while (request->state == ACTIVE) {
1137 run_active_slot(request->slot);
1138 #ifndef USE_CURL_MULTI
1139 request->curl_result = request->slot->curl_result;
1140 request->http_code = request->slot->http_code;
1141 request->slot = NULL;
1143 /* Use alternates if necessary */
1144 if (request->http_code == 404) {
1145 fetch_alternates(alt->base);
1146 if (request->repo->next != NULL) {
1147 request->repo = request->repo->next;
1148 start_request(request);
1151 finish_request(request);
1152 request->state = COMPLETE;
1157 if (request->state == ABORTED) {
1158 release_request(request);
1159 return error("Request for %s aborted", hex);
1162 if (request->curl_result != CURLE_OK && request->http_code != 416) {
1163 if (request->http_code == 404)
1164 ret = -1; /* Be silent, it is probably in a pack. */
1166 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
1167 request->errorstr, request->curl_result,
1168 request->http_code, hex);
1169 release_request(request);
1173 if (request->zret != Z_STREAM_END) {
1174 ret = error("File %s (%s) corrupt\n", hex, request->url);
1175 release_request(request);
1179 if (memcmp(request->sha1, request->real_sha1, 20)) {
1180 release_request(request);
1181 return error("File %s has bad hash\n", hex);
1184 if (request->rename < 0) {
1185 ret = error("unable to write sha1 filename %s: %s",
1187 strerror(request->rename));
1188 release_request(request);
1192 release_request(request);
1196 int fetch(unsigned char *sha1)
1198 struct alt_base *altbase = alt;
1200 if (!fetch_object(altbase, sha1))
1203 if (!fetch_pack(altbase, sha1))
1205 fetch_alternates(alt->base);
1206 altbase = altbase->next;
1208 return error("Unable to find %s under %s\n", sha1_to_hex(sha1),
1212 static inline int needs_quote(int ch)
1215 case '/': case '-': case '.':
1216 case 'A'...'Z': case 'a'...'z': case '0'...'9':
1223 static inline int hex(int v)
1225 if (v < 10) return '0' + v;
1226 else return 'A' + v - 10;
1229 static char *quote_ref_url(const char *base, const char *ref)
1233 int len, baselen, ch;
1235 baselen = strlen(base);
1236 len = baselen + 6; /* "refs/" + NUL */
1237 for (cp = ref; (ch = *cp) != 0; cp++, len++)
1238 if (needs_quote(ch))
1239 len += 2; /* extra two hex plus replacement % */
1240 qref = xmalloc(len);
1241 memcpy(qref, base, baselen);
1242 memcpy(qref + baselen, "refs/", 5);
1243 for (cp = ref, dp = qref + baselen + 5; (ch = *cp) != 0; cp++) {
1244 if (needs_quote(ch)) {
1246 *dp++ = hex((ch >> 4) & 0xF);
1247 *dp++ = hex(ch & 0xF);
1257 int fetch_ref(char *ref, unsigned char *sha1)
1261 struct buffer buffer;
1262 char *base = alt->base;
1263 struct active_request_slot *slot;
1266 buffer.buffer = hex;
1269 url = quote_ref_url(base, ref);
1270 slot = get_active_slot();
1271 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
1272 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
1273 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
1274 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
1275 if (start_active_slot(slot)) {
1276 run_active_slot(slot);
1277 if (slot->curl_result != CURLE_OK)
1278 return error("Couldn't get %s for %s\n%s",
1279 url, ref, curl_errorstr);
1281 return error("Unable to start request");
1285 get_sha1_hex(hex, sha1);
1289 int main(int argc, char **argv)
1294 struct active_request_slot *slot;
1295 char *low_speed_limit;
1296 char *low_speed_time;
1300 while (arg < argc && argv[arg][0] == '-') {
1301 if (argv[arg][1] == 't') {
1303 } else if (argv[arg][1] == 'c') {
1305 } else if (argv[arg][1] == 'a') {
1309 } else if (argv[arg][1] == 'v') {
1311 } else if (argv[arg][1] == 'w') {
1312 write_ref = argv[arg + 1];
1314 } else if (!strcmp(argv[arg], "--recover")) {
1319 if (argc < arg + 2) {
1320 usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url");
1323 commit_id = argv[arg];
1324 url = argv[arg + 1];
1326 curl_global_init(CURL_GLOBAL_ALL);
1328 #ifdef USE_CURL_MULTI
1330 char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
1331 if (http_max_requests != NULL)
1332 max_requests = atoi(http_max_requests);
1335 curlm = curl_multi_init();
1336 if (curlm == NULL) {
1337 fprintf(stderr, "Error creating curl multi handle.\n");
1342 if (getenv("GIT_SSL_NO_VERIFY"))
1343 curl_ssl_verify = 0;
1345 ssl_cert = getenv("GIT_SSL_CERT");
1346 #if LIBCURL_VERSION_NUM >= 0x070902
1347 ssl_key = getenv("GIT_SSL_KEY");
1349 #if LIBCURL_VERSION_NUM >= 0x070908
1350 ssl_capath = getenv("GIT_SSL_CAPATH");
1352 ssl_cainfo = getenv("GIT_SSL_CAINFO");
1354 low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT");
1355 if (low_speed_limit != NULL)
1356 curl_low_speed_limit = strtol(low_speed_limit, NULL, 10);
1357 low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
1358 if (low_speed_time != NULL)
1359 curl_low_speed_time = strtol(low_speed_time, NULL, 10);
1361 git_config(http_options);
1363 if (curl_ssl_verify == -1)
1364 curl_ssl_verify = 1;
1366 #ifdef USE_CURL_MULTI
1367 if (max_requests < 1)
1368 max_requests = DEFAULT_MAX_REQUESTS;
1371 pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache");
1372 no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
1373 no_range_header = curl_slist_append(no_range_header, "Range:");
1375 #ifndef NO_CURL_EASY_DUPHANDLE
1376 curl_default = get_curl_handle();
1379 alt = xmalloc(sizeof(*alt));
1381 alt->got_indices = 0;
1385 if (pull(commit_id))
1388 curl_slist_free_all(pragma_header);
1389 curl_slist_free_all(no_pragma_header);
1390 curl_slist_free_all(no_range_header);
1391 #ifndef NO_CURL_EASY_DUPHANDLE
1392 curl_easy_cleanup(curl_default);
1394 slot = active_queue_head;
1395 while (slot != NULL) {
1397 if (get_verbosely) {
1398 curl_easy_getinfo(slot->curl,
1399 CURLINFO_EFFECTIVE_URL,
1401 fprintf(stderr, "Waiting for %s\n", wait_url);
1403 run_active_slot(slot);
1405 if (slot->curl != NULL)
1406 curl_easy_cleanup(slot->curl);
1409 #ifdef USE_CURL_MULTI
1410 curl_multi_cleanup(curlm);
1412 curl_global_cleanup();