d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fetch(1): process truncated transfer as soft failure

Browse Source 
Let "fetch -a" resume truncated transfer automatically
perform another attempt if it obtained some new data in previous one
making progress.

This makes it more robust against frequent but transient network failures.
For example:

=> sqlite-src-3370200.zip doesn't seem to exist in /usr/ports/distfiles/.
=> Attempting to fetch https://www.sqlite.org/2022/sqlite-src-3370200.zip
sqlite-src-3370200.zip                          3% of   12 MB   45 kBps 04m24s
fetch: sqlite-src-3370200.zip appears to be truncated: 524288/13145234 bytes
sqlite-src-3370200.zip                         10% of   12 MB   67 kBps 02m56s
fetch: sqlite-src-3370200.zip appears to be truncated: 1327104/13145234 bytes
sqlite-src-3370200.zip                         28% of   12 MB  123 kBps 01m14s
fetch: sqlite-src-3370200.zip appears to be truncated: 3735552/13145234 bytes
sqlite-src-3370200.zip                         54% of   12 MB  253 kBps    24s
fetch: sqlite-src-3370200.zip appears to be truncated: 7176192/13145234 bytes
sqlite-src-3370200.zip                         62% of   12 MB   90 kBps    55s
fetch: sqlite-src-3370200.zip appears to be truncated: 8241152/13145234 bytes
sqlite-src-3370200.zip                         82% of   12 MB  113 kBps    20s
fetch: sqlite-src-3370200.zip appears to be truncated: 10862592/13145234 bytes
sqlite-src-3370200.zip                                  12 MB  185 kBps    12s
===> Fetching all distfiles required by sqlite3-3.37.2,1 for building

MFC after:	1 month
This commit is contained in:
Eugene Grosbein 2022-01-24 11:03:42 +07:00
parent f72926eab0
commit e3bad5f7aa
1 changed files with 17 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
usr.bin/fetch/fetch.c
View File

@ -433,11 +433,11 @@ fetch(char *URL, const char *path)
struct xferstat xs;
FILE *f, *of;
size_t size, readcnt, wr;
off_t count;
off_t count, size_prev;
char flags[8];
const char *slash;
char *tmppath;
int r;
int r, tries;
unsigned timeout;
char *ptr;
@ -537,6 +537,9 @@ fetch(char *URL, const char *path)
goto success;
}
tries = 1;
again:
r = 0;
/*
* If the -r flag was specified, we have to compare the local
* and remote files, so we should really do a fetchStat()
@ -553,7 +556,7 @@ fetch(char *URL, const char *path)
sb.st_size = -1;
if (!o_stdout) {
r = stat(path, &sb);
if (r == 0 && r_flag && S_ISREG(sb.st_mode)) {
if (r == 0 && (r_flag || tries > 1) && S_ISREG(sb.st_mode)) {
url->offset = sb.st_size;
} else if (r == -1 || !S_ISREG(sb.st_mode)) {
/*
@ -568,6 +571,7 @@ fetch(char *URL, const char *path)
goto failure;
}
}
size_prev = sb.st_size;
/* start the transfer */
if (timeout)
@ -629,7 +633,7 @@ fetch(char *URL, const char *path)
of = stdout;
} else if (r_flag && sb.st_size != -1) {
/* resume mode, local file exists */
if (!F_flag && us.mtime && sb.st_mtime != us.mtime) {
if (!F_flag && us.mtime && sb.st_mtime != us.mtime && tries == 1) {
/* no match! have to refetch */
fclose(f);
/* if precious, warn the user and give up */
@ -717,6 +721,8 @@ fetch(char *URL, const char *path)
slash = path;
else
++slash;
if(tmppath != NULL)
free(tmppath);
asprintf(&tmppath, "%.*s.fetch.XXXXXX.%s",
(int)(slash - path), path, slash);
if (tmppath != NULL) {
@ -829,6 +835,13 @@ fetch(char *URL, const char *path)
if (us.size != -1 && count < us.size) {
warnx("%s appears to be truncated: %jd/%jd bytes",
path, (intmax_t)count, (intmax_t)us.size);
if(!o_stdout && a_flag && us.size > size_prev) {
fclose(f);
if (w_secs)
sleep(w_secs);
tries++;
goto again;
}
goto failure_keep;
}