Add support for HTTP 1.1 If-Modified-Since behavior.
fetch(1) accepts a new argument -i <file> that if specified will cause the file to be downloaded only if it is more recent than the mtime of <file>. libfetch(3) accepts the mtime in the url structure and a flag to indicate when this behavior is desired. PR: bin/87841 Submitted by: Jukka A. Ukkonen <jau@iki.fi> (partially) Reviewed by: des, ru MFC after: 3 weeks
This commit is contained in:
parent
0a420d08ae
commit
7f92799f67
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=186124
@ -25,7 +25,7 @@
|
||||
.\"
|
||||
.\" $FreeBSD$
|
||||
.\"
|
||||
.Dd December 18, 2007
|
||||
.Dd December 14, 2008
|
||||
.Dt FETCH 3
|
||||
.Os
|
||||
.Sh NAME
|
||||
@ -165,9 +165,16 @@ struct url {
|
||||
char *doc;
|
||||
off_t offset;
|
||||
size_t length;
|
||||
time_t ims_time;
|
||||
};
|
||||
.Ed
|
||||
.Pp
|
||||
The
|
||||
.Va ims_time
|
||||
field stores the time value for
|
||||
.Li If-Modified-Since
|
||||
HTTP requests.
|
||||
.Pp
|
||||
The pointer returned by
|
||||
.Fn fetchMakeURL
|
||||
or
|
||||
@ -353,6 +360,22 @@ and
|
||||
.Fn fetchPutHTTP
|
||||
will use a direct connection even if a proxy server is defined.
|
||||
.Pp
|
||||
If the
|
||||
.Ql i
|
||||
(if-modified-since) flag is specified, and
|
||||
the
|
||||
.Va ims_time
|
||||
field is set in
|
||||
.Vt "struct url" ,
|
||||
then
|
||||
.Fn fetchXGetHTTP
|
||||
and
|
||||
.Fn fetchGetHTTP
|
||||
will send a conditional
|
||||
.Li If-Modified-Since
|
||||
HTTP header to only fetch the content if it is newer than
|
||||
.Va ims_time .
|
||||
.Pp
|
||||
Since there seems to be no good way of implementing the HTTP PUT
|
||||
method in a manner consistent with the rest of the
|
||||
.Nm fetch
|
||||
|
@ -46,6 +46,7 @@ struct url {
|
||||
char *doc;
|
||||
off_t offset;
|
||||
size_t length;
|
||||
time_t ims_time;
|
||||
};
|
||||
|
||||
struct url_stat {
|
||||
|
@ -63,6 +63,7 @@ __FBSDID("$FreeBSD$");
|
||||
|
||||
#include <sys/param.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/time.h>
|
||||
|
||||
#include <ctype.h>
|
||||
#include <err.h>
|
||||
@ -92,6 +93,7 @@ __FBSDID("$FreeBSD$");
|
||||
#define HTTP_MOVED_PERM 301
|
||||
#define HTTP_MOVED_TEMP 302
|
||||
#define HTTP_SEE_OTHER 303
|
||||
#define HTTP_NOT_MODIFIED 304
|
||||
#define HTTP_TEMP_REDIRECT 307
|
||||
#define HTTP_NEED_AUTH 401
|
||||
#define HTTP_NEED_PROXY_AUTH 407
|
||||
@ -797,20 +799,23 @@ FILE *
|
||||
http_request(struct url *URL, const char *op, struct url_stat *us,
|
||||
struct url *purl, const char *flags)
|
||||
{
|
||||
char timebuf[80];
|
||||
char hbuf[MAXHOSTNAMELEN + 7], *host;
|
||||
conn_t *conn;
|
||||
struct url *url, *new;
|
||||
int chunked, direct, need_auth, noredirect, verbose;
|
||||
int chunked, direct, ims, need_auth, noredirect, verbose;
|
||||
int e, i, n, val;
|
||||
off_t offset, clength, length, size;
|
||||
time_t mtime;
|
||||
const char *p;
|
||||
FILE *f;
|
||||
hdr_t h;
|
||||
char hbuf[MAXHOSTNAMELEN + 7], *host;
|
||||
struct tm *timestruct;
|
||||
|
||||
direct = CHECK_FLAG('d');
|
||||
noredirect = CHECK_FLAG('A');
|
||||
verbose = CHECK_FLAG('v');
|
||||
ims = CHECK_FLAG('i');
|
||||
|
||||
if (direct && purl) {
|
||||
fetchFreeURL(purl);
|
||||
@ -879,6 +884,14 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
|
||||
op, url->doc);
|
||||
}
|
||||
|
||||
if (ims && url->ims_time) {
|
||||
timestruct = gmtime((time_t *)&url->ims_time);
|
||||
(void)strftime(timebuf, 80, "%a, %d %b %Y %T GMT",
|
||||
timestruct);
|
||||
if (verbose)
|
||||
fetch_info("If-Modified-Since: %s", timebuf);
|
||||
http_cmd(conn, "If-Modified-Since: %s", timebuf);
|
||||
}
|
||||
/* virtual host */
|
||||
http_cmd(conn, "Host: %s", host);
|
||||
|
||||
@ -940,6 +953,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
|
||||
switch (http_get_reply(conn)) {
|
||||
case HTTP_OK:
|
||||
case HTTP_PARTIAL:
|
||||
case HTTP_NOT_MODIFIED:
|
||||
/* fine */
|
||||
break;
|
||||
case HTTP_MOVED_PERM:
|
||||
@ -1074,7 +1088,10 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
|
||||
}
|
||||
|
||||
/* we have a hit or an error */
|
||||
if (conn->err == HTTP_OK || conn->err == HTTP_PARTIAL || HTTP_ERROR(conn->err))
|
||||
if (conn->err == HTTP_OK
|
||||
|| conn->err == HTTP_NOT_MODIFIED
|
||||
|| conn->err == HTTP_PARTIAL
|
||||
|| HTTP_ERROR(conn->err))
|
||||
break;
|
||||
|
||||
/* all other cases: we got a redirect */
|
||||
@ -1102,6 +1119,11 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
|
||||
(long long)offset, (long long)length,
|
||||
(long long)size, (long long)clength));
|
||||
|
||||
if (conn->err == HTTP_NOT_MODIFIED) {
|
||||
http_seterr(HTTP_NOT_MODIFIED);
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
/* check for inconsistencies */
|
||||
if (clength != -1 && length != -1 && clength != length) {
|
||||
http_seterr(HTTP_PROTOCOL_ERROR);
|
||||
|
@ -29,7 +29,7 @@
|
||||
.\"
|
||||
.\" $FreeBSD$
|
||||
.\"
|
||||
.Dd March 11, 2003
|
||||
.Dd December 14, 2008
|
||||
.Dt FETCH 1
|
||||
.Os
|
||||
.Sh NAME
|
||||
@ -39,6 +39,7 @@
|
||||
.Nm
|
||||
.Op Fl 146AadFlMmnPpqRrsUv
|
||||
.Op Fl B Ar bytes
|
||||
.Op Fl i Ar file
|
||||
.Op Fl N Ar file
|
||||
.Op Fl o Ar file
|
||||
.Op Fl S Ar bytes
|
||||
@ -48,6 +49,7 @@
|
||||
.Nm
|
||||
.Op Fl 146AadFlMmnPpqRrsUv
|
||||
.Op Fl B Ar bytes
|
||||
.Op Fl i Ar file
|
||||
.Op Fl N Ar file
|
||||
.Op Fl o Ar file
|
||||
.Op Fl S Ar bytes
|
||||
@ -116,6 +118,12 @@ The file to retrieve is located on the host
|
||||
.Ar host .
|
||||
This option is deprecated and is provided for backward compatibility
|
||||
only.
|
||||
.It Fl i Ar file
|
||||
If-Modified-Since mode: the remote file will only be retrieved if it
|
||||
is newer than
|
||||
.Ar file
|
||||
on the local host.
|
||||
(HTTP only)
|
||||
.It Fl l
|
||||
If the target is a file-scheme URL, make a symbolic link to the target
|
||||
rather than trying to copy it.
|
||||
@ -249,6 +257,12 @@ If multiple URLs are listed on the command line,
|
||||
.Nm
|
||||
will attempt to retrieve each one of them in turn, and will return
|
||||
zero only if they were all successfully retrieved.
|
||||
.Pp
|
||||
If the
|
||||
.Fl i
|
||||
argument is used and the remote file is not newer than the
|
||||
specified file then the command will still return success,
|
||||
although no file is transferred.
|
||||
.Sh SEE ALSO
|
||||
.Xr fetch 3
|
||||
.Sh HISTORY
|
||||
|
@ -60,6 +60,8 @@ int d_flag; /* -d: direct connection */
|
||||
int F_flag; /* -F: restart without checking mtime */
|
||||
char *f_filename; /* -f: file to fetch */
|
||||
char *h_hostname; /* -h: host to fetch from */
|
||||
int i_flag; /* -i: specify input file for mtime comparison */
|
||||
char *i_filename; /* name of input file */
|
||||
int l_flag; /* -l: link rather than copy file: URLs */
|
||||
int m_flag; /* -[Mm]: mirror mode */
|
||||
char *N_filename; /* -N: netrc file name */
|
||||
@ -382,6 +384,14 @@ fetch(char *URL, const char *path)
|
||||
if (A_flag)
|
||||
strcat(flags, "A");
|
||||
timeout = T_secs ? T_secs : http_timeout;
|
||||
if (i_flag) {
|
||||
if (stat(i_filename, &sb)) {
|
||||
warn("%s: stat()", i_filename);
|
||||
goto failure;
|
||||
}
|
||||
url->ims_time = sb.st_mtime;
|
||||
strcat(flags, "i");
|
||||
}
|
||||
}
|
||||
|
||||
/* set the protocol timeout. */
|
||||
@ -449,7 +459,14 @@ fetch(char *URL, const char *path)
|
||||
goto signal;
|
||||
if (f == NULL) {
|
||||
warnx("%s: %s", URL, fetchLastErrString);
|
||||
goto failure;
|
||||
if (i_flag && strcmp(url->scheme, SCHEME_HTTP) == 0
|
||||
&& fetchLastErrCode == FETCH_OK
|
||||
&& strcmp(fetchLastErrString, "Not Modified") == 0) {
|
||||
/* HTTP Not Modified Response, return OK. */
|
||||
r = 0;
|
||||
goto done;
|
||||
} else
|
||||
goto failure;
|
||||
}
|
||||
if (sigint)
|
||||
goto signal;
|
||||
@ -713,9 +730,9 @@ usage(void)
|
||||
{
|
||||
fprintf(stderr, "%s\n%s\n%s\n%s\n",
|
||||
"usage: fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]",
|
||||
" [-T seconds] [-w seconds] URL ...",
|
||||
" [-T seconds] [-w seconds] [-i file] URL ...",
|
||||
" fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]",
|
||||
" [-T seconds] [-w seconds] -h host -f file [-c dir]");
|
||||
" [-T seconds] [-w seconds] [-i file] -h host -f file [-c dir]");
|
||||
}
|
||||
|
||||
|
||||
@ -732,7 +749,7 @@ main(int argc, char *argv[])
|
||||
int c, e, r;
|
||||
|
||||
while ((c = getopt(argc, argv,
|
||||
"146AaB:bc:dFf:Hh:lMmN:nPpo:qRrS:sT:tUvw:")) != -1)
|
||||
"146AaB:bc:dFf:Hh:i:lMmN:nPpo:qRrS:sT:tUvw:")) != -1)
|
||||
switch (c) {
|
||||
case '1':
|
||||
once_flag = 1;
|
||||
@ -777,6 +794,10 @@ main(int argc, char *argv[])
|
||||
case 'h':
|
||||
h_hostname = optarg;
|
||||
break;
|
||||
case 'i':
|
||||
i_flag = 1;
|
||||
i_filename = optarg;
|
||||
break;
|
||||
case 'l':
|
||||
l_flag = 1;
|
||||
break;
|
||||
|
Loading…
Reference in New Issue
Block a user