d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for HTTP 1.1 If-Modified-Since behavior.

Browse Source 
fetch(1) accepts a new argument -i <file> that if specified will cause
the file to be downloaded only if it is more recent than the mtime of
<file>.

libfetch(3) accepts the mtime in the url structure and a flag to
indicate when this behavior is desired.

PR:		bin/87841
Submitted by:	Jukka A. Ukkonen <jau@iki.fi> (partially)
Reviewed by:	des, ru
MFC after:	3 weeks
This commit is contained in:
Murray Stokely 2008-12-15 08:27:44 +00:00
parent 0a420d08ae
commit 7f92799f67
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=186124
5 changed files with 90 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
lib/libfetch/fetch.3
View File

@ -25,7 +25,7 @@
.\"
.\" $FreeBSD$
.\"
.Dd December 18, 2007
.Dd December 14, 2008
.Dt FETCH 3
.Os
.Sh NAME
@ -165,9 +165,16 @@ struct url {
char *doc;
off_t offset;
size_t length;
time_t ims_time;
};
.Ed
.Pp
The
.Va ims_time
field stores the time value for
.Li If-Modified-Since
HTTP requests.
.Pp
The pointer returned by
.Fn fetchMakeURL
or
@ -353,6 +360,22 @@ and
.Fn fetchPutHTTP
will use a direct connection even if a proxy server is defined.
.Pp
If the
.Ql i
(if-modified-since) flag is specified, and
the
.Va ims_time
field is set in
.Vt "struct url" ,
then
.Fn fetchXGetHTTP
and
.Fn fetchGetHTTP
will send a conditional
.Li If-Modified-Since
HTTP header to only fetch the content if it is newer than
.Va ims_time .
.Pp
Since there seems to be no good way of implementing the HTTP PUT
method in a manner consistent with the rest of the
.Nm fetch

1
lib/libfetch/fetch.h
View File

@ -46,6 +46,7 @@ struct url {
char *doc;
off_t offset;
size_t length;
time_t ims_time;
};
struct url_stat {

28
lib/libfetch/http.c
View File

@ -63,6 +63,7 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <ctype.h>
#include <err.h>
@ -92,6 +93,7 @@ __FBSDID("$FreeBSD$");
#define HTTP_MOVED_PERM 301
#define HTTP_MOVED_TEMP 302
#define HTTP_SEE_OTHER 303
#define HTTP_NOT_MODIFIED 304
#define HTTP_TEMP_REDIRECT 307
#define HTTP_NEED_AUTH 401
#define HTTP_NEED_PROXY_AUTH 407
@ -797,20 +799,23 @@ FILE *
http_request(struct url *URL, const char *op, struct url_stat *us,
struct url *purl, const char *flags)
{
char timebuf[80];
char hbuf[MAXHOSTNAMELEN + 7], *host;
conn_t *conn;
struct url *url, *new;
int chunked, direct, need_auth, noredirect, verbose;
int chunked, direct, ims, need_auth, noredirect, verbose;
int e, i, n, val;
off_t offset, clength, length, size;
time_t mtime;
const char *p;
FILE *f;
hdr_t h;
char hbuf[MAXHOSTNAMELEN + 7], *host;
struct tm *timestruct;
direct = CHECK_FLAG('d');
noredirect = CHECK_FLAG('A');
verbose = CHECK_FLAG('v');
ims = CHECK_FLAG('i');
if (direct && purl) {
fetchFreeURL(purl);
@ -879,6 +884,14 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
op, url->doc);
}
if (ims && url->ims_time) {
timestruct = gmtime((time_t *)&url->ims_time);
(void)strftime(timebuf, 80, "%a, %d %b %Y %T GMT",
timestruct);
if (verbose)
fetch_info("If-Modified-Since: %s", timebuf);
http_cmd(conn, "If-Modified-Since: %s", timebuf);
}
/* virtual host */
http_cmd(conn, "Host: %s", host);
@ -940,6 +953,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
switch (http_get_reply(conn)) {
case HTTP_OK:
case HTTP_PARTIAL:
case HTTP_NOT_MODIFIED:
/* fine */
break;
case HTTP_MOVED_PERM:
@ -1074,7 +1088,10 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
}
/* we have a hit or an error */
if (conn->err == HTTP_OK || conn->err == HTTP_PARTIAL || HTTP_ERROR(conn->err))
if (conn->err == HTTP_OK
|| conn->err == HTTP_NOT_MODIFIED
|| conn->err == HTTP_PARTIAL
|| HTTP_ERROR(conn->err))
break;
/* all other cases: we got a redirect */
@ -1102,6 +1119,11 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
(long long)offset, (long long)length,
(long long)size, (long long)clength));
if (conn->err == HTTP_NOT_MODIFIED) {
http_seterr(HTTP_NOT_MODIFIED);
return (NULL);
}
/* check for inconsistencies */
if (clength != -1 && length != -1 && clength != length) {
http_seterr(HTTP_PROTOCOL_ERROR);

16
usr.bin/fetch/fetch.1
View File

@ -29,7 +29,7 @@
.\"
.\" $FreeBSD$
.\"
.Dd March 11, 2003
.Dd December 14, 2008
.Dt FETCH 1
.Os
.Sh NAME
@ -39,6 +39,7 @@
.Nm
.Op Fl 146AadFlMmnPpqRrsUv
.Op Fl B Ar bytes
.Op Fl i Ar file
.Op Fl N Ar file
.Op Fl o Ar file
.Op Fl S Ar bytes
@ -48,6 +49,7 @@
.Nm
.Op Fl 146AadFlMmnPpqRrsUv
.Op Fl B Ar bytes
.Op Fl i Ar file
.Op Fl N Ar file
.Op Fl o Ar file
.Op Fl S Ar bytes
@ -116,6 +118,12 @@ The file to retrieve is located on the host
.Ar host .
This option is deprecated and is provided for backward compatibility
only.
.It Fl i Ar file
If-Modified-Since mode: the remote file will only be retrieved if it
is newer than
.Ar file
on the local host.
(HTTP only)
.It Fl l
If the target is a file-scheme URL, make a symbolic link to the target
rather than trying to copy it.
@ -249,6 +257,12 @@ If multiple URLs are listed on the command line,
.Nm
will attempt to retrieve each one of them in turn, and will return
zero only if they were all successfully retrieved.
.Pp
If the
.Fl i
argument is used and the remote file is not newer than the
specified file then the command will still return success,
although no file is transferred.
.Sh SEE ALSO
.Xr fetch 3
.Sh HISTORY

29
usr.bin/fetch/fetch.c
View File

@ -60,6 +60,8 @@ int d_flag; /* -d: direct connection */
int F_flag; /* -F: restart without checking mtime */
char *f_filename; /* -f: file to fetch */
char *h_hostname; /* -h: host to fetch from */
int i_flag; /* -i: specify input file for mtime comparison */
char *i_filename; /* name of input file */
int l_flag; /* -l: link rather than copy file: URLs */
int m_flag; /* -[Mm]: mirror mode */
char *N_filename; /* -N: netrc file name */
@ -382,6 +384,14 @@ fetch(char *URL, const char *path)
if (A_flag)
strcat(flags, "A");
timeout = T_secs ? T_secs : http_timeout;
if (i_flag) {
if (stat(i_filename, &sb)) {
warn("%s: stat()", i_filename);
goto failure;
}
url->ims_time = sb.st_mtime;
strcat(flags, "i");
}
}
/* set the protocol timeout. */
@ -449,7 +459,14 @@ fetch(char *URL, const char *path)
goto signal;
if (f == NULL) {
warnx("%s: %s", URL, fetchLastErrString);
goto failure;
if (i_flag && strcmp(url->scheme, SCHEME_HTTP) == 0
&& fetchLastErrCode == FETCH_OK
&& strcmp(fetchLastErrString, "Not Modified") == 0) {
/* HTTP Not Modified Response, return OK. */
r = 0;
goto done;
} else
goto failure;
}
if (sigint)
goto signal;
@ -713,9 +730,9 @@ usage(void)
{
fprintf(stderr, "%s\n%s\n%s\n%s\n",
"usage: fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]",
" [-T seconds] [-w seconds] URL ...",
" [-T seconds] [-w seconds] [-i file] URL ...",
" fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]",
" [-T seconds] [-w seconds] -h host -f file [-c dir]");
" [-T seconds] [-w seconds] [-i file] -h host -f file [-c dir]");
}
@ -732,7 +749,7 @@ main(int argc, char *argv[])
int c, e, r;
while ((c = getopt(argc, argv,
"146AaB:bc:dFf:Hh:lMmN:nPpo:qRrS:sT:tUvw:")) != -1)
"146AaB:bc:dFf:Hh:i:lMmN:nPpo:qRrS:sT:tUvw:")) != -1)
switch (c) {
case '1':
once_flag = 1;
@ -777,6 +794,10 @@ main(int argc, char *argv[])
case 'h':
h_hostname = optarg;
break;
case 'i':
i_flag = 1;
i_filename = optarg;
break;
case 'l':
l_flag = 1;
break;