d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

- Implement rsync support in csup, which is chosen as a protocol for regular

Browse Source 
files if the client supports it. The support is implemented with an API to
  operate on files, calculating the rolling checksum and md5 checksum for the
  blocks etc.
- Remove unneeded stream_filter_stop and stream_flush before stream_close.
This commit is contained in:
Ulf Lilleengen 2008-10-25 10:54:28 +00:00
parent 763db4b138
commit 4136ca9478
6 changed files with 469 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contrib/csup/Makefile
View File

@ -10,7 +10,7 @@ PROG= csup
SRCS= attrstack.c config.c detailer.c diff.c fattr.c fixups.c fnmatch.c \
globtree.c idcache.c keyword.c lister.c main.c misc.c mux.c parse.y \
pathcomp.c proto.c status.c stream.c threads.c token.l updater.c \
rcsfile.c rcsparse.c lex.rcs.c
rcsfile.c rcsparse.c lex.rcs.c rsyncfile.c
CFLAGS+= -I. -I${.CURDIR} -g -pthread -DHAVE_FFLAGS -DNDEBUG
WARNS?= 1

2
contrib/csup/config.c
View File

@ -133,8 +133,6 @@ config_init(const char *file, struct coll *override, int overridemask)
coll->co_options &= ~CO_CHECKRCS;
/* In recent versions, we always try to set the file modes. */
coll->co_options |= CO_SETMODE;
/* XXX We don't support the rsync updating algorithm yet. */
coll->co_options |= CO_NORSYNC;
error = config_parse_refusefiles(coll);
if (error)
goto bad;

34
contrib/csup/detailer.c
View File

@ -43,6 +43,7 @@
#include "mux.h"
#include "proto.h"
#include "rcsfile.h"
#include "rsyncfile.h"
#include "status.h"
#include "stream.h"
@ -67,6 +68,7 @@ static int detailer_dofile_co(struct detailer *, struct coll *,
static int detailer_dofile_rcs(struct detailer *, struct coll *,
char *, char *);
static int detailer_dofile_regular(struct detailer *, char *, char *);
static int detailer_dofile_rsync(struct detailer *, char *, char *);
static int detailer_checkrcsattr(struct detailer *, struct coll *, char *,
struct fattr *, int);
int detailer_send_details(struct detailer *, struct coll *, char *,
@ -106,7 +108,6 @@ detailer(void *arg)
xasprintf(&args->errmsg, "Detailer failed: "
"Network read failure: %s", strerror(errno));
}
lprintf(-1, "Error is '%s'\n", args->errmsg);
args->status = STATUS_TRANSIENTFAILURE;
break;
case DETAILER_ERR_WRITE:
@ -343,7 +344,6 @@ detailer_coll(struct detailer *d, struct coll *coll, struct status *st)
lprintf(-1, "Server warning: %s\n", msg);
break;
default:
lprintf(-1, "Line: %s, cmd %s\n", line, cmd);
return (DETAILER_ERR_PROTO);
}
stream_flush(wr);
@ -389,6 +389,33 @@ detailer_dofile_regular(struct detailer *d, char *name, char *path)
return (0);
}
/*
* Tell the server to update a file with the rsync algorithm.
*/
static int
detailer_dofile_rsync(struct detailer *d, char *name, char *path)
{
struct stream *wr;
struct rsyncfile *rf;
rf = rsync_open(path, 0, 1);
if (rf == NULL) {
/* Fallback if we fail in opening it. */
proto_printf(wr, "A %s\n", name);
return (0);
}
wr = d->wr;
proto_printf(wr, "r %s %z %z\n", name, rsync_filesize(rf),
rsync_blocksize(rf));
/* Detail the blocks. */
while (rsync_nextblock(rf) != 0) {
proto_printf(wr, "%s %s\n", rsync_rsum(rf), rsync_blockmd5(rf));
lprintf(-1, "%s %s\n", rsync_rsum(rf), rsync_blockmd5(rf));
}
proto_printf(wr, ".\n");
rsync_close(rf);
}
/*
* Tell the server to update an RCS file that we have, or send it if we don't.
*/
@ -572,6 +599,9 @@ detailer_send_details(struct detailer *d, struct coll *coll, char *name,
} else if (fattr_type(fa) == FT_FILE) {
if (isrcs(name, &len) && !(coll->co_options & CO_NORCS)) {
detailer_dofile_rcs(d, coll, name, path);
} else if (!(coll->co_options & CO_NORSYNC) ||
!globtree_test(coll->co_norsync, name)) {
detailer_dofile_rsync(d, name, path);
} else {
detailer_dofile_regular(d, name, path);
}

228
contrib/csup/rsyncfile.c Normal file
View File

@ -0,0 +1,228 @@
/*-
* Copyright (c) 2008, Ulf Lilleengen <lulf@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#include <errno.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <unistd.h>
#include "misc.h"
#include "fattr.h"
#include "rsyncfile.h"
#define MINBLOCKSIZE 1024
#define MAXBLOCKSIZE (16 * 1024)
#define RECEIVEBUFFERSIZE (15 * 1024)
#define BLOCKINFOSIZE 26
#define SEARCHREGION 10
#define MAXBLOCKS (RECEIVEBUFFERSIZE / BLOCKINFOSIZE)
#define CHAR_OFFSET 3
#define RSUM_SIZE 9
struct rsyncfile {
char *start;
char *buf;
char *end;
size_t blocksize;
size_t fsize;
struct fattr *fa;
int fd;
char *blockptr;
int blocknum;
char blockmd5[MD5_DIGEST_SIZE];
char rsumstr[RSUM_SIZE];
uint32_t rsum;
};
static size_t rsync_chooseblocksize(size_t);
static uint32_t rsync_rollsum(uint8_t *, size_t);
/* Open a file and initialize variable for rsync operation. */
struct rsyncfile *
rsync_open(char *path, size_t blocksize, int read)
{
struct rsyncfile *rf;
struct stat st;
int error;
rf = malloc(sizeof(*rf));
if (rf == NULL)
return (NULL);
error = stat(path, &st);
if (error) {
free(rf);
return (NULL);
}
rf->fsize = st.st_size;
rf->fa = fattr_fromstat(&st);
rf->fd = open(path, read ? O_RDONLY : O_RDWR);
if (rf->fd < 0) {
free(rf);
return (NULL);
}
rf->buf = mmap(0, rf->fsize, PROT_READ, MAP_SHARED, rf->fd, 0);
if (rf->buf == MAP_FAILED) {
free(rf);
return (NULL);
}
rf->start = rf->buf;
rf->end = rf->buf + rf->fsize;
rf->blocksize = (blocksize == 0 ? rsync_chooseblocksize(rf->fsize) :
blocksize);
rf->blockptr = rf->buf;
rf->blocknum = 0;
return (rf);
}
/* Close and free all resources related to an rsync file transfer. */
int
rsync_close(struct rsyncfile *rf)
{
int error;
error = munmap(rf->buf, rf->fsize);
if (error)
return (error);
close(rf->fd);
free(rf);
}
/*
* Choose the most appropriate block size for an rsync transfer. Modeled
* algorithm after cvsup.
*/
static size_t
rsync_chooseblocksize(size_t fsize)
{
size_t bestrem, blocksize, bs, hisearch, losearch, rem;
blocksize = fsize / MAXBLOCKS;
losearch = blocksize - SEARCHREGION;
hisearch = blocksize + SEARCHREGION;
if (losearch < MINBLOCKSIZE) {
losearch = MINBLOCKSIZE;
hisearch = losearch + (2 * SEARCHREGION);
} else if (hisearch > MAXBLOCKSIZE) {
hisearch = MAXBLOCKSIZE;
losearch = hisearch - (2 * SEARCHREGION);
}
bestrem = MAXBLOCKSIZE;
for (bs = losearch; bs <= hisearch;) {
rem = fsize % bs;
if (rem < bestrem) {
bestrem = rem;
blocksize = bs;
}
}
return (bestrem);
}
/* Get the next rsync block of a file. */
int
rsync_nextblock(struct rsyncfile *rf)
{
uint32_t rolling;
char *ptr;
MD5_CTX ctx;
size_t blocksize, i;
if (rf->blockptr >= rf->end)
return (0);
blocksize = min((rf->end - rf->blockptr), rf->blocksize);
/* Calculate MD5 of the block. */
MD5_Init(&ctx);
MD5_Update(&ctx, rf->blockptr, blocksize);
MD5_End(rf->blockmd5, &ctx);
rf->rsum = rsync_rollsum(rf->blockptr, blocksize);
snprintf(rf->rsumstr, RSUM_SIZE, "%x", rf->rsum);
rf->blocknum++;
rf->blockptr += blocksize;
return (1);
}
/* Get the rolling checksum of a file. */
static uint32_t
rsync_rollsum(uint8_t *buf, size_t len)
{
uint32_t a, b;
uint8_t *ptr, *limit;
a = b = 0;
ptr = buf;
limit = buf + len;
while (ptr < limit) {
a += *ptr + CHAR_OFFSET;
b += a;
ptr++;
}
return ((b << 16) | a);
}
/* Get running sum so far. */
char *
rsync_rsum(struct rsyncfile *rf)
{
return (rf->rsumstr);
}
/* Get MD5 of current block. */
char *
rsync_blockmd5(struct rsyncfile *rf)
{
return (rf->blockmd5);
}
/* Accessor for blocksize. */
size_t
rsync_blocksize(struct rsyncfile *rf)
{
return (rf->blocksize);
}
/* Accessor for filesize. */
size_t
rsync_filesize(struct rsyncfile *rf)
{
return (rf->fsize);
}

42
contrib/csup/rsyncfile.h Normal file
View File

@ -0,0 +1,42 @@
/*-
* Copyright (c) 2008, Ulf Lilleengen <lulf@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#ifndef _RSYNCFILE_H_
#define _RSYNCFILE_H_
struct rsyncfile;
struct rsyncfile *rsync_open(char *, size_t, int);
int rsync_nextblock(struct rsyncfile *);
char *rsync_rsum(struct rsyncfile *);
char *rsync_blockmd5(struct rsyncfile *);
int rsync_close(struct rsyncfile *);
size_t rsync_blocksize(struct rsyncfile *);
size_t rsync_filesize(struct rsyncfile *);
#endif /* !_RSYNCFILE_H_ */

174
contrib/csup/updater.c
View File

@ -115,6 +115,8 @@ static int updater_rcsedit(struct updater *, struct file_update *, char *,
char *);
int updater_append_file(struct updater *, struct file_update *,
off_t);
static int updater_rsync(struct updater *, struct file_update *, size_t);
static int updater_read_checkout(struct stream *, struct stream *);
static struct file_update *
fup_new(struct coll *coll, struct status *st)
@ -345,7 +347,7 @@ updater_docoll(struct updater *up, struct file_update *fup, int isfixups)
struct coll *coll;
struct statusrec srbuf, *sr;
struct fattr *rcsattr, *tmp;
char *cmd, *line, *msg, *attr;
char *cmd, *blocksize, *line, *msg, *attr;
char *name, *tag, *date, *revdate;
char *expand, *wantmd5, *revnum;
char *optstr, *rcsopt, *pos;
@ -601,7 +603,30 @@ updater_docoll(struct updater *up, struct file_update *fup, int isfixups)
if (error)
return (error);
break;
case 'r': /* XXX: rsync support. */
case 'r':
name = proto_get_ascii(&line);
attr = proto_get_ascii(&line);
blocksize = proto_get_ascii(&line);
wantmd5 = proto_get_ascii(&line);
if (name == NULL || attr == NULL || blocksize == NULL ||
wantmd5 == NULL) {
return (UPDATER_ERR_PROTO);
}
error = fup_prepare(fup, name, 0);
if (error)
return (UPDATER_ERR_PROTO);
sr->sr_type = SR_FILELIVE;
fup->wantmd5 = xstrdup(wantmd5);
fup->temppath = tempname(fup->destpath);
sr = &fup->srbuf;
sr->sr_file = xstrdup(name);
sr->sr_serverattr = fattr_decode(attr);
if (sr->sr_serverattr == NULL)
return (UPDATER_ERR_PROTO);
error = updater_rsync(up, fup, strtol(blocksize, NULL,
10));
if (error)
return (error);
break;
case 'I':
@ -1457,8 +1482,6 @@ updater_addfile(struct updater *up, struct file_update *fup, char *attr)
remains -= nread;
stream_write(to, buf, nread);
} while (remains > 0);
stream_flush(to);
stream_filter_stop(to);
stream_close(to);
line = stream_getln(up->rd, NULL);
if (line == NULL)
@ -1749,8 +1772,6 @@ updater_rcsedit(struct updater *up, struct file_update *fup, char *name,
}
stream_filter_start(dest, STREAM_FILTER_MD5RCS, md5);
error = rcsfile_write(rf, dest);
stream_flush(dest);
stream_filter_stop(dest);
stream_close(dest);
rcsfile_free(rf);
if (error)
@ -1927,8 +1948,6 @@ updater_append_file(struct updater *up, struct file_update *fup, off_t pos)
bytes -= nread;
stream_write(to, buf, nread);
} while (bytes > 0);
stream_flush(to);
stream_filter_stop(to);
stream_close(to);
line = stream_getln(up->rd, NULL);
@ -1959,3 +1978,142 @@ updater_append_file(struct updater *up, struct file_update *fup, off_t pos)
return (error);
return (0);
}
/*
* Read file data from stream of checkout commands, and write it to the
* destination.
*/
static int
updater_read_checkout(struct stream *src, struct stream *dest)
{
char *line;
size_t size;
ssize_t nbytes;
int error, first;
first = 1;
line = stream_getln(src, &size);
while (line != NULL) {
if (line[size - 1] == '\n')
size--;
if ((size == 1 && *line == '.') ||
(size == 2 && strncmp(line, ".+", 2) == 0))
break;
if (size >= 2 && strncmp(line, "..", 2) == 0) {
size--;
line++;
}
if (!first) {
nbytes = stream_write(dest, "\n", 1);
if (nbytes == -1)
return (UPDATER_ERR_MSG);
}
nbytes = stream_write(dest, line, size);
if (nbytes == -1)
return (UPDATER_ERR_MSG);
line = stream_getln(src, &size);
first = 0;
}
if (line == NULL)
return (UPDATER_ERR_READ);
if (size == 1 && *line == '.') {
nbytes = stream_write(dest, "\n", 1);
if (nbytes == -1)
return (UPDATER_ERR_MSG);
}
return (0);
}
/* Update file using the rsync protocol. */
static int
updater_rsync(struct updater *up, struct file_update *fup, size_t blocksize)
{
struct statusrec *sr;
struct coll *coll;
struct stream *to;
char md5[MD5_DIGEST_SIZE];
char *buf, *line;
int error, orig;
size_t size, blocknum, blockstart, blockcount;
ssize_t nbytes;
sr = &fup->srbuf;
lprintf(1, " Rsync %s\n", fup->coname);
/* First open all files that we are going to work on. */
to = stream_open_file(fup->temppath, O_WRONLY | O_CREAT | O_TRUNC,
0600);
if (to == NULL) {
xasprintf(&up->errmsg, "%s: Cannot create: %s",
fup->temppath, strerror(errno));
return (UPDATER_ERR_MSG);
}
orig = open(fup->destpath, O_RDONLY);
if (orig < 0) {
xasprintf(&up->errmsg, "%s: Cannot open: %s",
fup->destpath, strerror(errno));
return (UPDATER_ERR_MSG);
}
stream_filter_start(to, STREAM_FILTER_MD5, md5);
error = updater_read_checkout(up->rd, to);
if (error) {
xasprintf(&up->errmsg, "%s: Cannot write: %s", fup->temppath,
strerror(errno));
return (error);
}
/* Buffer must contain blocksize bytes. */
buf = xmalloc(blocksize);
/* Done with the initial text, read and write chunks. */
line = stream_getln(up->rd, NULL);
while (line != NULL) {
if (strcmp(line, ".") == 0)
break;
error = UPDATER_ERR_PROTO;
if (proto_get_sizet(&line, &blockstart, 10) != 0)
goto bad;
if (proto_get_sizet(&line, &blockcount, 10) != 0)
goto bad;
/* Read blocks from original file. */
lseek(orig, SEEK_SET, (blocksize * blockstart));
blocknum = 0;
error = UPDATER_ERR_MSG;
for (blocknum = 0; blocknum < blockcount; blocknum++) {
nbytes = read(orig, buf, blocksize);
if (nbytes < 0) {
xasprintf(&up->errmsg, "%s: Cannot read: %s",
fup->destpath, strerror(errno));
goto bad;
}
nbytes = stream_write(to, buf, nbytes);
if (nbytes == -1) {
xasprintf(&up->errmsg, "%s: Cannot write: %s",
fup->temppath, strerror(errno));
goto bad;
}
}
/* Get the remaining text from the server. */
error = updater_read_checkout(up->rd, to);
if (error) {
xasprintf(&up->errmsg, "%s: Cannot write: %s",
fup->temppath, strerror(errno));
goto bad;
}
line = stream_getln(up->rd, NULL);
}
stream_close(to);
close(orig);
sr->sr_clientattr = fattr_frompath(fup->destpath, FATTR_NOFOLLOW);
if (sr->sr_clientattr == NULL)
return (UPDATER_ERR_PROTO);
fattr_override(sr->sr_clientattr, sr->sr_serverattr,
FA_MODTIME | FA_MASK);
error = updater_updatefile(up, fup, md5, 0);
fup->wantmd5 = NULL; /* So that it doesn't get freed. */
bad:
free(buf);
return (error);
}