Add file_open(): the underlying system call of openat().

CloudABI purely operates on file descriptor rights (CAP_*). File
descriptor access modes (O_ACCMODE) are emulated on top of rights.

Instead of accepting the traditional flags argument, file_open() copies
in an fdstat_t object that contains the initial rights the descriptor
should have, but also file descriptor flags that should persist after
opening (APPEND, NONBLOCK, *SYNC). Only flags that don't persist (EXCL,
TRUNC, CREAT, DIRECTORY) are passed in as an argument.

file_open() first converts the rights, the persistent flags and the
non-persistent flags to fflags. It then calls into vn_open(). If
successful, it installs the file descriptor with the requested
rights, trimming off rights that don't apply to the type of
the file that has been opened.

Unlike kern_openat(), this function does not support /dev/fd/*. I can't
think of a reason why we need to support this for CloudABI.

Obtained from:	https://github.com/NuxiNL/freebsd
Differential Revision:	https://reviews.freebsd.org/D3235

sys/compat/cloudabi/cloudabi_fd.c

@@ -290,7 +290,7 @@ cloudabi_convert_filetype(const struct file *fp)
 }
 
 /* Removes rights that conflict with the file descriptor type. */
-static void
+void
 cloudabi_remove_conflicting_rights(cloudabi_filetype_t filetype,
     cloudabi_rights_t *base, cloudabi_rights_t *inheriting)
 {
@@ -499,6 +499,25 @@ cloudabi_sys_fd_stat_get(struct thread *td,
 	return (copyout(&fsb, (void *)uap->buf, sizeof(fsb)));
 }
 
+/* Converts CloudABI rights to a set of Capsicum capabilities. */
+int
+cloudabi_convert_rights(cloudabi_rights_t in, cap_rights_t *out)
+{
+
+	cap_rights_init(out);
+#define MAPPING(cloudabi, ...) do {			\
+	if (in & (cloudabi)) {				\
+		cap_rights_set(out, ##__VA_ARGS__);	\
+		in &= ~(cloudabi);			\
+	}						\
+} while (0);
+	RIGHTS_MAPPINGS
+#undef MAPPING
+	if (in != 0)
+		return (ENOTCAPABLE);
+	return (0);
+}
+
 int
 cloudabi_sys_fd_stat_put(struct thread *td,
     struct cloudabi_sys_fd_stat_put_args *uap)

sys/compat/cloudabi/cloudabi_file.c

@@ -197,9 +197,128 @@ int
 cloudabi_sys_file_open(struct thread *td,
     struct cloudabi_sys_file_open_args *uap)
 {
+	cloudabi_fdstat_t fds;
+	cap_rights_t rights;
+	struct filecaps fcaps = {};
+	struct nameidata nd;
+	struct file *fp;
+	struct vnode *vp;
+	char *path;
+	int error, fd, fflags;
+	bool read, write;
 
-	/* Not implemented. */
-	return (ENOSYS);
+	error = copyin(uap->fds, &fds, sizeof(fds));
+	if (error != 0)
+		return (error);
+
+	/* All the requested rights should be set on the descriptor. */
+	error = cloudabi_convert_rights(
+	    fds.fs_rights_base | fds.fs_rights_inheriting, &rights);
+	if (error != 0)
+		return (error);
+	cap_rights_set(&rights, CAP_LOOKUP);
+
+	/* Convert rights to corresponding access mode. */
+	read = (fds.fs_rights_base & (CLOUDABI_RIGHT_FD_READ |
+	    CLOUDABI_RIGHT_FILE_READDIR | CLOUDABI_RIGHT_MEM_MAP_EXEC)) != 0;
+	write = (fds.fs_rights_base & (CLOUDABI_RIGHT_FD_DATASYNC |
+	    CLOUDABI_RIGHT_FD_WRITE | CLOUDABI_RIGHT_FILE_ALLOCATE |
+	    CLOUDABI_RIGHT_FILE_STAT_FPUT_SIZE)) != 0;
+	fflags = read ? write ? FREAD | FWRITE : FREAD : FWRITE;
+
+	/* Convert open flags. */
+	if ((uap->oflags & CLOUDABI_O_CREAT) != 0) {
+		fflags |= O_CREAT;
+		cap_rights_set(&rights, CAP_CREATE);
+	}
+	if ((uap->oflags & CLOUDABI_O_DIRECTORY) != 0)
+		fflags |= O_DIRECTORY;
+	if ((uap->oflags & CLOUDABI_O_EXCL) != 0)
+		fflags |= O_EXCL;
+	if ((uap->oflags & CLOUDABI_O_TRUNC) != 0) {
+		fflags |= O_TRUNC;
+		cap_rights_set(&rights, CAP_FTRUNCATE);
+	}
+	if ((fds.fs_flags & CLOUDABI_FDFLAG_APPEND) != 0)
+		fflags |= O_APPEND;
+	if ((fds.fs_flags & CLOUDABI_FDFLAG_NONBLOCK) != 0)
+		fflags |= O_NONBLOCK;
+	if ((fds.fs_flags & (CLOUDABI_FDFLAG_SYNC | CLOUDABI_FDFLAG_DSYNC |
+	    CLOUDABI_FDFLAG_RSYNC)) != 0) {
+		fflags |= O_SYNC;
+		cap_rights_set(&rights, CAP_FSYNC);
+	}
+	if ((uap->fd & CLOUDABI_LOOKUP_SYMLINK_FOLLOW) == 0)
+		fflags |= O_NOFOLLOW;
+	if (write && (fflags & (O_APPEND | O_TRUNC)) == 0)
+		cap_rights_set(&rights, CAP_SEEK);
+
+	/* Allocate new file descriptor. */
+	error = falloc_noinstall(td, &fp);
+	if (error != 0)
+		return (error);
+	fp->f_flag = fflags & FMASK;
+
+	/* Open path. */
+	error = copyin_path(uap->path, uap->pathlen, &path);
+	if (error != 0) {
+		fdrop(fp, td);
+		return (error);
+	}
+	NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, path, uap->fd,
+	    &rights, td);
+	error = vn_open(&nd, &fflags, 0777 & ~td->td_proc->p_fd->fd_cmask, fp);
+	cloudabi_freestr(path);
+	if (error != 0) {
+		/* Custom operations provided. */
+		if (error == ENXIO && fp->f_ops != &badfileops)
+			goto success;
+
+		/*
+		 * POSIX compliance: return ELOOP in case openat() is
+		 * called on a symbolic link and O_NOFOLLOW is set.
+		 */
+		if (error == EMLINK)
+			error = ELOOP;
+		fdrop(fp, td);
+		return (error);
+	}
+	NDFREE(&nd, NDF_ONLY_PNBUF);
+	filecaps_free(&nd.ni_filecaps);
+	fp->f_vnode = vp = nd.ni_vp;
+
+	/* Install vnode operations if no custom operations are provided. */
+	if (fp->f_ops == &badfileops) {
+		fp->f_seqcount = 1;
+		finit(fp, (fflags & FMASK) | (fp->f_flag & FHASLOCK),
+		    DTYPE_VNODE, vp, &vnops);
+	}
+	VOP_UNLOCK(vp, 0);
+
+	/* Truncate file. */
+	if (fflags & O_TRUNC) {
+		error = fo_truncate(fp, 0, td->td_ucred, td);
+		if (error != 0) {
+			fdrop(fp, td);
+			return (error);
+		}
+	}
+
+success:
+	/* Determine which Capsicum rights to set on the file descriptor. */
+	cloudabi_remove_conflicting_rights(cloudabi_convert_filetype(fp),
+	    &fds.fs_rights_base, &fds.fs_rights_inheriting);
+	cloudabi_convert_rights(fds.fs_rights_base | fds.fs_rights_inheriting,
+	    &fcaps.fc_rights);
+	if (cap_rights_is_set(&fcaps.fc_rights))
+		fcaps.fc_fcntls = CAP_FCNTL_SETFL;
+
+	error = finstall(td, fp, &fd, fflags, &fcaps);
+	fdrop(fp, td);
+	if (error != 0)
+		return (error);
+	td->td_retval[0] = fd;
+	return (0);
 }
 
 /* Converts a FreeBSD directory entry structure and writes it to userspace. */

sys/compat/cloudabi/cloudabi_util.h

@@ -50,6 +50,13 @@ void cloudabi_convert_sockaddr(const struct sockaddr *, socklen_t,
 /* Converts a file descriptor to a CloudABI file descriptor type. */
 cloudabi_filetype_t cloudabi_convert_filetype(const struct file *);
 
+/* Converts CloudABI rights to a set of Capsicum capabilities. */
+int cloudabi_convert_rights(cloudabi_rights_t, cap_rights_t *);
+
+/* Removes rights that conflict with the file descriptor type. */
+void cloudabi_remove_conflicting_rights(cloudabi_filetype_t,
+    cloudabi_rights_t *, cloudabi_rights_t *);
+
 /* Converts a struct timespec to a CloudABI timestamp. */
 int cloudabi_convert_timespec(const struct timespec *, cloudabi_timestamp_t *);